Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

34.8 KB
Raw Blame
package tests

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"path"
	"strings"
	"sync"
	"testing"
	"time"

	"connectrpc.com/connect"
	"github.com/google/uuid"
	"github.com/stretchr/testify/require"
	"google.golang.org/protobuf/types/known/durationpb"
	"google.golang.org/protobuf/types/known/fieldmaskpb"

	v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
)

// webhookCollector collects webhook requests for testing.
type webhookCollector struct {
	mu       sync.Mutex
	requests []webhookRequest
}

type webhookRequest struct {
	Method  string
	Headers http.Header
	Body    []byte
	Time    time.Time
}

func (c *webhookCollector) addRequest(r *http.Request) error {
	defer r.Body.Close()
	body, err := io.ReadAll(r.Body)
	if err != nil {
		return err
	}

	c.mu.Lock()
	defer c.mu.Unlock()
	c.requests = append(c.requests, webhookRequest{
		Method:  r.Method,
		Headers: r.Header,
		Body:    body,
		Time:    time.Now(),
	})
	return nil
}

func (c *webhookCollector) getRequests() []webhookRequest {
	c.mu.Lock()
	defer c.mu.Unlock()
	return append([]webhookRequest{}, c.requests...)
}

func (c *webhookCollector) reset() {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.requests = nil
}

// Helper to parse Slack webhook payload.
// Extracts section block texts from attachments[0].blocks (new format).
func parseSlackWebhook(body []byte) (title, description string, err error) {
	var payload map[string]any
	if err := json.Unmarshal(body, &payload); err != nil {
		return "", "", err
	}

	// New format: blocks are inside attachments[0].blocks.
	var blocks []any
	if attachments, ok := payload["attachments"].([]any); ok && len(attachments) > 0 {
		if att, ok := attachments[0].(map[string]any); ok {
			if b, ok := att["blocks"].([]any); ok {
				blocks = b
			}
		}
	}
	if blocks == nil {
		return "", "", nil
	}

	// Collect section block texts. Layout for issue events:
	// [0] event title (bold, with emoji/link)
	// [1] action description (e.g. "Admin created issue X")
	// [2] issue tile (*IssueName*) — bold-wrapped
	// [3] issue description (if present)
	var sectionTexts []string
	for _, block := range blocks {
		blockMap, ok := block.(map[string]any)
		if !ok {
			continue
		}
		if blockMap["type"] != "section" {
			continue
		}
		textMap, ok := blockMap["text"].(map[string]any)
		if !ok {
			continue
		}
		text, ok := textMap["text"].(string)
		if !ok {
			continue
		}
		sectionTexts = append(sectionTexts, text)
	}

	// Walk sections: find bold tile (*text*) as issue title,
	// and the next plain section after the tile as issue description.
	for i, s := range sectionTexts {
		if i == 0 {
			continue // skip event title
		}
		if strings.HasPrefix(s, "*") && strings.HasSuffix(s, "*") && !strings.Contains(s, "|") {
			title = strings.Trim(s, "*")
			// Next non-bold section is the issue description.
			if i+1 < len(sectionTexts) {
				next := sectionTexts[i+1]
				if !strings.HasPrefix(next, "*") || !strings.HasSuffix(next, "*") {
					description = next
				}
			}
			break
		}
	}
	return title, description, nil
}

// TestWebhookIntegration tests webhook functionality.
//
//nolint:tparallel // Subtests share one server and one webhook collector.
func TestWebhookIntegration(t *testing.T) {
	t.Parallel()
	// Localhost is allowed for SLACK and DINGTALK by TestMain, once, for the
	// whole package. Do not set it here: ValidateWebhookURL reads that map
	// unsynchronized from every parallel test that adds a webhook.

	ctx := context.Background()
	ctl, ctx := startWorkspace(ctx, t)

	// Test webhook server.
	//
	// Body MUST be the literal "ok" — the Slack plugin's postMessage
	// (backend/plugin/webhook/slack/slack.go:247-249) treats any other body as
	// a delivery failure and triggers webhook.Retry (up to 3 attempts, 5s
	// exponential backoff). Without an "ok" response, every webhook event
	// would be delivered 1-3 times depending on test timing, causing
	// nondeterministic counts in the requireWebhookCount assertions across
	// the trigger matrix.
	collector := &webhookCollector{}
	webhookServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if err := collector.addRequest(r); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		w.WriteHeader(http.StatusOK)
		_, _ = w.Write([]byte("ok"))
	}))
	// Cleanup, not defer: the parallel subtests below run after this body returns.
	t.Cleanup(webhookServer.Close)

	// Create a single instance for all tests
	pgContainer := sharedPgTarget(t)

	instanceResp, err := ctl.instanceServiceClient.CreateInstance(ctx, connect.NewRequest(&v1pb.CreateInstanceRequest{
		InstanceId: generateRandomString("instance"),
		Instance: &v1pb.Instance{
			SyncDatabases: &v1pb.SyncDatabases{},
			Title:         "test instance",
			Engine:        v1pb.Engine_POSTGRES,
			Environment:   new("environments/prod"),
			Activation:    true,
			DataSources:   []*v1pb.DataSource{pgContainer.adminDataSource()},
		},
	}))
	require.NoError(t, err)
	instance := instanceResp.Msg

	t.Run("TestWebhookOnASavedWebhookPostsToTheStoredURL", func(t *testing.T) {
		// A saved webhook reads back with no URL, so the console cannot send
		// the real one to TestWebhook: it never received it. The button has to
		// keep working anyway, which it does by sending the webhook's name and
		// no URL and letting the server look the stored one up. Delivery
		// landing on this collector is what proves the lookup happened; without
		// it the empty URL would have been rejected as invalid instead.
		collector.reset()

		project := ctl.createTestProject(ctx, t, "webhook-redacted-test")
		addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_CREATED})

		projectResp, err := ctl.projectServiceClient.GetProject(ctx, connect.NewRequest(&v1pb.GetProjectRequest{
			Name: project.Name,
		}))
		require.NoError(t, err)
		require.Len(t, projectResp.Msg.Webhooks, 1)
		saved := projectResp.Msg.Webhooks[0]
		require.Empty(t, saved.Url, "the read path must not hand the URL back")

		result, err := ctl.projectServiceClient.TestWebhook(ctx, connect.NewRequest(&v1pb.TestWebhookRequest{
			Project: project.Name,
			Webhook: saved,
		}))
		require.NoError(t, err)
		require.Empty(t, result.Msg.Error, "the test post has to reach the stored URL")
		require.Eventually(t, func() bool {
			for _, req := range collector.getRequests() {
				if strings.Contains(string(req.Body), "Test webhook") {
					return true
				}
			}
			return false
		}, webhookWaitTimeout, 200*time.Millisecond, "no test notification arrived at the webhook server")

		// A URL the caller typed is still tested as typed, which is what the
		// create form does before there is anything saved to look up.
		typed, err := ctl.projectServiceClient.TestWebhook(ctx, connect.NewRequest(&v1pb.TestWebhookRequest{
			Project: project.Name,
			Webhook: &v1pb.Webhook{
				Type:  v1pb.WebhookType_SLACK,
				Title: "typed by hand",
				Url:   webhookServer.URL,
			},
		}))
		require.NoError(t, err)
		require.Empty(t, typed.Msg.Error)
	})

	t.Run("TestWebhookRefusesToLookUpAWebhookItWasNotGiven", func(t *testing.T) {
		// Looking the stored URL up is what the caller cannot do for
		// themselves, so the three ways of asking for the wrong one are the
		// branches worth pinning. The cross-project case is the one that
		// matters: the ACL runs against the project in the request, not against
		// the project in the webhook name, so without the check a caller with
		// the permission on one project could make the server post through
		// another project's stored webhook.
		collector.reset()

		project := ctl.createTestProject(ctx, t, "webhook-lookup-guard")
		other := ctl.createTestProject(ctx, t, "webhook-lookup-other")
		addWebhookForEvents(ctx, t, ctl, other, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_CREATED})

		otherResp, err := ctl.projectServiceClient.GetProject(ctx, connect.NewRequest(&v1pb.GetProjectRequest{
			Name: other.Name,
		}))
		require.NoError(t, err)
		require.Len(t, otherResp.Msg.Webhooks, 1)

		for _, tc := range []struct {
			name    string
			webhook *v1pb.Webhook
			code    connect.Code
		}{
			{
				name:    "no URL and no name is not a request the server can complete",
				webhook: &v1pb.Webhook{Type: v1pb.WebhookType_SLACK, Title: "nameless"},
				code:    connect.CodeInvalidArgument,
			},
			{
				name:    "a webhook belonging to another project",
				webhook: &v1pb.Webhook{Name: otherResp.Msg.Webhooks[0].Name, Type: v1pb.WebhookType_SLACK, Title: "not mine"},
				code:    connect.CodeInvalidArgument,
			},
			{
				name: "a webhook that does not exist",
				webhook: &v1pb.Webhook{
					Name:  fmt.Sprintf("%s/webhooks/does-not-exist", project.Name),
					Type:  v1pb.WebhookType_SLACK,
					Title: "gone",
				},
				code: connect.CodeNotFound,
			},
		} {
			t.Run(tc.name, func(t *testing.T) {
				_, err := ctl.projectServiceClient.TestWebhook(ctx, connect.NewRequest(&v1pb.TestWebhookRequest{
					Project: project.Name,
					Webhook: tc.webhook,
				}))
				require.Error(t, err)
				require.Equal(t, tc.code, connect.CodeOf(err))
			})
		}

		require.Empty(t, collector.getRequests(),
			"none of those may reach the other project's webhook")

		// The caller picks the type the stored URL gets validated against, and
		// the validator names the URL it rejects, so a deliberate mismatch was
		// a way of reading back a URL the read path withholds.
		_, err = ctl.projectServiceClient.TestWebhook(ctx, connect.NewRequest(&v1pb.TestWebhookRequest{
			Project: other.Name,
			Webhook: &v1pb.Webhook{
				Name:  otherResp.Msg.Webhooks[0].Name,
				Type:  v1pb.WebhookType_GOOGLE_CHAT,
				Title: "tell me where you post",
			},
		}))
		require.Error(t, err)
		require.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err))
		require.NotContains(t, err.Error(), "127.0.0.1",
			"the refusal must not name the stored URL the caller cannot read")
		require.Contains(t, err.Error(), "GOOGLE_CHAT",
			"it names the type instead, which is the half the caller supplied")
	})

	t.Run("TestWebhookReportsAFailureWithoutTheURL", func(t *testing.T) {
		// The posters wrap their failure with the URL they posted to, and this
		// response is the one place that error reaches a caller. Testing a
		// saved webhook that cannot be delivered to would otherwise read the
		// URL back out of the error, which is the read the redaction closed.
		// Port 1 on loopback refuses, so the failure is the transport one every
		// poster wraps.
		const deadURL = "http://127.0.0.1:1/services/unreachable-on-purpose"

		project := ctl.createTestProject(ctx, t, "webhook-error-redaction")
		addWebhookForEvents(ctx, t, ctl, project, deadURL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_CREATED})

		projectResp, err := ctl.projectServiceClient.GetProject(ctx, connect.NewRequest(&v1pb.GetProjectRequest{
			Name: project.Name,
		}))
		require.NoError(t, err)
		require.Len(t, projectResp.Msg.Webhooks, 1)

		result, err := ctl.projectServiceClient.TestWebhook(ctx, connect.NewRequest(&v1pb.TestWebhookRequest{
			Project: project.Name,
			Webhook: projectResp.Msg.Webhooks[0],
		}))
		require.NoError(t, err)
		require.NotEmpty(t, result.Msg.Error, "the post has to have failed for this to be testing anything")
		require.NotContains(t, result.Msg.Error, "unreachable-on-purpose",
			"the failure must not hand back the URL the read path withheld")
		require.NotContains(t, result.Msg.Error, "127.0.0.1:1")
		require.Equal(t, "the test notification could not be delivered to the stored webhook URL",
			result.Msg.Error, "a transport failure has no status code to report")

		// The other half: a URL the caller typed comes back with the poster's
		// message intact, because there is nothing to withhold from the person
		// who supplied it.
		typedFailure, err := ctl.projectServiceClient.TestWebhook(ctx, connect.NewRequest(&v1pb.TestWebhookRequest{
			Project: project.Name,
			Webhook: &v1pb.Webhook{
				Type:  v1pb.WebhookType_SLACK,
				Title: "typed by hand",
				Url:   deadURL,
			},
		}))
		require.NoError(t, err)
		require.Contains(t, typedFailure.Msg.Error, "unreachable-on-purpose")
		require.Contains(t, typedFailure.Msg.Error, "connection refused")
	})

	t.Run("IssueWithPlanWebhookPayload", func(t *testing.T) {
		collector.reset()

		project := ctl.createTestProject(ctx, t, "byt9398-i1")
		addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_CREATED})

		planDesc := "This plan creates a new database with important schema changes"
		planResp, err := ctl.planServiceClient.CreatePlan(ctx, connect.NewRequest(&v1pb.CreatePlanRequest{
			Parent: project.Name,
			Plan: &v1pb.Plan{
				Name:        "Database Migration Plan",
				Description: planDesc,
				Specs: []*v1pb.Plan_Spec{{
					Id: uuid.NewString(),
					Config: &v1pb.Plan_Spec_CreateDatabaseConfig{
						CreateDatabaseConfig: &v1pb.Plan_CreateDatabaseConfig{
							Target:   instance.Name,
							Database: "testdb",
						},
					},
				}},
			},
		}))
		require.NoError(t, err)

		_, err = ctl.issueServiceClient.CreateIssue(ctx, connect.NewRequest(&v1pb.CreateIssueRequest{
			Parent: project.Name,
			Issue: &v1pb.Issue{
				Title:       "Test webhook issue",
				Description: "",
				Type:        v1pb.Issue_DATABASE_CHANGE,
				Plan:        planResp.Msg.Name,
			},
		}))
		require.NoError(t, err)

		// Deterministic wait — replaces the original time.Sleep(5*time.Second).
		waitForWebhookCount(t, collector, project.Name, "Issue created", 1)

		// Find the captured request and verify payload contents.
		var foundCorrectWebhook bool
		for _, req := range collector.getRequests() {
			require.Equal(t, "POST", req.Method)
			require.Contains(t, req.Headers.Get("Content-Type"), "application/json")

			_, desc, err := parseSlackWebhook(req.Body)
			require.NoError(t, err)
			// The webhook payload uses the plan's description (issue-title support is incomplete).
			if desc != planDesc {
				continue
			}
			foundCorrectWebhook = true

			// I1 assertions: event identity and payload contents.
			require.True(t, matchesEvent(req, project.Name, "Issue created"),
				"first webhook section should contain 'Issue created'; got %q",
				firstSlackSectionText(req.Body))

			body := string(req.Body)
			require.Contains(t, body, project.Name, "payload should reference the project resource name")
			require.Contains(t, body, "Test webhook issue", "payload should contain the issue title")
			require.Contains(t, body, fmt.Sprintf("/projects/%s/issues/", path.Base(project.Name)),
				"payload should link to the project's issue resource")
			break
		}
		require.True(t, foundCorrectWebhook, "expected a webhook whose Slack description matches the plan description")
	})

	t.Run("PipelineCompleted_AllTasksDone", func(t *testing.T) {
		t.Parallel()
		project := ctl.createTestProject(ctx, t, "byt9398-c1")
		require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c1_a", ""))
		require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c1_b", ""))
		addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_PIPELINE_FAILED, v1pb.Activity_PIPELINE_COMPLETED})

		plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
			{seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c1_a")},
			{seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c1_b")},
		})
		runAllTasks(ctx, t, ctl, plan)

		waitForWebhookCount(t, collector, project.Name, "Rollout completed", 1)
		requireWebhookCount(t, collector, project.Name, "Rollout failed", 0)
	})

	t.Run("PipelineCompleted_MixedRecovery", func(t *testing.T) {
		t.Parallel()
		project := ctl.createTestProject(ctx, t, "byt9398-c7")
		for _, database := range []string{"byt9398_c7_done", "byt9398_c7_skip", "byt9398_c7_retry", "byt9398_c7_skipfailed"} {
			require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, database, ""))
		}
		addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_PIPELINE_FAILED, v1pb.Activity_PIPELINE_COMPLETED})

		plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
			{seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c7_done")},
			{seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c7_skip")},
			{seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c7_retry")},
			{seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c7_skipfailed")},
		})

		rollout := createRolloutOnly(ctx, t, ctl, plan)
		skipTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c7_skip"))
		runTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c7_done"))
		runTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c7_retry"))
		runTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c7_skipfailed"))

		waitForWebhookCount(t, collector, project.Name, "Rollout failed", 1)
		waitForAllTasksTerminal(ctx, t, ctl, rollout, 60*time.Second)
		requireWebhookCount(t, collector, project.Name, "Rollout completed", 0)

		// Unblock dbRetry only — dbSkipFailed's __force_fail_target table remains
		// absent in its own database, so its retry would still fail.
		unblockFailingTask(t, pgContainer, "byt9398_c7_retry")
		runTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c7_retry"))
		waitForTaskStatus(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c7_retry"), v1pb.Task_DONE, 30*time.Second)
		skipTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c7_skipfailed"))

		waitForWebhookCount(t, collector, project.Name, "Rollout completed", 1)
		requireWebhookCount(t, collector, project.Name, "Rollout failed", 1)
	})

	t.Run("PipelineFailed_SingleTaskFails", func(t *testing.T) {
		t.Parallel()
		project := ctl.createTestProject(ctx, t, "byt9398-f1")
		require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_f1_fail", ""))
		addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_PIPELINE_FAILED})
		plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
			{seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_f1_fail")},
		})
		runAllTasks(ctx, t, ctl, plan)
		waitForWebhookCount(t, collector, project.Name, "Rollout failed", 1)
	})

	t.Run("PipelineFailed_RetryFailsAgain", func(t *testing.T) {
		t.Parallel()
		project := ctl.createTestProject(ctx, t, "byt9398-f3")
		require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_f3_fail", ""))
		addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_PIPELINE_FAILED})
		plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
			{seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_f3_fail")},
		})
		rollout := runAllTasks(ctx, t, ctl, plan)

		waitForWebhookCount(t, collector, project.Name, "Rollout failed", 1)
		waitForAllTasksTerminal(ctx, t, ctl, rollout, 30*time.Second)

		// BatchRunTasks resets the dedup row before enqueuing the retry, so the
		// second failure must re-fire PIPELINE_FAILED. We deliberately do NOT call
		// unblockFailingTask — the retry should fail again.
		retryFailedTasks(ctx, t, ctl, rollout)
		waitForWebhookCount(t, collector, project.Name, "Rollout failed", 2)
	})

	t.Run("IssueCreated_NoLeakageFromOtherEvents", func(t *testing.T) {
		collector.reset()
		project := ctl.createTestProject(ctx, t, "byt9398-i2")
		require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_i2_db", ""))

		disableSelfApproval(ctx, t, ctl, project)
		installWorkspaceApprovalRule(ctx, t, ctl, project, []string{"roles/projectOwner"})
		appr := provisionApprover(ctx, t, ctl, project, "i2", "roles/projectOwner")

		collector.reset()
		addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_CREATED})

		plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
			{seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_i2_db")},
		})
		issue := createIssueForPlan(ctx, t, ctl, project, plan, "I2 issue")

		waitForWebhookCount(t, collector, project.Name, "Issue created", 1)
		waitForIssuePending(ctx, t, ctl, issue)

		// Drive an approval. The webhook is NOT subscribed to ISSUE_APPROVED, so the
		// count must stay at 1. Wait for the issue to actually reach APPROVED before
		// asserting — that proves the approval pipeline ran to completion and any
		// approval-related webhooks have already had their chance to fire.
		approveIssueAs(ctx, t, ctl, issue, appr)
		waitForIssueApproved(ctx, t, ctl, issue)
		requireWebhookCount(t, collector, project.Name, "Issue created", 1)
		requireWebhookCount(t, collector, project.Name, "Issue approved", 0)
	})

	t.Run("IssueApprovalRequested_FiresWhenRequired", func(t *testing.T) {
		collector.reset()
		project := ctl.createTestProject(ctx, t, "byt9398-a1")
		require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_a1_db", ""))

		disableSelfApproval(ctx, t, ctl, project)
		installWorkspaceApprovalRule(ctx, t, ctl, project, []string{"roles/projectOwner"})
		_ = provisionApprover(ctx, t, ctl, project, "a1", "roles/projectOwner")

		collector.reset()
		addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_APPROVAL_REQUESTED})

		plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
			{seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_a1_db")},
		})
		issue := createIssueForPlan(ctx, t, ctl, project, plan, "A1 issue")
		waitForIssuePending(ctx, t, ctl, issue)

		waitForWebhookCount(t, collector, project.Name, "Approval required", 1)
	})

	t.Run("IssueApprovalRequested_ExcludesLastPlanEditor", func(t *testing.T) {
		project := ctl.createTestProject(ctx, t, "bot121-last-editor")
		require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "bot121_last_editor_db", ""))
		_, err := ctl.projectServiceClient.UpdateProject(ctx, connect.NewRequest(&v1pb.UpdateProjectRequest{
			Project: &v1pb.Project{
				Name:                        project.Name,
				AllowLastPlanEditorApproval: false,
			},
			UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"allow_last_plan_editor_approval"}},
		}))
		require.NoError(t, err)

		clearWorkspaceApprovalRules(ctx, t, ctl)
		installWorkspaceApprovalRule(ctx, t, ctl, project, []string{"roles/projectOwner"})
		editor := provisionApprover(ctx, t, ctl, project, "webhook-last-editor", "roles/projectOwner")
		other := provisionApprover(ctx, t, ctl, project, "webhook-other", "roles/projectOwner")
		for _, user := range []struct {
			email string
			phone string
		}{
			{editor.Email, "+8613800000001"},
			{other.Email, "+8613800000002"},
		} {
			_, err := ctl.userServiceClient.UpdateUser(ctx, connect.NewRequest(&v1pb.UpdateUserRequest{
				User:       &v1pb.User{Name: "users/" + user.email, Phone: user.phone},
				UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"phone"}},
			}))
			require.NoError(t, err)
		}

		plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
			{seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "bot121_last_editor_db")},
		})
		withImpersonation(ctx, t, ctl, editor, func() {
			resp, err := ctl.planServiceClient.UpdatePlan(ctx, connect.NewRequest(&v1pb.UpdatePlanRequest{
				Plan:       &v1pb.Plan{Name: plan.Name, Specs: plan.Specs},
				UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"specs"}},
			}))
			require.NoError(t, err)
			plan = resp.Msg
		})

		dingtalkCollector := &webhookCollector{}
		dingtalkServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			if err := dingtalkCollector.addRequest(r); err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			w.Header().Set("Content-Type", "application/json")
			_, _ = w.Write([]byte(`{"errcode":0}`))
		}))
		defer dingtalkServer.Close()
		_, err = ctl.projectServiceClient.AddWebhook(ctx, connect.NewRequest(&v1pb.AddWebhookRequest{
			Project: project.Name,
			Webhook: &v1pb.Webhook{
				Type:              v1pb.WebhookType_DINGTALK,
				Title:             "last-editor-recipient-test",
				Url:               dingtalkServer.URL,
				NotificationTypes: []v1pb.Activity_Type{v1pb.Activity_ISSUE_APPROVAL_REQUESTED},
			},
		}))
		require.NoError(t, err)

		issue := createIssueForPlan(ctx, t, ctl, project, plan, "last editor webhook recipients")
		waitForIssuePending(ctx, t, ctl, issue)
		require.Eventually(t, func() bool {
			return len(dingtalkCollector.getRequests()) == 1
		}, webhookWaitTimeout, 100*time.Millisecond)

		var payload struct {
			Markdown struct {
				Text string `json:"text"`
			} `json:"markdown"`
		}
		require.NoError(t, json.Unmarshal(dingtalkCollector.getRequests()[0].Body, &payload))
		require.NotContains(t, payload.Markdown.Text, "@13800000001")
		require.Contains(t, payload.Markdown.Text, "@13800000002")

		dingtalkCollector.reset()
		_, err = ctl.issueServiceClient.CreateIssue(ctx, connect.NewRequest(&v1pb.CreateIssueRequest{
			Parent: project.Name,
			Issue: &v1pb.Issue{
				Title: "planless webhook recipients",
				Type:  v1pb.Issue_ROLE_GRANT,
				RoleGrant: &v1pb.RoleGrant{
					Role: "roles/projectDeveloper",
					User: "users/" + editor.Email,
				},
			},
		}))
		require.NoError(t, err)
		require.Eventually(t, func() bool {
			return len(dingtalkCollector.getRequests()) == 1
		}, webhookWaitTimeout, 100*time.Millisecond)
		require.NoError(t, json.Unmarshal(dingtalkCollector.getRequests()[0].Body, &payload))
		require.Contains(t, payload.Markdown.Text, "@13800000001")
		require.Contains(t, payload.Markdown.Text, "@13800000002")
	})

	t.Run("IssueApprovalRequested_NotFiredWhenUnused", func(t *testing.T) {
		collector.reset()
		project := ctl.createTestProject(ctx, t, "byt9398-a2")
		// The workspace has a default catch-all approval rule. Clear it so that
		// no rule applies to A2's project, then assert no ISSUE_APPROVAL_REQUESTED fires.
		clearWorkspaceApprovalRules(ctx, t, ctl)
		require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_a2_db", ""))

		collector.reset()
		addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_APPROVAL_REQUESTED})

		plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
			{seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_a2_db")},
		})
		issue := createIssueForPlan(ctx, t, ctl, project, plan, "A2 issue")

		// Wait for the approval-finding pipeline to finish (any terminal status —
		// with no rule applying, the issue auto-resolves). After this wait the
		// runner has had its chance to fire any approval-related webhook, so the
		// absence assertion is no longer racing the runner.
		waitForApprovalFindingDone(ctx, t, ctl, issue)
		requireWebhookCount(t, collector, project.Name, "Approval required", 0)
	})

	t.Run("IssueApprovalRequested_FiresOnceForAnAccessGrant", func(t *testing.T) {
		collector.reset()
		project := ctl.createTestProject(ctx, t, "access-grant-approval")
		require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "access_grant_approval_db", ""))
		require.NoError(t, ctl.allowJustInTimeAccess(ctx, project))

		clearWorkspaceApprovalRules(ctx, t, ctl)
		disableSelfApproval(ctx, t, ctl, project)
		installWorkspaceApprovalRule(ctx, t, ctl, project, []string{"roles/projectOwner"})
		appr := provisionApprover(ctx, t, ctl, project, "access-grant", "roles/projectOwner")

		collector.reset()
		addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{
			v1pb.Activity_ISSUE_APPROVAL_REQUESTED,
			v1pb.Activity_ISSUE_APPROVED,
		})

		grant, err := ctl.accessGrantServiceClient.CreateAccessGrant(ctx, connect.NewRequest(&v1pb.CreateAccessGrantRequest{
			Parent: project.Name,
			AccessGrant: &v1pb.AccessGrant{
				Targets:    []string{dbTargetName(instance, "access_grant_approval_db")},
				Query:      "SELECT 1",
				Reason:     "one read",
				Expiration: &v1pb.AccessGrant_Ttl{Ttl: durationpb.New(time.Hour)},
			},
		}))
		require.NoError(t, err)
		require.Equal(t, v1pb.AccessGrant_PENDING, grant.Msg.Status)
		waitForWebhookCount(t, collector, project.Name, "Approval required", 1)

		issue, err := ctl.issueServiceClient.GetIssue(ctx, connect.NewRequest(&v1pb.GetIssueRequest{Name: grant.Msg.Issue}))
		require.NoError(t, err)
		approveIssueAs(ctx, t, ctl, issue.Msg, appr)
		// The approval's webhook is sent after CreateAccessGrant returned, so
		// waiting for it gives a second "Approval required" time to arrive.
		waitForWebhookCount(t, collector, project.Name, "Issue approved", 1)
		requireWebhookCount(t, collector, project.Name, "Approval required", 1)

		active, err := ctl.accessGrantServiceClient.GetAccessGrant(ctx, connect.NewRequest(&v1pb.GetAccessGrantRequest{Name: grant.Msg.Name}))
		require.NoError(t, err)
		require.Equal(t, v1pb.AccessGrant_ACTIVE, active.Msg.Status)
	})

	t.Run("IssueApproved_SingleStep", func(t *testing.T) {
		collector.reset()
		project := ctl.createTestProject(ctx, t, "byt9398-ap1")
		require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_ap1_db", ""))

		clearWorkspaceApprovalRules(ctx, t, ctl)
		disableSelfApproval(ctx, t, ctl, project)
		installWorkspaceApprovalRule(ctx, t, ctl, project, []string{"roles/projectOwner"})
		appr := provisionApprover(ctx, t, ctl, project, "ap1", "roles/projectOwner")

		collector.reset()
		addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_APPROVED})

		plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
			{seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_ap1_db")},
		})
		issue := createIssueForPlan(ctx, t, ctl, project, plan, "AP1 issue")
		waitForIssuePending(ctx, t, ctl, issue)

		approveIssueAs(ctx, t, ctl, issue, appr)
		waitForWebhookCount(t, collector, project.Name, "Issue approved", 1)
	})

	t.Run("IssueApproved_MultiStepOnlyFiresAtFinal", func(t *testing.T) {
		collector.reset()
		project := ctl.createTestProject(ctx, t, "byt9398-ap2")
		require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_ap2_db", ""))

		clearWorkspaceApprovalRules(ctx, t, ctl)
		disableSelfApproval(ctx, t, ctl, project)
		installWorkspaceApprovalRule(ctx, t, ctl, project,
			[]string{"roles/projectDeveloper", "roles/projectOwner"})
		apprStep1 := provisionApprover(ctx, t, ctl, project, "ap2-step1", "roles/projectDeveloper")
		apprStep2 := provisionApprover(ctx, t, ctl, project, "ap2-step2", "roles/projectOwner")

		collector.reset()
		addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_APPROVED})

		plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
			{seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_ap2_db")},
		})
		issue := createIssueForPlan(ctx, t, ctl, project, plan, "AP2 issue")
		waitForIssuePending(ctx, t, ctl, issue)

		// After step-1 approval the issue must remain PENDING (step 2 still required).
		// waitForIssuePending is a stable post-condition — by the time it returns the
		// approval runner has updated the next-step approver state and the gRPC
		// handler has already decided not to call NotifyIssueApproved (because
		// approved == false). No goroutine for ISSUE_APPROVED was ever spawned, so
		// the count assertion below is race-free.
		approveIssueAs(ctx, t, ctl, issue, apprStep1)
		waitForIssuePending(ctx, t, ctl, issue)
		requireWebhookCount(t, collector, project.Name, "Issue approved", 0)

		approveIssueAs(ctx, t, ctl, issue, apprStep2)
		waitForWebhookCount(t, collector, project.Name, "Issue approved", 1)
	})

	t.Run("IssueSentBack_FiresOnRejection", func(t *testing.T) {
		collector.reset()
		project := ctl.createTestProject(ctx, t, "byt9398-sb1")
		require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_sb1_db", ""))

		clearWorkspaceApprovalRules(ctx, t, ctl)
		disableSelfApproval(ctx, t, ctl, project)
		installWorkspaceApprovalRule(ctx, t, ctl, project, []string{"roles/projectOwner"})
		appr := provisionApprover(ctx, t, ctl, project, "sb1", "roles/projectOwner")

		collector.reset()
		addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_SENT_BACK})

		plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
			{seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_sb1_db")},
		})
		issue := createIssueForPlan(ctx, t, ctl, project, plan, "SB1 issue")
		waitForIssuePending(ctx, t, ctl, issue)

		rejectIssueAs(ctx, t, ctl, issue, appr, "needs more context")
		waitForWebhookCount(t, collector, project.Name, "Issue sent back", 1)
	})

	t.Run("IssueSentBack_ThenReapproved_E2E", func(t *testing.T) {
		collector.reset()
		project := ctl.createTestProject(ctx, t, "byt9398-sb2")
		require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_sb2_db", ""))

		clearWorkspaceApprovalRules(ctx, t, ctl)
		disableSelfApproval(ctx, t, ctl, project)
		installWorkspaceApprovalRule(ctx, t, ctl, project, []string{"roles/projectOwner"})
		appr := provisionApprover(ctx, t, ctl, project, "sb2", "roles/projectOwner")

		collector.reset()
		addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{
			v1pb.Activity_ISSUE_SENT_BACK,
			v1pb.Activity_ISSUE_APPROVED,
			v1pb.Activity_ISSUE_APPROVAL_REQUESTED,
		})

		plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
			{seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_sb2_db")},
		})
		issue := createIssueForPlan(ctx, t, ctl, project, plan, "SB2 issue")
		waitForIssuePending(ctx, t, ctl, issue)

		rejectIssueAs(ctx, t, ctl, issue, appr, "fix and resubmit")
		waitForWebhookCount(t, collector, project.Name, "Issue sent back", 1)

		// Issue creator (default ctl token) re-requests approval. Note: RequestIssue
		// does NOT reset ApprovalFindingDone, so waitForIssuePending would return
		// immediately. Wait for the second "Approval required" webhook instead —
		// RequestIssue dispatches the review workflow's approval-requested event.
		requestIssueAsCreator(ctx, t, ctl, issue, "addressed feedback")
		waitForWebhookCount(t, collector, project.Name, "Approval required", 2)

		approveIssueAs(ctx, t, ctl, issue, appr)
		waitForWebhookCount(t, collector, project.Name, "Issue approved", 1)
	})

	// Regression for the second Codex P1 finding on PR #20212. Empty BatchSkipTasks
	// must be rejected at the API boundary, otherwise it sneaks past the
	// task-parse, environmentSet permission, and store mutation paths and would
	// run ResetPlanWebhookDelivery on a request that did nothing — re-arming the
	// dedup ledger for any plan the caller can name.
	t.Run("BatchSkipTasks_RejectsEmptyTasks", func(t *testing.T) {
		project := ctl.createTestProject(ctx, t, "byt9398-empty-skip")
		// Parent format doesn't matter here; the empty-tasks guard fires first.
		_, err := ctl.rolloutServiceClient.BatchSkipTasks(ctx, connect.NewRequest(&v1pb.BatchSkipTasksRequest{
			Parent: project.Name + "/plans/-/rollouts/-/stages/-",
			Tasks:  []string{},
		}))
		require.Error(t, err)
		require.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err),
			"BatchSkipTasks must reject empty Tasks at the API boundary")
	})
}