Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(review): fence the run before inserting its results
rebelicea618ca41f037
diff --git a/backend/store/review_run.go b/backend/store/review_run.goindex eb212ffd88..95674ee580 100644--- a/backend/store/review_run.go+++ b/backend/store/review_run.go@@ -156,8 +156,9 @@ func (s *Store) ClaimAvailableReviewRuns(ctx context.Context, replicaID string)// superseded: the slot was reset or reaped since the claim, and nothing was// written.//-// Lock order: the issue's review comments, then the run slot, the order the-// project purge deletes them in.+// Lock order: the issue's review comments, then the run slot, then the issue+// (the result insert's foreign key), the order the project purge deletes them+// in. The fence therefore runs before the insert.func (s *Store) CompleteReviewRun(ctx context.Context, claimed *ClaimedReviewRun, replicaID string, status storepb.ReviewRun_Status, payload *storepb.ReviewRunPayload, results []*IssueCommentMessage) (bool, error) {switch status {case storepb.ReviewRun_DONE:@@ -202,23 +203,6 @@ func (s *Store) CompleteReviewRun(ctx context.Context, claimed *ClaimedReviewRunif _, err := tx.ExecContext(ctx, query, args...); err != nil {return false, errors.Wrapf(err, "failed to resolve previous review results")}- if len(resultPayloads) > 0 {- // created_at is offset by the ordinal for the same reason as in- // CreateIssueComments: the batch keeps its order.- q := qb.Q().Space(`- INSERT INTO issue_comment (creator, project, issue_id, payload, thread_state, created_at, updated_at)- SELECT NULL, ?, ?, c.payload, 'OPEN', t.at, t.at- FROM unnest(?::JSONB[]) WITH ORDINALITY AS c(payload, ordinality),- LATERAL (SELECT now() + ((c.ordinality - 1) * interval '1 microsecond')) AS t(at)- `, claimed.ProjectID, claimed.IssueUID, resultPayloads)- query, args, err := q.ToSQL()- if err != nil {- return false, errors.Wrapf(err, "failed to build sql")- }- if _, err := tx.ExecContext(ctx, query, args...); err != nil {- return false, errors.Wrapf(err, "failed to post review results")- }- }}q := qb.Q().Space(`@@ -229,12 +213,10 @@ func (s *Store) CompleteReviewRun(ctx context.Context, claimed *ClaimedReviewRun`, status.String(), payloadBytes,claimed.ProjectID, claimed.IssueUID, claimed.Type,storepb.ReviewRun_RUNNING.String(), replicaID, claimed.Attempt)-query, args, err := q.ToSQL()if err != nil {return false, errors.Wrapf(err, "failed to build sql")}-result, err := tx.ExecContext(ctx, query, args...)if err != nil {return false, errors.Wrapf(err, "failed to complete review run")@@ -244,9 +226,28 @@ func (s *Store) CompleteReviewRun(ctx context.Context, claimed *ClaimedReviewRunreturn false, errors.Wrapf(err, "failed to get rows affected")}if rowsAffected == 0 {- // Superseded: the rollback discards the results.+ // Superseded: the rollback discards the resolution.return false, nil}++ if len(resultPayloads) > 0 {+ // created_at is offset by the ordinal for the same reason as in+ // CreateIssueComments: the batch keeps its order.+ q := qb.Q().Space(`+ INSERT INTO issue_comment (creator, project, issue_id, payload, thread_state, created_at, updated_at)+ SELECT NULL, ?, ?, c.payload, 'OPEN', t.at, t.at+ FROM unnest(?::JSONB[]) WITH ORDINALITY AS c(payload, ordinality),+ LATERAL (SELECT now() + ((c.ordinality - 1) * interval '1 microsecond')) AS t(at)+ `, claimed.ProjectID, claimed.IssueUID, resultPayloads)+ query, args, err := q.ToSQL()+ if err != nil {+ return false, errors.Wrapf(err, "failed to build sql")+ }+ if _, err := tx.ExecContext(ctx, query, args...); err != nil {+ return false, errors.Wrapf(err, "failed to post review results")+ }+ }+if err := tx.Commit(); err != nil {return false, errors.Wrapf(err, "failed to commit tx")}diff --git a/backend/store/review_run_lock_order_test.go b/backend/store/review_run_lock_order_test.goindex 3decc004d3..a8ea343405 100644--- a/backend/store/review_run_lock_order_test.go+++ b/backend/store/review_run_lock_order_test.go@@ -11,35 +11,38 @@ import ("github.com/bytebase/bytebase/backend/store")-// A prior OPEN rule result gives the completion's resolve UPDATE a row to-// lock, and the barrier trigger parks the completion right after it, before-// the run slot is touched.+// reviewRunLockOrderSeed seeds a RUNNING rule run claimed by replica-1.const reviewRunLockOrderSeed = `INSERT INTO issue (id, creator, project, name, status, type)VALUES (301, '[email protected]', '%[1]s', 'Issue', 'OPEN', 'DATABASE_CHANGE');- INSERT INTO issue_comment (project, issue_id, payload, thread_state)- VALUES ('%[1]s', 301, '{"comment":"prior","reviewMetadata":{"runType":"RULE","ruleType":"SYNTAX","priority":"P0"}}', 'OPEN');INSERT INTO review_run (project, issue_id, type, attempt, status, replica_id)VALUES ('%[1]s', 301, 'RULE', 0, 'RUNNING', 'replica-1');`+// reviewRunPriorResultSeed adds a prior OPEN rule result, which gives the+// completion's resolve UPDATE a row to lock.+const reviewRunPriorResultSeed = `+ INSERT INTO issue_comment (project, issue_id, payload, thread_state)+ VALUES ('%[1]s', 301, '{"comment":"prior","reviewMetadata":{"runType":"RULE","ruleType":"SYNTAX","priority":"P0"}}', 'OPEN');+`+func reviewRunLockOrderCompletion(fixture *storePostgresFixture, projectID string) (bool, error) {claimed := &store.ClaimedReviewRun{ProjectID: projectID, IssueUID: reviewResultIssue, Type: store.ReviewRunTypeRule, Attempt: 0}return fixture.store.CompleteReviewRun(fixture.ctx, claimed, "replica-1", storepb.ReviewRun_DONE, nil,[]*store.IssueCommentMessage{reviewResult(projectID, "new", storepb.ReviewRuleType_REQUIRE_WHERE)})}-// TestCompleteReviewRunLockOrder parks a DONE completion between its comment-// writes and its slot fence and races the two writers that share those rows:-// a re-run, which must win and leave the completion with nothing written, and-// a project purge, which must wait for the completion rather than deadlock-// with it.+// TestCompleteReviewRunLockOrder parks a DONE completion mid-transaction and+// races the two writers that share its rows: a re-run, which must win and+// leave the completion with nothing written, and a project purge, which must+// wait for the completion rather than deadlock with it, whether the resolve+// UPDATE locked a prior result or the first lock is the run slot's.func TestCompleteReviewRunLockOrder(t *testing.T) {t.Parallel()t.Run("re-run lands mid-completion", func(t *testing.T) {t.Parallel()- fixture := newStorePostgresFixture(t, fmt.Sprintf(reviewRunLockOrderSeed, "default"))+ fixture := newStorePostgresFixture(t, fmt.Sprintf(reviewRunLockOrderSeed+reviewRunPriorResultSeed, "default"))const barrierID = 9971barrier := newMaintenanceLockBarrier(fixture.ctx, t, fixture.db, barrierID)installMaintenanceLockBarrier(t, fixture.db, barrierID, "AFTER UPDATE ON issue_comment FOR EACH STATEMENT")@@ -80,58 +83,77 @@ func TestCompleteReviewRunLockOrder(t *testing.T) {require.Equal(t, int64(1), attempt)})- t.Run("purge waits for the completion", func(t *testing.T) {- t.Parallel()- fixture := newStorePostgresFixture(t, fmt.Sprintf(reviewRunLockOrderSeed, "project-a"))- const barrierID = 9972- barrier := newMaintenanceLockBarrier(fixture.ctx, t, fixture.db, barrierID)- installMaintenanceLockBarrier(t, fixture.db, barrierID, "AFTER UPDATE ON issue_comment FOR EACH STATEMENT")+ // The purge deletes issue_comment, then review_run, then issue. With a+ // prior result it waits on the comment row the completion resolved; with+ // none, on the run slot the completion fenced. Either way the completion+ // never waits on the purge.+ for _, tc := range []struct {+ name string+ seed string+ barrierID int+ trigger string+ prior []reviewComment+ }{+ {+ name: "purge waits on the resolved result",+ seed: reviewRunLockOrderSeed + reviewRunPriorResultSeed,+ barrierID: 9972,+ trigger: "AFTER UPDATE ON issue_comment FOR EACH STATEMENT",+ prior: []reviewComment{{Comment: "prior", ThreadState: "RESOLVED"}},+ },+ {+ name: "purge waits on the fenced run slot",+ seed: reviewRunLockOrderSeed,+ barrierID: 9973,+ trigger: "AFTER INSERT ON issue_comment FOR EACH STATEMENT",+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ t.Parallel()+ fixture := newStorePostgresFixture(t, fmt.Sprintf(tc.seed, "project-a"))+ barrier := newMaintenanceLockBarrier(fixture.ctx, t, fixture.db, tc.barrierID)+ installMaintenanceLockBarrier(t, fixture.db, tc.barrierID, tc.trigger)- type completion struct {- updated bool- err error- }- completed := make(chan completion, 1)- go func() {- updated, err := reviewRunLockOrderCompletion(fixture, "project-a")- completed <- completion{updated: updated, err: err}- }()- waitForMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)- completionPID := maintenanceBarrierWaitingPID(fixture.ctx, t, fixture.db, barrierID)+ type completion struct {+ updated bool+ err error+ }+ completed := make(chan completion, 1)+ go func() {+ updated, err := reviewRunLockOrderCompletion(fixture, "project-a")+ completed <- completion{updated: updated, err: err}+ }()+ waitForMaintenanceBarrier(fixture.ctx, t, fixture.db, tc.barrierID)+ completionPID := maintenanceBarrierWaitingPID(fixture.ctx, t, fixture.db, tc.barrierID)- purged := make(chan error, 1)- go func() {- purged <- fixture.store.DeleteProjects(fixture.ctx, "default", "project-a")- }()- // The purge deletes issue_comment first and waits on the row the- // completion resolved; the completion never waits on the purge.- waitForBackendBlockedByPID(fixture.ctx, t, fixture.db, completionPID)- barrier.release(t)+ purged := make(chan error, 1)+ go func() {+ purged <- fixture.store.DeleteProjects(fixture.ctx, "default", "project-a")+ }()+ waitForBackendBlockedByPID(fixture.ctx, t, fixture.db, completionPID)+ barrier.release(t)- select {- case c := <-completed:- require.NoError(t, c.err)- require.True(t, c.updated, "the completion commits while the purge waits")- case <-time.After(maintenanceLockWait):- t.Fatal("timed out waiting for the completion")- }- // The result the completion inserted is invisible to the purge's- // comment DELETE, so its issue DELETE fails the foreign key: the- // accepted best-effort outcome for a writer racing a purge, which a- // retry then completes.- select {- case err := <-purged:- require.ErrorContains(t, err, "issue_comment_project_issue_id_fkey")- case <-time.After(maintenanceLockWait):- t.Fatal("timed out waiting for the purge")- }- require.Equal(t, []reviewComment{{Comment: "prior", ThreadState: "RESOLVED"}, {Comment: "new", ThreadState: "OPEN"}},- listReviewComments(t, fixture, "project-a"), "the failed purge rolled back; the completion's writes stand")- require.NoError(t, fixture.store.DeleteProjects(fixture.ctx, "default", "project-a"))- require.Empty(t, listReviewComments(t, fixture, "project-a"))- var runs int- require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,- `SELECT COUNT(*) FROM review_run WHERE project = 'project-a'`).Scan(&runs))- require.Zero(t, runs)- })+ select {+ case c := <-completed:+ require.NoError(t, c.err)+ require.True(t, c.updated, "the completion commits while the purge waits")+ case <-time.After(maintenanceLockWait):+ t.Fatal("timed out waiting for the completion")+ }+ // The result the completion inserted is invisible to the purge's+ // comment DELETE, so its issue DELETE fails the foreign key: the+ // accepted best-effort outcome for a writer racing a purge, which+ // a retry then completes.+ select {+ case err := <-purged:+ require.ErrorContains(t, err, "issue_comment_project_issue_id_fkey")+ case <-time.After(maintenanceLockWait):+ t.Fatal("timed out waiting for the purge")+ }+ require.Equal(t, append(tc.prior, reviewComment{Comment: "new", ThreadState: "OPEN"}),+ listReviewComments(t, fixture, "project-a"), "the failed purge rolled back; the completion's writes stand")+ require.NoError(t, fixture.store.DeleteProjects(fixture.ctx, "default", "project-a"))+ require.Empty(t, listReviewComments(t, fixture, "project-a"))+ })+ }}