Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(review): fence the run before inserting its results

rebelicea618ca41f037
diff --git a/backend/store/review_run.go b/backend/store/review_run.go
index eb212ffd88..95674ee580 100644
--- a/backend/store/review_run.go
+++ b/backend/store/review_run.go
@@ -156,8 +156,9 @@ func (s *Store) ClaimAvailableReviewRuns(ctx context.Context, replicaID string)
// superseded: the slot was reset or reaped since the claim, and nothing was
// written.
//
-// Lock order: the issue's review comments, then the run slot, the order the
-// project purge deletes them in.
+// Lock order: the issue's review comments, then the run slot, then the issue
+// (the result insert's foreign key), the order the project purge deletes them
+// in. The fence therefore runs before the insert.
func (s *Store) CompleteReviewRun(ctx context.Context, claimed *ClaimedReviewRun, replicaID string, status storepb.ReviewRun_Status, payload *storepb.ReviewRunPayload, results []*IssueCommentMessage) (bool, error) {
switch status {
case storepb.ReviewRun_DONE:
@@ -202,23 +203,6 @@ func (s *Store) CompleteReviewRun(ctx context.Context, claimed *ClaimedReviewRun
if _, err := tx.ExecContext(ctx, query, args...); err != nil {
return false, errors.Wrapf(err, "failed to resolve previous review results")
}
- if len(resultPayloads) > 0 {
- // created_at is offset by the ordinal for the same reason as in
- // CreateIssueComments: the batch keeps its order.
- q := qb.Q().Space(`
- INSERT INTO issue_comment (creator, project, issue_id, payload, thread_state, created_at, updated_at)
- SELECT NULL, ?, ?, c.payload, 'OPEN', t.at, t.at
- FROM unnest(?::JSONB[]) WITH ORDINALITY AS c(payload, ordinality),
- LATERAL (SELECT now() + ((c.ordinality - 1) * interval '1 microsecond')) AS t(at)
- `, claimed.ProjectID, claimed.IssueUID, resultPayloads)
- query, args, err := q.ToSQL()
- if err != nil {
- return false, errors.Wrapf(err, "failed to build sql")
- }
- if _, err := tx.ExecContext(ctx, query, args...); err != nil {
- return false, errors.Wrapf(err, "failed to post review results")
- }
- }
}
q := qb.Q().Space(`
@@ -229,12 +213,10 @@ func (s *Store) CompleteReviewRun(ctx context.Context, claimed *ClaimedReviewRun
`, status.String(), payloadBytes,
claimed.ProjectID, claimed.IssueUID, claimed.Type,
storepb.ReviewRun_RUNNING.String(), replicaID, claimed.Attempt)
-
query, args, err := q.ToSQL()
if err != nil {
return false, errors.Wrapf(err, "failed to build sql")
}
-
result, err := tx.ExecContext(ctx, query, args...)
if err != nil {
return false, errors.Wrapf(err, "failed to complete review run")
@@ -244,9 +226,28 @@ func (s *Store) CompleteReviewRun(ctx context.Context, claimed *ClaimedReviewRun
return false, errors.Wrapf(err, "failed to get rows affected")
}
if rowsAffected == 0 {
- // Superseded: the rollback discards the results.
+ // Superseded: the rollback discards the resolution.
return false, nil
}
+
+ if len(resultPayloads) > 0 {
+ // created_at is offset by the ordinal for the same reason as in
+ // CreateIssueComments: the batch keeps its order.
+ q := qb.Q().Space(`
+ INSERT INTO issue_comment (creator, project, issue_id, payload, thread_state, created_at, updated_at)
+ SELECT NULL, ?, ?, c.payload, 'OPEN', t.at, t.at
+ FROM unnest(?::JSONB[]) WITH ORDINALITY AS c(payload, ordinality),
+ LATERAL (SELECT now() + ((c.ordinality - 1) * interval '1 microsecond')) AS t(at)
+ `, claimed.ProjectID, claimed.IssueUID, resultPayloads)
+ query, args, err := q.ToSQL()
+ if err != nil {
+ return false, errors.Wrapf(err, "failed to build sql")
+ }
+ if _, err := tx.ExecContext(ctx, query, args...); err != nil {
+ return false, errors.Wrapf(err, "failed to post review results")
+ }
+ }
+
if err := tx.Commit(); err != nil {
return false, errors.Wrapf(err, "failed to commit tx")
}
diff --git a/backend/store/review_run_lock_order_test.go b/backend/store/review_run_lock_order_test.go
index 3decc004d3..a8ea343405 100644
--- a/backend/store/review_run_lock_order_test.go
+++ b/backend/store/review_run_lock_order_test.go
@@ -11,35 +11,38 @@ import (
"github.com/bytebase/bytebase/backend/store"
)
-// A prior OPEN rule result gives the completion's resolve UPDATE a row to
-// lock, and the barrier trigger parks the completion right after it, before
-// the run slot is touched.
+// reviewRunLockOrderSeed seeds a RUNNING rule run claimed by replica-1.
const reviewRunLockOrderSeed = `
INSERT INTO issue (id, creator, project, name, status, type)
VALUES (301, '[email protected]', '%[1]s', 'Issue', 'OPEN', 'DATABASE_CHANGE');
- INSERT INTO issue_comment (project, issue_id, payload, thread_state)
- VALUES ('%[1]s', 301, '{"comment":"prior","reviewMetadata":{"runType":"RULE","ruleType":"SYNTAX","priority":"P0"}}', 'OPEN');
INSERT INTO review_run (project, issue_id, type, attempt, status, replica_id)
VALUES ('%[1]s', 301, 'RULE', 0, 'RUNNING', 'replica-1');
`
+// reviewRunPriorResultSeed adds a prior OPEN rule result, which gives the
+// completion's resolve UPDATE a row to lock.
+const reviewRunPriorResultSeed = `
+ INSERT INTO issue_comment (project, issue_id, payload, thread_state)
+ VALUES ('%[1]s', 301, '{"comment":"prior","reviewMetadata":{"runType":"RULE","ruleType":"SYNTAX","priority":"P0"}}', 'OPEN');
+`
+
func reviewRunLockOrderCompletion(fixture *storePostgresFixture, projectID string) (bool, error) {
claimed := &store.ClaimedReviewRun{ProjectID: projectID, IssueUID: reviewResultIssue, Type: store.ReviewRunTypeRule, Attempt: 0}
return fixture.store.CompleteReviewRun(fixture.ctx, claimed, "replica-1", storepb.ReviewRun_DONE, nil,
[]*store.IssueCommentMessage{reviewResult(projectID, "new", storepb.ReviewRuleType_REQUIRE_WHERE)})
}
-// TestCompleteReviewRunLockOrder parks a DONE completion between its comment
-// writes and its slot fence and races the two writers that share those rows:
-// a re-run, which must win and leave the completion with nothing written, and
-// a project purge, which must wait for the completion rather than deadlock
-// with it.
+// TestCompleteReviewRunLockOrder parks a DONE completion mid-transaction and
+// races the two writers that share its rows: a re-run, which must win and
+// leave the completion with nothing written, and a project purge, which must
+// wait for the completion rather than deadlock with it, whether the resolve
+// UPDATE locked a prior result or the first lock is the run slot's.
func TestCompleteReviewRunLockOrder(t *testing.T) {
t.Parallel()
t.Run("re-run lands mid-completion", func(t *testing.T) {
t.Parallel()
- fixture := newStorePostgresFixture(t, fmt.Sprintf(reviewRunLockOrderSeed, "default"))
+ fixture := newStorePostgresFixture(t, fmt.Sprintf(reviewRunLockOrderSeed+reviewRunPriorResultSeed, "default"))
const barrierID = 9971
barrier := newMaintenanceLockBarrier(fixture.ctx, t, fixture.db, barrierID)
installMaintenanceLockBarrier(t, fixture.db, barrierID, "AFTER UPDATE ON issue_comment FOR EACH STATEMENT")
@@ -80,58 +83,77 @@ func TestCompleteReviewRunLockOrder(t *testing.T) {
require.Equal(t, int64(1), attempt)
})
- t.Run("purge waits for the completion", func(t *testing.T) {
- t.Parallel()
- fixture := newStorePostgresFixture(t, fmt.Sprintf(reviewRunLockOrderSeed, "project-a"))
- const barrierID = 9972
- barrier := newMaintenanceLockBarrier(fixture.ctx, t, fixture.db, barrierID)
- installMaintenanceLockBarrier(t, fixture.db, barrierID, "AFTER UPDATE ON issue_comment FOR EACH STATEMENT")
+ // The purge deletes issue_comment, then review_run, then issue. With a
+ // prior result it waits on the comment row the completion resolved; with
+ // none, on the run slot the completion fenced. Either way the completion
+ // never waits on the purge.
+ for _, tc := range []struct {
+ name string
+ seed string
+ barrierID int
+ trigger string
+ prior []reviewComment
+ }{
+ {
+ name: "purge waits on the resolved result",
+ seed: reviewRunLockOrderSeed + reviewRunPriorResultSeed,
+ barrierID: 9972,
+ trigger: "AFTER UPDATE ON issue_comment FOR EACH STATEMENT",
+ prior: []reviewComment{{Comment: "prior", ThreadState: "RESOLVED"}},
+ },
+ {
+ name: "purge waits on the fenced run slot",
+ seed: reviewRunLockOrderSeed,
+ barrierID: 9973,
+ trigger: "AFTER INSERT ON issue_comment FOR EACH STATEMENT",
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+ fixture := newStorePostgresFixture(t, fmt.Sprintf(tc.seed, "project-a"))
+ barrier := newMaintenanceLockBarrier(fixture.ctx, t, fixture.db, tc.barrierID)
+ installMaintenanceLockBarrier(t, fixture.db, tc.barrierID, tc.trigger)
- type completion struct {
- updated bool
- err error
- }
- completed := make(chan completion, 1)
- go func() {
- updated, err := reviewRunLockOrderCompletion(fixture, "project-a")
- completed <- completion{updated: updated, err: err}
- }()
- waitForMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)
- completionPID := maintenanceBarrierWaitingPID(fixture.ctx, t, fixture.db, barrierID)
+ type completion struct {
+ updated bool
+ err error
+ }
+ completed := make(chan completion, 1)
+ go func() {
+ updated, err := reviewRunLockOrderCompletion(fixture, "project-a")
+ completed <- completion{updated: updated, err: err}
+ }()
+ waitForMaintenanceBarrier(fixture.ctx, t, fixture.db, tc.barrierID)
+ completionPID := maintenanceBarrierWaitingPID(fixture.ctx, t, fixture.db, tc.barrierID)
- purged := make(chan error, 1)
- go func() {
- purged <- fixture.store.DeleteProjects(fixture.ctx, "default", "project-a")
- }()
- // The purge deletes issue_comment first and waits on the row the
- // completion resolved; the completion never waits on the purge.
- waitForBackendBlockedByPID(fixture.ctx, t, fixture.db, completionPID)
- barrier.release(t)
+ purged := make(chan error, 1)
+ go func() {
+ purged <- fixture.store.DeleteProjects(fixture.ctx, "default", "project-a")
+ }()
+ waitForBackendBlockedByPID(fixture.ctx, t, fixture.db, completionPID)
+ barrier.release(t)
- select {
- case c := <-completed:
- require.NoError(t, c.err)
- require.True(t, c.updated, "the completion commits while the purge waits")
- case <-time.After(maintenanceLockWait):
- t.Fatal("timed out waiting for the completion")
- }
- // The result the completion inserted is invisible to the purge's
- // comment DELETE, so its issue DELETE fails the foreign key: the
- // accepted best-effort outcome for a writer racing a purge, which a
- // retry then completes.
- select {
- case err := <-purged:
- require.ErrorContains(t, err, "issue_comment_project_issue_id_fkey")
- case <-time.After(maintenanceLockWait):
- t.Fatal("timed out waiting for the purge")
- }
- require.Equal(t, []reviewComment{{Comment: "prior", ThreadState: "RESOLVED"}, {Comment: "new", ThreadState: "OPEN"}},
- listReviewComments(t, fixture, "project-a"), "the failed purge rolled back; the completion's writes stand")
- require.NoError(t, fixture.store.DeleteProjects(fixture.ctx, "default", "project-a"))
- require.Empty(t, listReviewComments(t, fixture, "project-a"))
- var runs int
- require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,
- `SELECT COUNT(*) FROM review_run WHERE project = 'project-a'`).Scan(&runs))
- require.Zero(t, runs)
- })
+ select {
+ case c := <-completed:
+ require.NoError(t, c.err)
+ require.True(t, c.updated, "the completion commits while the purge waits")
+ case <-time.After(maintenanceLockWait):
+ t.Fatal("timed out waiting for the completion")
+ }
+ // The result the completion inserted is invisible to the purge's
+ // comment DELETE, so its issue DELETE fails the foreign key: the
+ // accepted best-effort outcome for a writer racing a purge, which
+ // a retry then completes.
+ select {
+ case err := <-purged:
+ require.ErrorContains(t, err, "issue_comment_project_issue_id_fkey")
+ case <-time.After(maintenanceLockWait):
+ t.Fatal("timed out waiting for the purge")
+ }
+ require.Equal(t, append(tc.prior, reviewComment{Comment: "new", ThreadState: "OPEN"}),
+ listReviewComments(t, fixture, "project-a"), "the failed purge rolled back; the completion's writes stand")
+ require.NoError(t, fixture.store.DeleteProjects(fixture.ctx, "default", "project-a"))
+ require.Empty(t, listReviewComments(t, fixture, "project-a"))
+ })
+ }
}