Skip to content

Open source AI coding agent. Designed for large projects and real world tasks.

debugging lock/txn issues, setting aside for a moment

Dane Schneider900cda3c37cd
diff --git a/app/cli/stream_tui/update.go b/app/cli/stream_tui/update.go
index e041cf64..40fb2709 100644
--- a/app/cli/stream_tui/update.go
+++ b/app/cli/stream_tui/update.go
@@ -334,9 +334,8 @@ func (m *streamUIModel) streamUpdate(msg *shared.StreamMessage, deferUIUpdate bo
return m.checkMissingFile(msg)
case shared.StreamMessageReply:
- // log.Println("streamUpdate received reply message")
- // log.Println("Stream message reply:")
- // log.Println(spew.Sdump(msg))
+ log.Println("Stream message reply:")
+ log.Println(spew.Sdump(msg))
// ignore empty reply messages
if msg.ReplyChunk == "" {
diff --git a/app/server/db/branch_helpers.go b/app/server/db/branch_helpers.go
index b59a8959..5b1d8c18 100644
--- a/app/server/db/branch_helpers.go
+++ b/app/server/db/branch_helpers.go
@@ -170,12 +170,14 @@ func DeleteBranch(orgId, planId, branch string) error {
// Ensure that rollback is attempted in case of failure
defer func() {
- if err != nil {
- if rbErr := tx.Rollback(); rbErr != nil {
- log.Printf("transaction rollback error: %v\n", rbErr)
+ if rbErr := tx.Rollback(); rbErr != nil {
+ if rbErr == sql.ErrTxDone {
+ log.Println("attempted to roll back transaction, but it was already committed")
} else {
- log.Println("transaction rolled back")
+ log.Printf("transaction rollback error: %v\n", rbErr)
}
+ } else {
+ log.Println("transaction rolled back")
}
}()
diff --git a/app/server/db/invite_helpers.go b/app/server/db/invite_helpers.go
index c1361548..a9a6c04e 100644
--- a/app/server/db/invite_helpers.go
+++ b/app/server/db/invite_helpers.go
@@ -123,12 +123,14 @@ func AcceptInvite(invite *Invite, inviteeId string) error {
// Ensure that rollback is attempted in case of failure
defer func() {
- if err != nil {
- if rbErr := tx.Rollback(); rbErr != nil {
- log.Printf("transaction rollback error: %v\n", rbErr)
+ if rbErr := tx.Rollback(); rbErr != nil {
+ if rbErr == sql.ErrTxDone {
+ log.Println("attempted to roll back transaction, but it was already committed")
} else {
- log.Println("transaction rolled back")
+ log.Printf("transaction rollback error: %v\n", rbErr)
}
+ } else {
+ log.Println("transaction rolled back")
}
}()
diff --git a/app/server/db/locks.go b/app/server/db/locks.go
index e679b086..c68bbcd8 100644
--- a/app/server/db/locks.go
+++ b/app/server/db/locks.go
@@ -1,12 +1,15 @@
package db
import (
+ "bytes"
"context"
"database/sql"
"fmt"
"log"
"math/rand"
+ "runtime"
"runtime/debug"
+ "strconv"
"strings"
"time"
@@ -38,6 +41,17 @@ func LockRepo(params LockRepoParams) (string, error) {
}
func lockRepo(params LockRepoParams, numRetry int) (string, error) {
+ start := time.Now()
+ goroutineID := getGoroutineID() // You'll need to implement this
+
+ log.Printf("[Lock][%d] START lock attempt for plan %s scope %s (retry %d) at %v",
+ goroutineID, params.PlanId, params.Scope, numRetry, start)
+
+ defer func() {
+ elapsed := time.Since(start)
+ log.Printf("[Lock][%d] END lock attempt took %v", goroutineID, elapsed)
+ }()
+
log.Printf("locking repo. orgId: %s | planId: %s | scope: %s | branch %s | numRetry %d \n", params.OrgId, params.PlanId, params.Scope, params.Branch, numRetry)
// spew.Dump(params)
@@ -69,21 +83,39 @@ func lockRepo(params LockRepoParams, numRetry int) (string, error) {
tx, err := Conn.BeginTxx(ctx, &sql.TxOptions{Isolation: sql.LevelRepeatableRead})
if err != nil {
+ log.Printf("[Lock][%d] Error starting transaction after %v: %v",
+ goroutineID, time.Since(start), err)
return "", fmt.Errorf("error starting transaction: %v", err)
}
// Ensure that rollback is attempted in case of failure
defer func() {
- if err != nil {
- if rbErr := tx.Rollback(); rbErr != nil {
- log.Printf("transaction rollback error: %v\n", rbErr)
+ if rbErr := tx.Rollback(); rbErr != nil {
+ if rbErr == sql.ErrTxDone {
+ log.Println("attempted to roll back transaction, but it was already committed")
} else {
- log.Println("transaction rolled back")
+ log.Printf("transaction rollback error: %v\n", rbErr)
}
+ } else {
+ log.Println("transaction rolled back")
}
}()
- query := "SELECT id, org_id, user_id, plan_id, plan_build_id, scope, branch, last_heartbeat_at, created_at FROM repo_locks WHERE plan_id = $1 FOR UPDATE"
+ forUpdate := params.Scope == LockScopeWrite
+
+ selectStart := time.Now()
+ if forUpdate {
+ log.Printf("[Lock][%d] Starting SELECT FOR UPDATE at %v", goroutineID, selectStart)
+ } else {
+ log.Printf("[Lock][%d] Starting SELECT FOR SHARE at %v", goroutineID, selectStart)
+ }
+
+ query := "SELECT id, org_id, user_id, plan_id, plan_build_id, scope, branch, last_heartbeat_at, created_at FROM repo_locks WHERE plan_id = $1"
+ if forUpdate {
+ query += " FOR UPDATE"
+ } else {
+ query += " FOR SHARE"
+ }
queryArgs := []interface{}{planId}
var locks []*repoLock
@@ -101,6 +133,14 @@ func lockRepo(params LockRepoParams, numRetry int) (string, error) {
}
log.Println("repo lock query executed")
+ if forUpdate {
+ log.Printf("[Lock][%d] SELECT FOR UPDATE took %v",
+ goroutineID, time.Since(selectStart))
+ } else {
+ log.Printf("[Lock][%d] SELECT FOR SHARE took %v",
+ goroutineID, time.Since(selectStart))
+ }
+
defer rows.Close()
var expiredLockIds []string
@@ -205,6 +245,9 @@ func lockRepo(params LockRepoParams, numRetry int) (string, error) {
log.Println("can acquire lock - inserting new lock")
+ insertStart := time.Now()
+ log.Printf("[Lock][%d] Starting INSERT at %v", goroutineID, insertStart)
+
// Insert the new lock
var lockPlanBuildId *string
if planBuildId != "" {
@@ -245,6 +288,9 @@ func lockRepo(params LockRepoParams, numRetry int) (string, error) {
return "", fmt.Errorf("error inserting new lock: %v", err)
}
+ log.Printf("[Lock][%d] INSERT took %v",
+ goroutineID, time.Since(insertStart))
+
// check if git lock file exists
// remove it if so
err = gitRemoveIndexLockFileIfExists(getPlanDir(orgId, planId))
@@ -301,19 +347,19 @@ func lockRepo(params LockRepoParams, numRetry int) (string, error) {
cancelFn()
return
}
- }
-
- // check if 0 rows were updated
- rowsAffected, err := res.RowsAffected()
- if err != nil {
- log.Printf("Error getting rows affected: %v\n", err)
- cancelFn()
- return
- }
+ } else {
+ // check if 0 rows were updated
+ rowsAffected, err := res.RowsAffected()
+ if err != nil {
+ log.Printf("Error getting rows affected: %v\n", err)
+ cancelFn()
+ return
+ }
- if rowsAffected == 0 {
- log.Printf("Lock not found: %s | stopping heartbeat loop\n", newLock.Id)
- return
+ if rowsAffected == 0 {
+ log.Printf("Lock not found: %s | stopping heartbeat loop\n", newLock.Id)
+ return
+ }
}
time.Sleep(lockHeartbeatInterval)
@@ -328,28 +374,71 @@ func lockRepo(params LockRepoParams, numRetry int) (string, error) {
}
func DeleteRepoLock(id string) error {
+ start := time.Now()
+ goroutineID := getGoroutineID()
+
+ log.Printf("[Delete][%d] START delete lock %s at %v", goroutineID, id, start)
stack := debug.Stack()
+ // Format truncated stack excluding runtime frames
stackTrace := formatStackTrace(stack)
+ log.Printf("DELETE_ATTEMPT | id: %s | stack:\n%s", id, stackTrace)
- log.Println()
- log.Printf("LOCK_DELETE_ATTEMPT | id: %s | stack:\n%s", id, stackTrace)
+ defer func() {
+ elapsed := time.Since(start)
+ log.Printf("[Delete][%d] END delete lock took %v", goroutineID, elapsed)
+ }()
+
+ // Start a new transaction for the delete
+ tx, err := Conn.BeginTx(context.Background(), &sql.TxOptions{Isolation: sql.LevelRepeatableRead})
+ if err != nil {
+ log.Printf("[Delete][%d] Error starting delete transaction: %v", goroutineID, err)
+ return fmt.Errorf("error starting delete transaction: %v", err)
+ }
+ defer tx.Rollback()
+
+ deleteStart := time.Now()
+ log.Printf("[Delete][%d] Starting DELETE at %v", goroutineID, deleteStart)
query := "DELETE FROM repo_locks WHERE id = $1"
- _, err := Conn.Exec(query, id)
+ result, err := tx.Exec(query, id)
if err != nil {
+ log.Printf("[Delete][%d] Error executing delete after %v: %v",
+ goroutineID, time.Since(deleteStart), err)
return fmt.Errorf("error removing lock: %v", err)
}
- log.Printf("LOCK_DELETED | id: %s", id)
- log.Println()
+ // Check if we actually deleted anything
+ rowsAffected, err := result.RowsAffected()
+ if err != nil {
+ log.Printf("[Delete][%d] Error checking rows affected: %v", goroutineID, err)
+ } else {
+ log.Printf("[Delete][%d] Deleted %d rows", goroutineID, rowsAffected)
+ }
+
+ if err = tx.Commit(); err != nil {
+ log.Printf("[Delete][%d] Error committing delete: %v", goroutineID, err)
+ return fmt.Errorf("error committing delete: %v", err)
+ }
+
+ log.Printf("[Delete][%d] DELETE completed in %v",
+ goroutineID, time.Since(deleteStart))
return nil
}
func formatStackTrace(stack []byte) string {
lines := strings.Split(string(stack), "\n")
- // Take first 5 meaningful lines of stack trace
- // Skip runtime frames (first 7 lines) and limit to next 5 lines
- relevantLines := lines[7:min(len(lines), 12)]
+ // Take first 10 meaningful lines of stack trace
+ // Skip runtime frames (first 7 lines) and limit to next 10 lines
+ relevantLines := lines[7:min(len(lines), 17)]
return strings.Join(relevantLines, "\n")
}
+
+func getGoroutineID() uint64 {
+ b := make([]byte, 64)
+ b = b[:runtime.Stack(b, false)]
+ b = bytes.TrimPrefix(b, []byte("goroutine "))
+ b = b[:bytes.IndexByte(b, ' ')]
+ n, _ := strconv.ParseUint(string(b), 10, 64)
+ return n
+}
diff --git a/app/server/db/plan_helpers.go b/app/server/db/plan_helpers.go
index 890b24f0..558cc46b 100644
--- a/app/server/db/plan_helpers.go
+++ b/app/server/db/plan_helpers.go
@@ -1,6 +1,7 @@
package db
import (
+ "database/sql"
"encoding/json"
"fmt"
"log"
@@ -26,12 +27,14 @@ func CreatePlan(orgId, projectId, userId, name string) (*Plan, error) {
// Ensure that rollback is attempted in case of failure
defer func() {
- if err != nil {
- if rbErr := tx.Rollback(); rbErr != nil {
- log.Printf("transaction rollback error: %v\n", rbErr)
+ if rbErr := tx.Rollback(); rbErr != nil {
+ if rbErr == sql.ErrTxDone {
+ log.Println("attempted to roll back transaction, but it was already committed")
} else {
- log.Println("transaction rolled back")
+ log.Printf("transaction rollback error: %v\n", rbErr)
}
+ } else {
+ log.Println("transaction rolled back")
}
}()
diff --git a/app/server/handlers/accounts.go b/app/server/handlers/accounts.go
index 42a6acd4..9382340e 100644
--- a/app/server/handlers/accounts.go
+++ b/app/server/handlers/accounts.go
@@ -1,6 +1,7 @@
package handlers
import (
+ "database/sql"
"encoding/json"
"io"
"log"
@@ -67,12 +68,14 @@ func CreateAccountHandler(w http.ResponseWriter, r *http.Request) {
// Ensure that rollback is attempted in case of failure
defer func() {
- if err != nil || apiErr != nil {
- if rbErr := tx.Rollback(); rbErr != nil {
- log.Printf("transaction rollback error: %v\n", rbErr)
+ if rbErr := tx.Rollback(); rbErr != nil {
+ if rbErr == sql.ErrTxDone {
+ log.Println("attempted to roll back transaction, but it was already committed")
} else {
- log.Println("transaction rolled back")
+ log.Printf("transaction rollback error: %v\n", rbErr)
}
+ } else {
+ log.Println("transaction rolled back")
}
}()
diff --git a/app/server/handlers/auth_helpers.go b/app/server/handlers/auth_helpers.go
index ed7cd5b0..ea9db723 100644
--- a/app/server/handlers/auth_helpers.go
+++ b/app/server/handlers/auth_helpers.go
@@ -1,6 +1,7 @@
package handlers
import (
+ "database/sql"
"encoding/base64"
"encoding/json"
"fmt"
@@ -354,12 +355,14 @@ func ValidateAndSignIn(w http.ResponseWriter, r *http.Request, req shared.SignIn
// Ensure that rollback is attempted in case of failure
defer func() {
- if err != nil {
- if rbErr := tx.Rollback(); rbErr != nil {
- log.Printf("transaction rollback error: %v\n", rbErr)
+ if rbErr := tx.Rollback(); rbErr != nil {
+ if rbErr == sql.ErrTxDone {
+ log.Println("attempted to roll back transaction, but it was already committed")
} else {
- log.Println("transaction rolled back")
+ log.Printf("transaction rollback error: %v\n", rbErr)
}
+ } else {
+ log.Println("transaction rolled back")
}
}()
diff --git a/app/server/handlers/branches.go b/app/server/handlers/branches.go
index c1aba3f5..bc2c583c 100644
--- a/app/server/handlers/branches.go
+++ b/app/server/handlers/branches.go
@@ -2,6 +2,7 @@ package handlers
import (
"context"
+ "database/sql"
"encoding/json"
"io"
"log"
@@ -127,12 +128,14 @@ func CreateBranchHandler(w http.ResponseWriter, r *http.Request) {
// Ensure that rollback is attempted in case of failure
defer func() {
- if err != nil {
- if rbErr := tx.Rollback(); rbErr != nil {
- log.Printf("transaction rollback error: %v\n", rbErr)
+ if rbErr := tx.Rollback(); rbErr != nil {
+ if rbErr == sql.ErrTxDone {
+ log.Println("attempted to roll back transaction, but it was already committed")
} else {
- log.Println("transaction rolled back")
+ log.Printf("transaction rollback error: %v\n", rbErr)
}
+ } else {
+ log.Println("transaction rolled back")
}
}()
diff --git a/app/server/handlers/invites.go b/app/server/handlers/invites.go
index ba619a38..0b9a3e9f 100644
--- a/app/server/handlers/invites.go
+++ b/app/server/handlers/invites.go
@@ -1,6 +1,7 @@
package handlers
import (
+ "database/sql"
"encoding/json"
"log"
"net/http"
@@ -133,12 +134,14 @@ func InviteUserHandler(w http.ResponseWriter, r *http.Request) {
// Ensure that rollback is attempted in case of failure
defer func() {
- if err != nil {
- if rbErr := tx.Rollback(); rbErr != nil {
- log.Printf("transaction rollback error: %v\n", rbErr)
+ if rbErr := tx.Rollback(); rbErr != nil {
+ if rbErr == sql.ErrTxDone {
+ log.Println("attempted to roll back transaction, but it was already committed")
} else {
- log.Println("transaction rolled back")
+ log.Printf("transaction rollback error: %v\n", rbErr)
}
+ } else {
+ log.Println("transaction rolled back")
}
}()
diff --git a/app/server/handlers/orgs.go b/app/server/handlers/orgs.go
index 45738484..98fff406 100644
--- a/app/server/handlers/orgs.go
+++ b/app/server/handlers/orgs.go
@@ -1,6 +1,7 @@
package handlers
import (
+ "database/sql"
"encoding/json"
"io"
"log"
@@ -94,12 +95,14 @@ func CreateOrgHandler(w http.ResponseWriter, r *http.Request) {
// Ensure that rollback is attempted in case of failure
defer func() {
- if err != nil || apiErr != nil {
- if rbErr := tx.Rollback(); rbErr != nil {
- log.Printf("transaction rollback error: %v\n", rbErr)
+ if rbErr := tx.Rollback(); rbErr != nil {
+ if rbErr == sql.ErrTxDone {
+ log.Println("attempted to roll back transaction, but it was already committed")
} else {
- log.Println("transaction rolled back")
+ log.Printf("transaction rollback error: %v\n", rbErr)
}
+ } else {
+ log.Println("transaction rolled back")
}
}()
diff --git a/app/server/handlers/projects.go b/app/server/handlers/projects.go
index ba69c909..cc815056 100644
--- a/app/server/handlers/projects.go
+++ b/app/server/handlers/projects.go
@@ -1,6 +1,7 @@
package handlers
import (
+ "database/sql"
"encoding/json"
"io"
"log"
@@ -52,12 +53,14 @@ func CreateProjectHandler(w http.ResponseWriter, r *http.Request) {
// Ensure that rollback is attempted in case of failure
defer func() {
- if err != nil {
- if rbErr := tx.Rollback(); rbErr != nil {
- log.Printf("transaction rollback error: %v\n", rbErr)
+ if rbErr := tx.Rollback(); rbErr != nil {
+ if rbErr == sql.ErrTxDone {
+ log.Println("attempted to roll back transaction, but it was already committed")
} else {
- log.Println("transaction rolled back")
+ log.Printf("transaction rollback error: %v\n", rbErr)
}
+ } else {
+ log.Println("transaction rolled back")
}
}()
diff --git a/app/server/handlers/proxy_helper.go b/app/server/handlers/proxy_helper.go
index 7e912828..77cb9938 100644
--- a/app/server/handlers/proxy_helper.go
+++ b/app/server/handlers/proxy_helper.go
@@ -96,6 +96,8 @@ func proxyRequest(w http.ResponseWriter, originalRequest *http.Request, url stri
}
w.WriteHeader(resp.StatusCode)
+ log.Printf("Proxy forwarded successfully with status code: %d\n", resp.StatusCode)
+
// Copy the response body
if _, err := io.Copy(w, resp.Body); err != nil {
log.Printf("Error copying response body: %v\n", err)
diff --git a/app/server/handlers/settings.go b/app/server/handlers/settings.go
index 04775183..f53c4416 100644
--- a/app/server/handlers/settings.go
+++ b/app/server/handlers/settings.go
@@ -2,6 +2,7 @@ package handlers
import (
"context"
+ "database/sql"
"encoding/json"
"fmt"
"log"
@@ -209,12 +210,14 @@ func UpdateDefaultSettingsHandler(w http.ResponseWriter, r *http.Request) {
}
// Ensure that rollback is attempted in case of failure
defer func() {
- if err != nil {
- if rbErr := tx.Rollback(); rbErr != nil {
- log.Printf("transaction rollback error: %v\n", rbErr)
+ if rbErr := tx.Rollback(); rbErr != nil {
+ if rbErr == sql.ErrTxDone {
+ log.Println("attempted to roll back transaction, but it was already committed")
} else {
- log.Println("transaction rolled back")
+ log.Printf("transaction rollback error: %v\n", rbErr)
}
+ } else {
+ log.Println("transaction rolled back")
}
}()
diff --git a/app/server/handlers/users.go b/app/server/handlers/users.go
index c5152c58..6dcbc907 100644
--- a/app/server/handlers/users.go
+++ b/app/server/handlers/users.go
@@ -1,6 +1,7 @@
package handlers
import (
+ "database/sql"
"encoding/json"
"log"
"net/http"
@@ -193,12 +194,14 @@ func DeleteOrgUserHandler(w http.ResponseWriter, r *http.Request) {
// Ensure that rollback is attempted in case of failure
defer func() {
- if err != nil {
- if rbErr := tx.Rollback(); rbErr != nil {
- log.Printf("transaction rollback error: %v\n", rbErr)
+ if rbErr := tx.Rollback(); rbErr != nil {
+ if rbErr == sql.ErrTxDone {
+ log.Println("attempted to roll back transaction, but it was already committed")
} else {
- log.Println("transaction rolled back")
+ log.Printf("transaction rollback error: %v\n", rbErr)
}
+ } else {
+ log.Println("transaction rolled back")
}
}()
diff --git a/app/server/types/active_plan.go b/app/server/types/active_plan.go
index b879cdd9..3a63722a 100644
--- a/app/server/types/active_plan.go
+++ b/app/server/types/active_plan.go
@@ -182,7 +182,7 @@ func (ap *ActivePlan) FlushStreamBuffer() {
}
}
-const verboseStreamLogging = false
+const verboseStreamLogging = true
func (ap *ActivePlan) Stream(msg shared.StreamMessage) {
if verboseStreamLogging {