Open source AI coding agent. Designed for large projects and real world tasks.
Go aiai-agentsai-developer-toolsai-toolsclicommand-linedeveloper-toolsgitgolanggpt-4llmopenaipolyglot-programmingterminalterminal-basedterminal-ui
debugging lock/txn issues, setting aside for a moment
Dane Schneider900cda3c37cd
diff --git a/app/cli/stream_tui/update.go b/app/cli/stream_tui/update.goindex e041cf64..40fb2709 100644--- a/app/cli/stream_tui/update.go+++ b/app/cli/stream_tui/update.go@@ -334,9 +334,8 @@ func (m *streamUIModel) streamUpdate(msg *shared.StreamMessage, deferUIUpdate boreturn m.checkMissingFile(msg)case shared.StreamMessageReply:- // log.Println("streamUpdate received reply message")- // log.Println("Stream message reply:")- // log.Println(spew.Sdump(msg))+ log.Println("Stream message reply:")+ log.Println(spew.Sdump(msg))// ignore empty reply messagesif msg.ReplyChunk == "" {diff --git a/app/server/db/branch_helpers.go b/app/server/db/branch_helpers.goindex b59a8959..5b1d8c18 100644--- a/app/server/db/branch_helpers.go+++ b/app/server/db/branch_helpers.go@@ -170,12 +170,14 @@ func DeleteBranch(orgId, planId, branch string) error {// Ensure that rollback is attempted in case of failuredefer func() {- if err != nil {- if rbErr := tx.Rollback(); rbErr != nil {- log.Printf("transaction rollback error: %v\n", rbErr)+ if rbErr := tx.Rollback(); rbErr != nil {+ if rbErr == sql.ErrTxDone {+ log.Println("attempted to roll back transaction, but it was already committed")} else {- log.Println("transaction rolled back")+ log.Printf("transaction rollback error: %v\n", rbErr)}+ } else {+ log.Println("transaction rolled back")}}()diff --git a/app/server/db/invite_helpers.go b/app/server/db/invite_helpers.goindex c1361548..a9a6c04e 100644--- a/app/server/db/invite_helpers.go+++ b/app/server/db/invite_helpers.go@@ -123,12 +123,14 @@ func AcceptInvite(invite *Invite, inviteeId string) error {// Ensure that rollback is attempted in case of failuredefer func() {- if err != nil {- if rbErr := tx.Rollback(); rbErr != nil {- log.Printf("transaction rollback error: %v\n", rbErr)+ if rbErr := tx.Rollback(); rbErr != nil {+ if rbErr == sql.ErrTxDone {+ log.Println("attempted to roll back transaction, but it was already committed")} else {- log.Println("transaction rolled back")+ log.Printf("transaction rollback error: %v\n", rbErr)}+ } else {+ log.Println("transaction rolled back")}}()diff --git a/app/server/db/locks.go b/app/server/db/locks.goindex e679b086..c68bbcd8 100644--- a/app/server/db/locks.go+++ b/app/server/db/locks.go@@ -1,12 +1,15 @@package dbimport (+ "bytes""context""database/sql""fmt""log""math/rand"+ "runtime""runtime/debug"+ "strconv""strings""time"@@ -38,6 +41,17 @@ func LockRepo(params LockRepoParams) (string, error) {}func lockRepo(params LockRepoParams, numRetry int) (string, error) {+ start := time.Now()+ goroutineID := getGoroutineID() // You'll need to implement this++ log.Printf("[Lock][%d] START lock attempt for plan %s scope %s (retry %d) at %v",+ goroutineID, params.PlanId, params.Scope, numRetry, start)++ defer func() {+ elapsed := time.Since(start)+ log.Printf("[Lock][%d] END lock attempt took %v", goroutineID, elapsed)+ }()+log.Printf("locking repo. orgId: %s | planId: %s | scope: %s | branch %s | numRetry %d \n", params.OrgId, params.PlanId, params.Scope, params.Branch, numRetry)// spew.Dump(params)@@ -69,21 +83,39 @@ func lockRepo(params LockRepoParams, numRetry int) (string, error) {tx, err := Conn.BeginTxx(ctx, &sql.TxOptions{Isolation: sql.LevelRepeatableRead})if err != nil {+ log.Printf("[Lock][%d] Error starting transaction after %v: %v",+ goroutineID, time.Since(start), err)return "", fmt.Errorf("error starting transaction: %v", err)}// Ensure that rollback is attempted in case of failuredefer func() {- if err != nil {- if rbErr := tx.Rollback(); rbErr != nil {- log.Printf("transaction rollback error: %v\n", rbErr)+ if rbErr := tx.Rollback(); rbErr != nil {+ if rbErr == sql.ErrTxDone {+ log.Println("attempted to roll back transaction, but it was already committed")} else {- log.Println("transaction rolled back")+ log.Printf("transaction rollback error: %v\n", rbErr)}+ } else {+ log.Println("transaction rolled back")}}()- query := "SELECT id, org_id, user_id, plan_id, plan_build_id, scope, branch, last_heartbeat_at, created_at FROM repo_locks WHERE plan_id = $1 FOR UPDATE"+ forUpdate := params.Scope == LockScopeWrite++ selectStart := time.Now()+ if forUpdate {+ log.Printf("[Lock][%d] Starting SELECT FOR UPDATE at %v", goroutineID, selectStart)+ } else {+ log.Printf("[Lock][%d] Starting SELECT FOR SHARE at %v", goroutineID, selectStart)+ }++ query := "SELECT id, org_id, user_id, plan_id, plan_build_id, scope, branch, last_heartbeat_at, created_at FROM repo_locks WHERE plan_id = $1"+ if forUpdate {+ query += " FOR UPDATE"+ } else {+ query += " FOR SHARE"+ }queryArgs := []interface{}{planId}var locks []*repoLock@@ -101,6 +133,14 @@ func lockRepo(params LockRepoParams, numRetry int) (string, error) {}log.Println("repo lock query executed")+ if forUpdate {+ log.Printf("[Lock][%d] SELECT FOR UPDATE took %v",+ goroutineID, time.Since(selectStart))+ } else {+ log.Printf("[Lock][%d] SELECT FOR SHARE took %v",+ goroutineID, time.Since(selectStart))+ }+defer rows.Close()var expiredLockIds []string@@ -205,6 +245,9 @@ func lockRepo(params LockRepoParams, numRetry int) (string, error) {log.Println("can acquire lock - inserting new lock")+ insertStart := time.Now()+ log.Printf("[Lock][%d] Starting INSERT at %v", goroutineID, insertStart)+// Insert the new lockvar lockPlanBuildId *stringif planBuildId != "" {@@ -245,6 +288,9 @@ func lockRepo(params LockRepoParams, numRetry int) (string, error) {return "", fmt.Errorf("error inserting new lock: %v", err)}+ log.Printf("[Lock][%d] INSERT took %v",+ goroutineID, time.Since(insertStart))+// check if git lock file exists// remove it if soerr = gitRemoveIndexLockFileIfExists(getPlanDir(orgId, planId))@@ -301,19 +347,19 @@ func lockRepo(params LockRepoParams, numRetry int) (string, error) {cancelFn()return}- }-- // check if 0 rows were updated- rowsAffected, err := res.RowsAffected()- if err != nil {- log.Printf("Error getting rows affected: %v\n", err)- cancelFn()- return- }+ } else {+ // check if 0 rows were updated+ rowsAffected, err := res.RowsAffected()+ if err != nil {+ log.Printf("Error getting rows affected: %v\n", err)+ cancelFn()+ return+ }- if rowsAffected == 0 {- log.Printf("Lock not found: %s | stopping heartbeat loop\n", newLock.Id)- return+ if rowsAffected == 0 {+ log.Printf("Lock not found: %s | stopping heartbeat loop\n", newLock.Id)+ return+ }}time.Sleep(lockHeartbeatInterval)@@ -328,28 +374,71 @@ func lockRepo(params LockRepoParams, numRetry int) (string, error) {}func DeleteRepoLock(id string) error {+ start := time.Now()+ goroutineID := getGoroutineID()++ log.Printf("[Delete][%d] START delete lock %s at %v", goroutineID, id, start)stack := debug.Stack()+ // Format truncated stack excluding runtime framesstackTrace := formatStackTrace(stack)+ log.Printf("DELETE_ATTEMPT | id: %s | stack:\n%s", id, stackTrace)- log.Println()- log.Printf("LOCK_DELETE_ATTEMPT | id: %s | stack:\n%s", id, stackTrace)+ defer func() {+ elapsed := time.Since(start)+ log.Printf("[Delete][%d] END delete lock took %v", goroutineID, elapsed)+ }()++ // Start a new transaction for the delete+ tx, err := Conn.BeginTx(context.Background(), &sql.TxOptions{Isolation: sql.LevelRepeatableRead})+ if err != nil {+ log.Printf("[Delete][%d] Error starting delete transaction: %v", goroutineID, err)+ return fmt.Errorf("error starting delete transaction: %v", err)+ }+ defer tx.Rollback()++ deleteStart := time.Now()+ log.Printf("[Delete][%d] Starting DELETE at %v", goroutineID, deleteStart)query := "DELETE FROM repo_locks WHERE id = $1"- _, err := Conn.Exec(query, id)+ result, err := tx.Exec(query, id)if err != nil {+ log.Printf("[Delete][%d] Error executing delete after %v: %v",+ goroutineID, time.Since(deleteStart), err)return fmt.Errorf("error removing lock: %v", err)}- log.Printf("LOCK_DELETED | id: %s", id)- log.Println()+ // Check if we actually deleted anything+ rowsAffected, err := result.RowsAffected()+ if err != nil {+ log.Printf("[Delete][%d] Error checking rows affected: %v", goroutineID, err)+ } else {+ log.Printf("[Delete][%d] Deleted %d rows", goroutineID, rowsAffected)+ }++ if err = tx.Commit(); err != nil {+ log.Printf("[Delete][%d] Error committing delete: %v", goroutineID, err)+ return fmt.Errorf("error committing delete: %v", err)+ }++ log.Printf("[Delete][%d] DELETE completed in %v",+ goroutineID, time.Since(deleteStart))return nil}func formatStackTrace(stack []byte) string {lines := strings.Split(string(stack), "\n")- // Take first 5 meaningful lines of stack trace- // Skip runtime frames (first 7 lines) and limit to next 5 lines- relevantLines := lines[7:min(len(lines), 12)]+ // Take first 10 meaningful lines of stack trace+ // Skip runtime frames (first 7 lines) and limit to next 10 lines+ relevantLines := lines[7:min(len(lines), 17)]return strings.Join(relevantLines, "\n")}++func getGoroutineID() uint64 {+ b := make([]byte, 64)+ b = b[:runtime.Stack(b, false)]+ b = bytes.TrimPrefix(b, []byte("goroutine "))+ b = b[:bytes.IndexByte(b, ' ')]+ n, _ := strconv.ParseUint(string(b), 10, 64)+ return n+}diff --git a/app/server/db/plan_helpers.go b/app/server/db/plan_helpers.goindex 890b24f0..558cc46b 100644--- a/app/server/db/plan_helpers.go+++ b/app/server/db/plan_helpers.go@@ -1,6 +1,7 @@package dbimport (+ "database/sql""encoding/json""fmt""log"@@ -26,12 +27,14 @@ func CreatePlan(orgId, projectId, userId, name string) (*Plan, error) {// Ensure that rollback is attempted in case of failuredefer func() {- if err != nil {- if rbErr := tx.Rollback(); rbErr != nil {- log.Printf("transaction rollback error: %v\n", rbErr)+ if rbErr := tx.Rollback(); rbErr != nil {+ if rbErr == sql.ErrTxDone {+ log.Println("attempted to roll back transaction, but it was already committed")} else {- log.Println("transaction rolled back")+ log.Printf("transaction rollback error: %v\n", rbErr)}+ } else {+ log.Println("transaction rolled back")}}()diff --git a/app/server/handlers/accounts.go b/app/server/handlers/accounts.goindex 42a6acd4..9382340e 100644--- a/app/server/handlers/accounts.go+++ b/app/server/handlers/accounts.go@@ -1,6 +1,7 @@package handlersimport (+ "database/sql""encoding/json""io""log"@@ -67,12 +68,14 @@ func CreateAccountHandler(w http.ResponseWriter, r *http.Request) {// Ensure that rollback is attempted in case of failuredefer func() {- if err != nil || apiErr != nil {- if rbErr := tx.Rollback(); rbErr != nil {- log.Printf("transaction rollback error: %v\n", rbErr)+ if rbErr := tx.Rollback(); rbErr != nil {+ if rbErr == sql.ErrTxDone {+ log.Println("attempted to roll back transaction, but it was already committed")} else {- log.Println("transaction rolled back")+ log.Printf("transaction rollback error: %v\n", rbErr)}+ } else {+ log.Println("transaction rolled back")}}()diff --git a/app/server/handlers/auth_helpers.go b/app/server/handlers/auth_helpers.goindex ed7cd5b0..ea9db723 100644--- a/app/server/handlers/auth_helpers.go+++ b/app/server/handlers/auth_helpers.go@@ -1,6 +1,7 @@package handlersimport (+ "database/sql""encoding/base64""encoding/json""fmt"@@ -354,12 +355,14 @@ func ValidateAndSignIn(w http.ResponseWriter, r *http.Request, req shared.SignIn// Ensure that rollback is attempted in case of failuredefer func() {- if err != nil {- if rbErr := tx.Rollback(); rbErr != nil {- log.Printf("transaction rollback error: %v\n", rbErr)+ if rbErr := tx.Rollback(); rbErr != nil {+ if rbErr == sql.ErrTxDone {+ log.Println("attempted to roll back transaction, but it was already committed")} else {- log.Println("transaction rolled back")+ log.Printf("transaction rollback error: %v\n", rbErr)}+ } else {+ log.Println("transaction rolled back")}}()diff --git a/app/server/handlers/branches.go b/app/server/handlers/branches.goindex c1aba3f5..bc2c583c 100644--- a/app/server/handlers/branches.go+++ b/app/server/handlers/branches.go@@ -2,6 +2,7 @@ package handlersimport ("context"+ "database/sql""encoding/json""io""log"@@ -127,12 +128,14 @@ func CreateBranchHandler(w http.ResponseWriter, r *http.Request) {// Ensure that rollback is attempted in case of failuredefer func() {- if err != nil {- if rbErr := tx.Rollback(); rbErr != nil {- log.Printf("transaction rollback error: %v\n", rbErr)+ if rbErr := tx.Rollback(); rbErr != nil {+ if rbErr == sql.ErrTxDone {+ log.Println("attempted to roll back transaction, but it was already committed")} else {- log.Println("transaction rolled back")+ log.Printf("transaction rollback error: %v\n", rbErr)}+ } else {+ log.Println("transaction rolled back")}}()diff --git a/app/server/handlers/invites.go b/app/server/handlers/invites.goindex ba619a38..0b9a3e9f 100644--- a/app/server/handlers/invites.go+++ b/app/server/handlers/invites.go@@ -1,6 +1,7 @@package handlersimport (+ "database/sql""encoding/json""log""net/http"@@ -133,12 +134,14 @@ func InviteUserHandler(w http.ResponseWriter, r *http.Request) {// Ensure that rollback is attempted in case of failuredefer func() {- if err != nil {- if rbErr := tx.Rollback(); rbErr != nil {- log.Printf("transaction rollback error: %v\n", rbErr)+ if rbErr := tx.Rollback(); rbErr != nil {+ if rbErr == sql.ErrTxDone {+ log.Println("attempted to roll back transaction, but it was already committed")} else {- log.Println("transaction rolled back")+ log.Printf("transaction rollback error: %v\n", rbErr)}+ } else {+ log.Println("transaction rolled back")}}()diff --git a/app/server/handlers/orgs.go b/app/server/handlers/orgs.goindex 45738484..98fff406 100644--- a/app/server/handlers/orgs.go+++ b/app/server/handlers/orgs.go@@ -1,6 +1,7 @@package handlersimport (+ "database/sql""encoding/json""io""log"@@ -94,12 +95,14 @@ func CreateOrgHandler(w http.ResponseWriter, r *http.Request) {// Ensure that rollback is attempted in case of failuredefer func() {- if err != nil || apiErr != nil {- if rbErr := tx.Rollback(); rbErr != nil {- log.Printf("transaction rollback error: %v\n", rbErr)+ if rbErr := tx.Rollback(); rbErr != nil {+ if rbErr == sql.ErrTxDone {+ log.Println("attempted to roll back transaction, but it was already committed")} else {- log.Println("transaction rolled back")+ log.Printf("transaction rollback error: %v\n", rbErr)}+ } else {+ log.Println("transaction rolled back")}}()diff --git a/app/server/handlers/projects.go b/app/server/handlers/projects.goindex ba69c909..cc815056 100644--- a/app/server/handlers/projects.go+++ b/app/server/handlers/projects.go@@ -1,6 +1,7 @@package handlersimport (+ "database/sql""encoding/json""io""log"@@ -52,12 +53,14 @@ func CreateProjectHandler(w http.ResponseWriter, r *http.Request) {// Ensure that rollback is attempted in case of failuredefer func() {- if err != nil {- if rbErr := tx.Rollback(); rbErr != nil {- log.Printf("transaction rollback error: %v\n", rbErr)+ if rbErr := tx.Rollback(); rbErr != nil {+ if rbErr == sql.ErrTxDone {+ log.Println("attempted to roll back transaction, but it was already committed")} else {- log.Println("transaction rolled back")+ log.Printf("transaction rollback error: %v\n", rbErr)}+ } else {+ log.Println("transaction rolled back")}}()diff --git a/app/server/handlers/proxy_helper.go b/app/server/handlers/proxy_helper.goindex 7e912828..77cb9938 100644--- a/app/server/handlers/proxy_helper.go+++ b/app/server/handlers/proxy_helper.go@@ -96,6 +96,8 @@ func proxyRequest(w http.ResponseWriter, originalRequest *http.Request, url stri}w.WriteHeader(resp.StatusCode)+ log.Printf("Proxy forwarded successfully with status code: %d\n", resp.StatusCode)+// Copy the response bodyif _, err := io.Copy(w, resp.Body); err != nil {log.Printf("Error copying response body: %v\n", err)diff --git a/app/server/handlers/settings.go b/app/server/handlers/settings.goindex 04775183..f53c4416 100644--- a/app/server/handlers/settings.go+++ b/app/server/handlers/settings.go@@ -2,6 +2,7 @@ package handlersimport ("context"+ "database/sql""encoding/json""fmt""log"@@ -209,12 +210,14 @@ func UpdateDefaultSettingsHandler(w http.ResponseWriter, r *http.Request) {}// Ensure that rollback is attempted in case of failuredefer func() {- if err != nil {- if rbErr := tx.Rollback(); rbErr != nil {- log.Printf("transaction rollback error: %v\n", rbErr)+ if rbErr := tx.Rollback(); rbErr != nil {+ if rbErr == sql.ErrTxDone {+ log.Println("attempted to roll back transaction, but it was already committed")} else {- log.Println("transaction rolled back")+ log.Printf("transaction rollback error: %v\n", rbErr)}+ } else {+ log.Println("transaction rolled back")}}()diff --git a/app/server/handlers/users.go b/app/server/handlers/users.goindex c5152c58..6dcbc907 100644--- a/app/server/handlers/users.go+++ b/app/server/handlers/users.go@@ -1,6 +1,7 @@package handlersimport (+ "database/sql""encoding/json""log""net/http"@@ -193,12 +194,14 @@ func DeleteOrgUserHandler(w http.ResponseWriter, r *http.Request) {// Ensure that rollback is attempted in case of failuredefer func() {- if err != nil {- if rbErr := tx.Rollback(); rbErr != nil {- log.Printf("transaction rollback error: %v\n", rbErr)+ if rbErr := tx.Rollback(); rbErr != nil {+ if rbErr == sql.ErrTxDone {+ log.Println("attempted to roll back transaction, but it was already committed")} else {- log.Println("transaction rolled back")+ log.Printf("transaction rollback error: %v\n", rbErr)}+ } else {+ log.Println("transaction rolled back")}}()diff --git a/app/server/types/active_plan.go b/app/server/types/active_plan.goindex b879cdd9..3a63722a 100644--- a/app/server/types/active_plan.go+++ b/app/server/types/active_plan.go@@ -182,7 +182,7 @@ func (ap *ActivePlan) FlushStreamBuffer() {}}-const verboseStreamLogging = false+const verboseStreamLogging = truefunc (ap *ActivePlan) Stream(msg shared.StreamMessage) {if verboseStreamLogging {