Open source AI coding agent. Designed for large projects and real world tasks.
Go aiai-agentsai-developer-toolsai-toolsclicommand-linedeveloper-toolsgitgolanggpt-4llmopenaipolyglot-programmingterminalterminal-basedterminal-ui
better timeouts for locking. don't get diffs in chat mode after stream
Dane Schneider314e7fbf4999
diff --git a/app/cli/plan_exec/tell.go b/app/cli/plan_exec/tell.goindex a7a5bd2b..63d419f2 100644--- a/app/cli/plan_exec/tell.go+++ b/app/cli/plan_exec/tell.go@@ -205,32 +205,34 @@ func TellPlan(term.OutputErrorAndExit("Error starting stream UI: %v", err)}- diffs, apiErr := getDiffs(params)- numDiffs := len(diffs)- if apiErr != nil {- term.OutputErrorAndExit("Error getting plan diffs: %v", apiErr.Msg)- return- }- hasDiffs := numDiffs > 0-- fmt.Println()-- if tellStop && !isChatOnly && hasDiffs {- if hasDiffs {- // term.PrintCmds("", "continue", "diff", "diff --ui", "apply", "reject", "log")- showHotkeyMenu(diffs)- handleHotkey(diffs, params)- } else {- term.PrintCmds("", "continue", "log")- }- } else if !isDebugCmd && !isChatOnly && hasDiffs {- // term.PrintCmds("", "diff", "diff --ui", "apply", "reject", "log")- showHotkeyMenu(diffs)- handleHotkey(diffs, params)- } else if isChatOnly {+ if isChatOnly {if !term.IsRepl {term.PrintCmds("", "tell", "convo", "summary", "log")}+ } else {+ diffs, apiErr := getDiffs(params)+ numDiffs := len(diffs)+ if apiErr != nil {+ term.OutputErrorAndExit("Error getting plan diffs: %v", apiErr.Msg)+ return+ }+ hasDiffs := numDiffs > 0++ fmt.Println()++ if tellStop && hasDiffs {+ if hasDiffs {+ // term.PrintCmds("", "continue", "diff", "diff --ui", "apply", "reject", "log")+ showHotkeyMenu(diffs)+ handleHotkey(diffs, params)+ } else {+ term.PrintCmds("", "continue", "log")+ }+ } else if !isDebugCmd && hasDiffs {+ // term.PrintCmds("", "diff", "diff --ui", "apply", "reject", "log")+ showHotkeyMenu(diffs)+ handleHotkey(diffs, params)+ }}close(done)diff --git a/app/server/db/db.go b/app/server/db/db.goindex 3e05516f..230af8c1 100644--- a/app/server/db/db.go+++ b/app/server/db/db.go@@ -17,6 +17,10 @@ import (var Conn *sqlx.DB+const LockTimeout = 30000+const IdleInTransactionSessionTimeout = 90000+const StatementTimeout = 30000+func Connect() error {var err error@@ -38,9 +42,9 @@ func Connect() error {}if strings.Contains(dbUrl, "?") {- dbUrl += "&statement_timeout=10000&lock_timeout=10000&timezone=UTC&idle_in_transaction_session_timeout=30000"+ dbUrl += fmt.Sprintf("&statement_timeout=%d&lock_timeout=%d&timezone=UTC&idle_in_transaction_session_timeout=%d", StatementTimeout, LockTimeout, IdleInTransactionSessionTimeout)} else {- dbUrl += "?statement_timeout=10000&lock_timeout=10000&timezone=UTC&idle_in_transaction_session_timeout=30000"+ dbUrl += fmt.Sprintf("?statement_timeout=%d&lock_timeout=%d&timezone=UTC&idle_in_transaction_session_timeout=%d", StatementTimeout, LockTimeout, IdleInTransactionSessionTimeout)}Conn, err = sqlx.Connect("postgres", dbUrl)diff --git a/app/server/db/locks.go b/app/server/db/locks.goindex bea0442b..01835506 100644--- a/app/server/db/locks.go+++ b/app/server/db/locks.go@@ -6,6 +6,7 @@ import ("database/sql""fmt""log"+ "math""math/rand""runtime""runtime/debug"@@ -14,12 +15,15 @@ import ("time""github.com/lib/pq"+ "github.com/pkg/errors")-const lockHeartbeatInterval = 700 * time.Millisecond-const lockHeartbeatTimeout = 10 * time.Second+const lockHeartbeatInterval = 3 * time.Second+const lockHeartbeatTimeout = 60 * time.Secondconst maxRetries = 10-const initialRetryInterval = 100 * time.Millisecond+const initialRetryDelay = 100 * time.Millisecond+const backoffFactor = 2.0 // Exponential base+const jitterFraction = 0.3 // e.g. ±30% of the backoff// distributed locking to ensure only one user can write to a plan repo at a time// multiple readers are allowed, but read locks block writes@@ -90,50 +94,10 @@ func lockRepo(params LockRepoParams, numRetry int) (string, error) {return "", fmt.Errorf("invalid lock scope: %s", scope)}- spinRetryWithJitter := func() (string, error) {- // 20 attempts (~6-10 seconds)- if numRetry > 20 {- err := fmt.Errorf("plan is currently being updated by another user or process")- return "", err- }-- backoffBase := 300 * time.Millisecond- jitter := time.Duration(rand.Int63n(int64(200 * time.Millisecond)))- time.Sleep(backoffBase + jitter)- return lockRepo(params, numRetry+1)- }-- handleDeadlockError := func(err error) (string, error) {- if !isDeadlockError(err) {- return "", err- }-- if numRetry > maxRetries {- err = fmt.Errorf("plan is currently being updated by another user")- log.Println("max retries reached on serialization error, returning error")- return "", err- }-- // Calculate exponential backoff with small jitter- backoff := initialRetryInterval * time.Duration(1<<numRetry)- jitter := time.Duration(rand.Float64() * float64(backoff) * 0.1) // 10% jitter- wait := backoff + jitter-- log.Printf("Serialization or deadlock error, retrying transaction after %v: %v\n", wait, err)-- select {- case <-ctx.Done():- return "", fmt.Errorf("context finished during retry transaction")- case <-time.After(wait):- log.Printf("Retrying transaction after %v\n", wait)- return lockRepo(params, numRetry+1)- }- }-tx, err := Conn.BeginTxx(ctx, &sql.TxOptions{Isolation: sql.LevelRepeatableRead})if err != nil {- log.Printf("[Lock][%d] Error starting transaction after %v: %v",- goroutineID, time.Since(start), err)+ log.Printf("[Lock][%d] Error starting transaction %v",+ goroutineID, err)return "", fmt.Errorf("error starting transaction: %v", err)}@@ -174,7 +138,9 @@ func lockRepo(params LockRepoParams, numRetry int) (string, error) {_, err = tx.Exec(lockablePlanIdQuery, planId)if err != nil {- return handleDeadlockError(err)+ return retryWithExponentialBackoff(params.Ctx, err, numRetry, func(nextAttempt int) (string, error) {+ return lockRepo(params, nextAttempt)+ })}query := "SELECT id, org_id, user_id, plan_id, plan_build_id, scope, branch, last_heartbeat_at, created_at FROM repo_locks WHERE plan_id = $1"@@ -189,7 +155,9 @@ func lockRepo(params LockRepoParams, numRetry int) (string, error) {log.Println("obtaining repo lock with query")repoLockRows, err := tx.Query(query, queryArgs...)if err != nil {- return handleDeadlockError(err)+ return retryWithExponentialBackoff(params.Ctx, err, numRetry, func(nextAttempt int) (string, error) {+ return lockRepo(params, nextAttempt)+ })}log.Println("repo lock query executed")@@ -220,6 +188,10 @@ func lockRepo(params LockRepoParams, numRetry int) (string, error) {}}+ if err := repoLockRows.Err(); err != nil {+ return "", fmt.Errorf("error iterating over repo locks: %v", err)+ }+if len(expiredLockIds) > 0 {log.Printf("deleting expired locks: %v", expiredLockIds)@@ -267,7 +239,9 @@ func lockRepo(params LockRepoParams, numRetry int) (string, error) {if !canAcquire {log.Println("can't acquire lock.", "numRetry:", numRetry)- return spinRetryWithJitter()+ return retryWithExponentialBackoff(params.Ctx, err, numRetry, func(nextAttempt int) (string, error) {+ return lockRepo(params, nextAttempt)+ })}log.Println("can acquire lock - inserting new lock")@@ -318,8 +292,15 @@ func lockRepo(params LockRepoParams, numRetry int) (string, error) {).Scan(&insertedId)if err != nil {if err == sql.ErrNoRows {- log.Printf("no rows returned from insert query")- return "", fmt.Errorf("no rows returned from insert query")+ // Means ON CONFLICT DO NOTHING prevented insertion+ // => concurrency conflict => backoff & retry+ return retryWithExponentialBackoff(params.Ctx,+ errors.New("lock conflict: row not inserted"),+ numRetry,+ func(nextAttempt int) (string, error) {+ return lockRepo(params, nextAttempt)+ },+ )}return "", fmt.Errorf("error inserting new lock: %v", err)@@ -329,7 +310,9 @@ func lockRepo(params LockRepoParams, numRetry int) (string, error) {newLock.Id = insertedId.String} else {log.Printf("no rows returned from insert query, means there was a conflict")- return spinRetryWithJitter()+ return retryWithExponentialBackoff(params.Ctx, err, numRetry, func(nextAttempt int) (string, error) {+ return lockRepo(params, nextAttempt)+ })}log.Printf("[Lock][%d] INSERT took %v",@@ -447,18 +430,6 @@ func deleteRepoLock(id, planId string, numRetry int) error {log.Printf("[Delete][%d] END delete lock took %v", goroutineID, elapsed)}()- retryWithJitter := func() error {- // 10 attempts (~3-6 seconds)- if numRetry > 10 {- return fmt.Errorf("plan is currently being updated by another user or process")- }-- backoffBase := 300 * time.Millisecond- jitter := time.Duration(rand.Int63n(int64(200 * time.Millisecond)))- time.Sleep(backoffBase + jitter)- return deleteRepoLock(id, planId, numRetry+1)- }-var committed bool// Start a new transaction for the delete@@ -480,9 +451,17 @@ func deleteRepoLock(id, planId string, numRetry int) error {_, err = tx.Exec(lockablePlanIdQuery, planId)if err != nil {if isDeadlockError(err) {- log.Printf("[Delete][%d] Deadlock error, retrying delete after %v: %v",- goroutineID, time.Since(deleteStart), err)- return retryWithJitter()+ log.Printf("[Delete][%d] Deadlock error, retrying delete %v",+ goroutineID, err)+ _, err := retryWithExponentialBackoff(context.Background(), err, numRetry, func(nextAttempt int) (string, error) {+ return "", deleteRepoLock(id, planId, nextAttempt)+ })++ if err != nil {+ log.Printf("[Delete][%d] Error retrying delete: %v",+ goroutineID, err)+ return fmt.Errorf("error retrying delete: %v", err)+ }}return fmt.Errorf("error getting lockable plan id: %v", err)@@ -492,13 +471,21 @@ func deleteRepoLock(id, planId string, numRetry int) error {result, err := tx.Exec(query, id)if err != nil {if isDeadlockError(err) {- log.Printf("[Delete][%d] Deadlock error, retrying delete after %v: %v",- goroutineID, time.Since(deleteStart), err)- return retryWithJitter()+ log.Printf("[Delete][%d] Deadlock error, retrying delete %v",+ goroutineID, err)+ _, err := retryWithExponentialBackoff(context.Background(), err, numRetry, func(nextAttempt int) (string, error) {+ return "", deleteRepoLock(id, planId, nextAttempt)+ })++ if err != nil {+ log.Printf("[Delete][%d] Error retrying delete: %v",+ goroutineID, err)+ return fmt.Errorf("error retrying delete: %v", err)+ }}- log.Printf("[Delete][%d] Error executing delete after %v: %v",- goroutineID, time.Since(deleteStart), err)+ log.Printf("[Delete][%d] Error executing delete %v",+ goroutineID, err)return fmt.Errorf("error removing lock: %v", err)}@@ -551,3 +538,37 @@ func isDeadlockError(err error) bool {return false}++func retryWithExponentialBackoff(+ ctx context.Context,+ cause error,+ attempt int,+ nextCall func(int) (string, error),+) (string, error) {+ // If we have retried enough times, bail out.+ if attempt >= maxRetries {+ return "", fmt.Errorf("failed to acquire lock after %d attempts: %w", attempt, cause)+ }++ // Exponential delay: initialRetryDelay * 2^(attempt)+ backoff := time.Duration(float64(initialRetryDelay) * math.Pow(backoffFactor, float64(attempt)))+ // Add jitter: ± jitterFraction+ jitterRange := time.Duration(float64(backoff) * jitterFraction)+ jitter := time.Duration(rand.Int63n(int64(jitterRange)*2)) - jitterRange++ wait := backoff + jitter+ if wait < 0 {+ wait = 0+ }++ log.Printf("Lock/transaction conflict (attempt #%d). Retrying in %s... (cause: %v)", attempt, wait, cause)++ select {+ case <-ctx.Done():+ return "", fmt.Errorf("context canceled while waiting to retry: %w", ctx.Err())+ case <-time.After(wait):+ // Proceed with the next attempt.+ }++ return nextCall(attempt + 1)+}