Skip to content

Self-hosted workflows for coding-agent delivery. Orchestrate CodeBuddy, Codex and Claude Code with scripts and human approval gates. Includes Local workflows for coding assistants.

18.8 KB
Raw Blame
package repo

import (
	"crypto/rand"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"strings"
	"time"

	"gorm.io/gorm"
	"gorm.io/gorm/clause"
)

type WorkspaceCheckpoint struct {
	ArchiveID      string     `gorm:"size:64" json:"-"`
	ArchiveBytes   int64      `json:"-"`
	ArchiveChunks  int        `json:"-"`
	SaveFailures   int        `json:"-"`
	SaveRetryAt    *time.Time `json:"nextSaveAttemptAt,omitempty"`
	RunID          string     `gorm:"primaryKey;size:64" json:"-"`
	Archive        []byte     `gorm:"type:mediumblob" json:"-"`
	SHA256         string     `gorm:"size:64" json:"-"`
	LastUsedAt     time.Time  `json:"lastUsedAt"`
	SavedAt        *time.Time `json:"savedAt,omitempty"`
	FollowUp       bool       `json:"followUp"`
	Error          string     `gorm:"type:text" json:"error,omitempty"`
	LeaseHandoffID string     `gorm:"size:64" json:"-"`
	LeaseToken     string     `gorm:"size:64" json:"-"`
	LeaseUntil     *time.Time `json:"-"`
	// LeaseRenewedAt is when the lease was last taken or renewed, on the
	// database clock on MySQL (see leaseRenewalClock). LeaseUntil is written
	// by the holder's own clock, so it cannot tell another replica how long
	// ago the holder renewed.
	LeaseRenewedAt *time.Time `json:"-"`
	RestoreStatus  string     `gorm:"size:32" json:"-"`
	RestoreStarted *time.Time `json:"-"`
	RestoreError   string     `gorm:"type:text" json:"-"`
	// ReleasedAt is when a conversation switched away from the workspace; until
	// it is used again it is reclaimed without waiting out the idle hour.
	ReleasedAt *time.Time `json:"-"`
}

func (WorkspaceCheckpoint) TableName() string { return "run_workspace_checkpoints" }

// Released reports whether the workspace was released after its last use.
func (c WorkspaceCheckpoint) Released() bool {
	return c.ReleasedAt != nil && !c.LastUsedAt.After(*c.ReleasedAt)
}

var ErrWorkspaceInUse = errors.New("工作区正在使用或保存,请稍后重试")
var ErrWorkspaceCheckpointBackoff = errors.New("workspace checkpoint save is backing off")

const workspaceBackgroundSavePrefix = "save:"

// workspaceUseTerm is how long a workspace lease lasts from its last renewal.
const workspaceUseTerm = 15 * time.Minute

func (r *RunsRepo) WorkspaceCheckpoint(runID string) (WorkspaceCheckpoint, bool) {
	var item WorkspaceCheckpoint
	err := r.DB.Omit("archive").Where("run_id = ?", runID).Take(&item).Error
	return item, err == nil
}
func (r *RunsRepo) WorkspaceCheckpointArchive(runID string) ([]byte, error) {
	source, err := r.OpenWorkspaceCheckpointArchive(runID)
	if err != nil {
		return nil, err
	}
	defer source.Close()
	return io.ReadAll(source)
}
func (r *RunsRepo) BeginWorkspaceUse(runID string, now time.Time) (string, error) {
	token, _, err := r.BeginWorkspaceUsePreemptingSave(runID, now)
	return token, err
}

// Foreground use can replace a background archive lease. Execution, restore
// and maintenance/delete leases remain exclusive. The old save must still
// publish and finish by token CAS, so it cannot affect the new use.
func (r *RunsRepo) BeginWorkspaceUsePreemptingSave(runID string, now time.Time) (string, string, error) {
	var previous string
	token, err := r.beginWorkspaceLease(runID, now, true, false, &previous)
	return token, previous, err
}

// HandoffWorkspaceUseToSave makes the archive interruptible without an unlocked
// gap. SaveRetryAt also lets maintenance recover a crashed background save.
func (r *RunsRepo) HandoffWorkspaceUseToSave(runID, token string, now time.Time) (string, error) {
	if token == "" || strings.HasPrefix(token, workspaceBackgroundSavePrefix) {
		return "", ErrWorkspaceInUse
	}
	saveToken := workspaceBackgroundSavePrefix + token
	err := r.DB.Transaction(func(tx *gorm.DB) error {
		if _, err := r.takeRunForUpdate(tx, runID); err != nil {
			return err
		}
		result := tx.Model(&WorkspaceCheckpoint{}).Where("run_id = ? AND lease_token = ? AND lease_until > ?", runID, token, now).Updates(map[string]any{
			"lease_token": saveToken, "lease_handoff_id": "", "last_used_at": now, "save_retry_at": now,
		})
		if result.Error != nil {
			return result.Error
		}
		if result.RowsAffected != 1 {
			return ErrWorkspaceInUse
		}
		return nil
	})
	return saveToken, err
}
func (r *RunsRepo) BeginWorkspaceCheckpointSave(runID string, now time.Time) (string, error) {
	return r.beginWorkspaceLease(runID, now, false, false)
}
func (r *RunsRepo) beginWorkspaceLease(runID string, now time.Time, touch, restoring bool, preempted ...*string) (string, error) {
	raw := make([]byte, 24)
	if _, err := rand.Read(raw); err != nil {
		return "", err
	}
	token := hex.EncodeToString(raw)
	err := r.DB.Transaction(func(tx *gorm.DB) error {
		run, err := r.takeRunForUpdate(tx, runID)
		if err != nil {
			return err
		}
		var checkpoint WorkspaceCheckpoint
		if err := tx.Omit("archive").Where("run_id = ?", runID).Take(&checkpoint).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
			return err
		}
		// An unfinished outcome on a bound workspace is retried, never assumed.
		unfinished := checkpoint.RestoreStatus == "failed" || checkpoint.RestoreStatus == "restoring"
		if restoring && run.WorkspaceCleaned == nil && !unfinished {
			token = ""
			return nil
		}
		if !touch && checkpoint.SaveRetryAt != nil && now.Before(*checkpoint.SaveRetryAt) {
			return ErrWorkspaceCheckpointBackoff
		}
		until := now.Add(workspaceUseTerm)
		lastUsed := now
		if !touch && run.Updated != nil {
			lastUsed = *run.Updated
		}
		row := WorkspaceCheckpoint{RunID: runID, LastUsedAt: lastUsed}
		if err := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&row).Error; err != nil {
			return err
		}
		values := map[string]any{"lease_token": token, "lease_until": until, "lease_renewed_at": r.leaseRenewalClock(now), "lease_handoff_id": ""}
		if touch {
			values["last_used_at"] = now
		}
		if restoring {
			values["restore_status"], values["restore_started"], values["restore_error"] = "restoring", now, ""
		} else if checkpoint.RestoreStatus == "restoring" {
			// Taking over an expired restore lease fences that restore out, so
			// record it as interrupted. A later binding under this lease can
			// then tell a stale outcome from a restore still in progress.
			values["restore_status"], values["restore_error"] = "failed", WorkspaceRestoreInterrupted
		}
		query := tx.Model(&WorkspaceCheckpoint{}).Where("run_id = ?", runID)
		if touch && !restoring {
			var current WorkspaceCheckpoint
			if err := tx.Omit("archive").Where("run_id = ?", runID).Take(&current).Error; err != nil {
				return err
			}
			if strings.HasPrefix(current.LeaseToken, workspaceBackgroundSavePrefix) && len(preempted) > 0 {
				*preempted[0] = current.LeaseToken
			}
			query = query.Where("lease_until IS NULL OR lease_until < ? OR lease_token LIKE ?", now, workspaceBackgroundSavePrefix+"%")
		} else {
			query = query.Where("lease_until IS NULL OR lease_until < ?", now)
		}
		result := query.Updates(values)
		if result.Error != nil {
			return result.Error
		}
		if result.RowsAffected != 1 {
			return ErrWorkspaceInUse
		}
		return nil
	})
	return token, err
}
func (r *RunsRepo) RenewWorkspaceUse(runID, token string, now time.Time) error {
	return r.DB.Transaction(func(tx *gorm.DB) error {
		if _, err := r.takeRunForUpdate(tx, runID); err != nil {
			return err
		}
		var checkpoint WorkspaceCheckpoint
		if err := tx.Omit("archive").Where("run_id = ? AND lease_token = ?", runID, token).Take(&checkpoint).Error; err != nil {
			if errors.Is(err, gorm.ErrRecordNotFound) {
				return ErrWorkspaceInUse
			}
			return err
		}
		// The row is locked through the run and was just read with this token;
		// an unchanged lease_until (same-millisecond renewal) is still a renewal.
		until := now.Add(workspaceUseTerm)
		return tx.Model(&WorkspaceCheckpoint{}).Where("run_id = ? AND lease_token = ?", runID, token).Updates(map[string]any{"lease_until": until, "lease_renewed_at": r.leaseRenewalClock(now)}).Error
	})
}

func (r *RunsRepo) FinishWorkspaceUse(runID, token string, now time.Time, archive []byte, saveError string) error {
	attemptedAt := time.Now().UTC()
	if now.After(attemptedAt) {
		attemptedAt = now
	}
	return r.finishWorkspaceUse(runID, token, now, attemptedAt, archive, saveError)
}
func (r *RunsRepo) FinishWorkspaceCheckpointSave(runID, token string, idleSince, attemptedAt time.Time, saveError string) error {
	return r.finishWorkspaceUse(runID, token, idleSince, attemptedAt, nil, saveError)
}
func (r *RunsRepo) finishWorkspaceUse(runID, token string, now, attemptedAt time.Time, archive []byte, saveError string) error {
	return r.DB.Transaction(func(tx *gorm.DB) error {
		if _, err := r.takeRunForUpdate(tx, runID); err != nil {
			return err
		}
		var item WorkspaceCheckpoint
		if err := tx.Omit("archive").Where("run_id = ? AND lease_token = ?", runID, token).Take(&item).Error; err != nil {
			if errors.Is(err, gorm.ErrRecordNotFound) {
				return ErrWorkspaceInUse
			}
			return err
		}
		if item.LastUsedAt.After(now) {
			now = item.LastUsedAt
		}
		values := map[string]any{"lease_token": "", "lease_until": nil, "lease_handoff_id": "", "last_used_at": now}
		if archive != nil {
			sum := sha256.Sum256(archive)
			values["archive"], values["archive_id"], values["archive_bytes"], values["archive_chunks"] = archive, "", len(archive), 0
			values["sha256"], values["saved_at"] = hex.EncodeToString(sum[:]), now
			values["error"], values["save_failures"], values["save_retry_at"] = "", 0, nil
		} else if saveError != "" {
			failures := min(item.SaveFailures+1, 12)
			delay := min(5*time.Minute*time.Duration(1<<uint(failures-1)), 6*time.Hour)
			next := attemptedAt.Add(delay)
			values["error"], values["save_failures"], values["save_retry_at"] = saveError, failures, next
		}
		result := tx.Model(&WorkspaceCheckpoint{}).Where("run_id = ? AND lease_token = ?", runID, token).Updates(values)
		if result.Error != nil {
			return result.Error
		}
		if result.RowsAffected != 1 {
			return ErrWorkspaceInUse
		}
		if archive != nil {
			return tx.Where("run_id = ?", runID).Delete(&workspaceCheckpointChunk{}).Error
		}
		return nil
	})
}
func (r *RunsRepo) ClaimCheckpointWorkspaceCleanup(runID, token string, revision int, now time.Time) (bool, error) {
	claimed := false
	err := r.DB.Transaction(func(tx *gorm.DB) error {
		row, err := r.takeRunForUpdate(tx, runID)
		if err != nil {
			return err
		}
		if row.Revision != revision || row.Status == "running" || row.Status == "queued" || row.WorkspaceCleaned != nil || row.CleanupNotBefore != nil && now.Before(*row.CleanupNotBefore) {
			return nil
		}
		var checkpoint WorkspaceCheckpoint
		if err = tx.Omit("archive").Where("run_id = ? AND lease_token = ? AND lease_until > ?", runID, token, now).Take(&checkpoint).Error; err != nil {
			return err
		}
		if checkpoint.SavedAt == nil || checkpoint.SHA256 == "" || checkpoint.Error != "" {
			return fmt.Errorf("工作区未成功保存,暂缓回收")
		}
		if checkpoint.FollowUp && !checkpoint.Released() && now.Before(checkpoint.LastUsedAt.Add(time.Hour)) {
			return nil
		}
		var count int64
		if err = tx.Model(&runNodeRow{}).Where("run_id = ? AND status = ?", runID, "running").Count(&count).Error; err != nil {
			return err
		}
		if count != 0 {
			return nil
		}
		result := tx.Model(&runRow{}).Where("id = ? AND revision = ?", runID, revision).Updates(map[string]any{"workspace_cleaned_at": now, "sandbox_id": ""})
		claimed = result.RowsAffected == 1
		return result.Error
	})
	return claimed, err
}
func (r *RunsRepo) RestoreCheckpointWorkspaceBinding(runID, token, host, root string, now time.Time) error {
	return r.DB.Transaction(func(tx *gorm.DB) error {
		if _, err := r.takeRunForUpdate(tx, runID); err != nil {
			return err
		}
		var count int64
		if err := tx.Model(&WorkspaceCheckpoint{}).Where("run_id = ? AND lease_token = ? AND lease_until > ?", runID, token, now).Count(&count).Error; err != nil {
			return err
		}
		if count != 1 {
			return ErrWorkspaceInUse
		}
		// A restore lease keeps "restoring" until FinishWorkspaceRestore records
		// its outcome. Any other lease that binds the workspace supersedes an
		// earlier failed or interrupted restore.
		if err := tx.Model(&WorkspaceCheckpoint{}).Where("run_id = ? AND lease_token = ? AND restore_status = ?", runID, token, "failed").
			Updates(map[string]any{"restore_status": "ready", "restore_error": ""}).Error; err != nil {
			return err
		}
		values := map[string]any{"workspace_cleaned_at": nil, "revision": gorm.Expr("revision + 1")}
		if host != "" {
			values["workspace_host"] = host
			values["workspace_root"] = root
		}
		return tx.Model(&runRow{}).Where("id = ?", runID).Updates(values).Error
	})
}
func (r *RunsRepo) CheckpointWorkspaceRunIDs() []string {
	var ids []string
	r.DB.Model(&WorkspaceCheckpoint{}).Pluck("run_id", &ids)
	return ids
}

func (r *RunsRepo) SaveWorkspaceCheckpoint(runID, token string, now time.Time, archive []byte) error {
	if len(archive) == 0 || len(archive) > workspaceCheckpointInlineBytes {
		return fmt.Errorf("invalid inline workspace checkpoint size")
	}
	sum := sha256.Sum256(archive)
	return r.DB.Transaction(func(tx *gorm.DB) error {
		if _, err := r.takeRunForUpdate(tx, runID); err != nil {
			return err
		}
		result := tx.Model(&WorkspaceCheckpoint{}).Where("run_id = ? AND lease_token = ? AND lease_until > ?", runID, token, time.Now().UTC()).Updates(map[string]any{"archive": archive, "archive_id": "", "archive_bytes": len(archive), "archive_chunks": 0, "sha256": hex.EncodeToString(sum[:]), "saved_at": now, "error": "", "save_failures": 0, "save_retry_at": nil})
		if result.Error != nil {
			return result.Error
		}
		if result.RowsAffected != 1 {
			return ErrWorkspaceInUse
		}
		return tx.Where("run_id = ?", runID).Delete(&workspaceCheckpointChunk{}).Error
	})
}

// MarkWorkspaceFollowUp validates the lease by a read, never by RowsAffected:
// MySQL reports changed rows, so two tools of one turn touching the workspace
// within the same millisecond (DATETIME(3)) would otherwise look like a lost
// lease and fail one of them with ErrWorkspaceInUse.
func (r *RunsRepo) MarkWorkspaceFollowUp(runID, token string, now time.Time) error {
	return r.DB.Transaction(func(tx *gorm.DB) error {
		if err := holdsWorkspaceLease(tx, runID, token); err != nil {
			return err
		}
		return tx.Model(&WorkspaceCheckpoint{}).Where("run_id = ? AND lease_token = ?", runID, token).Updates(map[string]any{"follow_up": true, "last_used_at": now}).Error
	})
}

func holdsWorkspaceLease(tx *gorm.DB, runID, token string) error {
	if token == "" {
		return ErrWorkspaceInUse
	}
	var count int64
	if err := tx.Model(&WorkspaceCheckpoint{}).Where("run_id = ? AND lease_token = ?", runID, token).Count(&count).Error; err != nil {
		return err
	}
	if count != 1 {
		return ErrWorkspaceInUse
	}
	return nil
}

// ReleaseAbandonedWorkspaceUse drops the foreground lease a reaped turn held.
// A dead API process cannot finish the lease it held in memory, and the next
// turn would fail for the rest of the 15 minute term.
//
// Two fences keep a live owner's lease in place. The token must be the one the
// reaped turn recorded, so a lease another turn took after the reap never
// matches. Its last renewal must also be older than the heartbeat timeout:
// every holder renews well within it (generations every 500ms, workspace turns
// every 30s), so a lease still being renewed by a process on any replica is
// kept. A background save token never matches a foreground token.
//
// The renewal age is measured on the clock that stamped lease_renewed_at (the
// database's on MySQL), so a holder whose clock runs behind the reader's does
// not look silent. A lease taken before that column existed has no stamp and
// falls back to its lease_until.
func (r *RunsRepo) ReleaseAbandonedWorkspaceUse(runID, token string, now time.Time) (bool, error) {
	runID, token = strings.TrimSpace(runID), strings.TrimSpace(token)
	if runID == "" || token == "" || strings.HasPrefix(token, workspaceBackgroundSavePrefix) {
		return false, nil
	}
	released := false
	err := r.DB.Transaction(func(tx *gorm.DB) error {
		if _, err := r.takeRunForUpdate(tx, runID); err != nil {
			if errors.Is(err, gorm.ErrRecordNotFound) {
				return nil
			}
			return err
		}
		silent := tx.Where("lease_renewed_at IS NULL AND lease_until <= ?", now.Add(workspaceUseTerm-assistantHeartbeatTimeout))
		if r.isMySQL() {
			silent = silent.Or("lease_renewed_at <= UTC_TIMESTAMP(3) - INTERVAL ? MICROSECOND", assistantHeartbeatTimeout.Microseconds())
		} else {
			silent = silent.Or("lease_renewed_at <= ?", now.Add(-assistantHeartbeatTimeout))
		}
		result := tx.Model(&WorkspaceCheckpoint{}).
			Where("run_id = ? AND lease_token = ? AND lease_until > ?", runID, token, now).
			Where(silent).
			Updates(map[string]any{"lease_token": "", "lease_until": nil, "lease_handoff_id": ""})
		released = result.RowsAffected == 1
		return result.Error
	})
	return released, err
}

// leaseRenewalClock stamps lease_renewed_at: the database's commit-time clock
// on MySQL, shared by every replica, and the caller's clock elsewhere.
func (r *RunsRepo) leaseRenewalClock(now time.Time) any {
	if r.isMySQL() {
		return clause.Expr{SQL: "UTC_TIMESTAMP(3)"}
	}
	return now
}

// ReleaseWorkspace marks a workspace no conversation is bound to any more, so
// the idle sweep saves and reclaims it on its next pass.
func (r *RunsRepo) ReleaseWorkspace(runID string, now time.Time) error {
	return r.DB.Transaction(func(tx *gorm.DB) error {
		row := WorkspaceCheckpoint{RunID: runID, LastUsedAt: now}
		if err := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&row).Error; err != nil {
			return err
		}
		return tx.Model(&WorkspaceCheckpoint{}).Where("run_id = ?", runID).Update("released_at", now).Error
	})
}

func (r *RunsRepo) TouchWorkspaceUse(runID string, now time.Time) error {
	return r.DB.Transaction(func(tx *gorm.DB) error {
		run, err := r.takeRunForUpdate(tx, runID)
		if err != nil {
			return err
		}
		if run.WorkspaceCleaned != nil {
			return ErrWorkspaceInUse
		}
		item := WorkspaceCheckpoint{RunID: runID, LastUsedAt: now, FollowUp: true}
		if err := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&item).Error; err != nil {
			return err
		}
		return tx.Model(&WorkspaceCheckpoint{}).Where("run_id = ?", runID).Updates(map[string]any{"last_used_at": now, "follow_up": true}).Error
	})
}

// Explicit deletion uses the same lease and marks the directory unavailable
// before filesystem removal, preventing retry from reusing it during deletion.
func (r *RunsRepo) ClaimWorkspaceDeletion(runID, token string, revision int, now time.Time) error {
	return r.DB.Transaction(func(tx *gorm.DB) error {
		row, err := r.takeRunForUpdate(tx, runID)
		if err != nil {
			return err
		}
		if row.Revision != revision || row.Status == "running" || row.Status == "waiting" || row.Status == "queued" {
			return ErrWorkspaceInUse
		}
		var count int64
		if err := tx.Model(&WorkspaceCheckpoint{}).Where("run_id = ? AND lease_token = ? AND lease_until > ?", runID, token, now).Count(&count).Error; err != nil {
			return err
		}
		if count != 1 {
			return ErrWorkspaceInUse
		}
		return tx.Model(&runRow{}).Where("id = ?", runID).Update("workspace_cleaned_at", now).Error
	})
}