Self-hosted workflows for coding-agent delivery. Orchestrate CodeBuddy, Codex and Claude Code with scripts and human approval gates. Includes Local workflows for coding assistants.
JavaScript agent-orchestrationagentic-workflowai-agentsai-codingclaude-codecode-reviewcodebuddycodexcoding-agentscursordeveloper-toolshuman-on-the-loopllmmcpmulti-agentsdlcself-hostedsopworkflow-automationworkflow-engine
package engine
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"time"
"agentflow/internal/domain"
"agentflow/internal/gitwork"
"agentflow/internal/repo"
"agentflow/internal/runtime"
)
const ConversationWorkspaceIdleTTL = time.Hour
type conversationWorkspaceTokenKey struct{}
type conversationWorkspaceLease struct {
token string
runID string
}
// WithConversationWorkspaceToken carries an already acquired workspace lease.
// Supplying the run ID limits reuse to that run, including during child recovery.
// The optional scope preserves legacy callers that only inspect the raw token;
// execution restoration requires an explicitly scoped lease.
func WithConversationWorkspaceToken(ctx context.Context, token string, runID ...string) context.Context {
lease := conversationWorkspaceLease{token: token}
if len(runID) > 0 {
lease.runID = runID[0]
}
return context.WithValue(ctx, conversationWorkspaceTokenKey{}, lease)
}
func ConversationWorkspaceToken(ctx context.Context, runID ...string) string {
lease, _ := ctx.Value(conversationWorkspaceTokenKey{}).(conversationWorkspaceLease)
if len(runID) > 0 && (lease.runID == "" || lease.runID != runID[0]) {
return ""
}
return lease.token
}
func (e *Engine) BeginConversationWorkspaceUse(id string) (string, error) {
token, previous, err := e.repo.BeginWorkspaceUsePreemptingSave(id, time.Now().UTC())
if err != nil || previous == "" {
return token, err
}
if value, ok := e.conversationSaves.Load(previous); ok {
value.(context.CancelFunc)()
}
// Another API instance may own the save. Its lease watcher cancels the
// export; wait only for the runner's read-only gate probe, never its archive.
if err = e.waitForBackgroundWorkspaceSave(id); err != nil {
_ = e.repo.FinishWorkspaceUse(id, token, time.Now().UTC(), nil, "")
return "", err
}
return token, nil
}
func (e *Engine) RenewConversationWorkspaceUse(id, token string) error {
return e.repo.RenewWorkspaceUse(id, token, time.Now().UTC())
}
// ReleaseConversationWorkspace hands a workspace a conversation switched away
// from to the idle sweep, which saves and reclaims it once nothing runs there.
func (e *Engine) ReleaseConversationWorkspace(id string) error {
return e.repo.ReleaseWorkspace(id, time.Now().UTC())
}
func (e *Engine) ConversationWorkspaceStatus(id string) (repo.WorkspaceCheckpoint, bool) {
return e.repo.WorkspaceCheckpoint(id)
}
func (e *Engine) conversationWorkspaceIdle(run domain.Run) bool {
if sharesWorkspace(run) || e.workspaceTerminals.HasRunScope(run.ID) {
return false
}
if e.WorkspaceActive != nil && e.WorkspaceActive(run) {
return false
}
if run.Status == "running" || run.Status == "queued" || run.HasActiveNode() {
return false
}
if run.Status == "waiting" && !run.IsWorkspace() {
return false
}
tasks, err := e.repo.ActiveRunnerTasksForRun(run.ID)
return err == nil && len(tasks) == 0
}
func (e *Engine) conversationGenerationRunning(run domain.Run) bool {
session, ok := e.repo.SharedAssistantSession(repo.RunAssistantSessionID(run.ID))
if !ok {
return false
}
job, ok := e.repo.PeekAssistantGeneration(session.UserID, session.ID)
return ok && job.Status == "running"
}
func (e *Engine) EndConversationWorkspaceUse(ctx context.Context, id, token string) error {
now := time.Now().UTC()
err := e.saveConversationWorkspaceAfterUse(ctx, id, token)
message := ""
if err != nil {
message = err.Error()
}
if finishErr := e.repo.FinishWorkspaceUse(id, token, now, nil, message); finishErr != nil {
return finishErr
}
return err
}
func (e *Engine) EnsureConversationWorkspace(ctx context.Context, id, token string) error {
_, err := e.PrepareConversationWorkspace(ctx, id, token)
return err
}
// MaterializeConversationWorkspace checks out an idle conversation workspace that
// has never had a workspace, so Main Bot can read and edit it before any
// command ran. A workspace that was saved or reclaimed is restored by
// PrepareConversationWorkspace instead.
func (e *Engine) MaterializeConversationWorkspace(ctx context.Context, id string) error {
run, ok := e.repo.RunState(id)
if !ok {
return fmt.Errorf("运行不存在")
}
if !run.IsWorkspace() || sharesWorkspace(run) || !run.WorkspaceCleanedAt.IsZero() || e.foreignWorkspaceBind(run) {
return nil
}
if meta, exists := e.repo.WorkspaceCheckpoint(id); exists && meta.SavedAt != nil {
return nil
}
if e.defaultExecutionRunner(run) != "" {
return e.materializeRunnerConversationWorkspace(ctx, run)
}
root, reason := e.LocalRunWorkspace(run)
if reason != "" {
return fmt.Errorf("%s", reason)
}
if _, err := os.Lstat(root); err == nil {
// Retry a checkout that failed earlier (e.g. before the Git Token was
// configured); never touch a repository that is already checked out.
missing := false
for _, dir := range repositoryWorkspaceDirs(root, run.RepositoryContexts()) {
if _, statErr := os.Stat(filepath.Join(dir, ".git")); statErr != nil {
missing = true
}
}
if !missing {
return nil
}
} else if !os.IsNotExist(err) {
return err
}
env, err := e.acquireConversationEnvironment(ctx, run)
if err != nil {
return err
}
env.nodeMu.Lock()
defer env.nodeMu.Unlock()
// Same checkout as an SOP node: the responsible user's Git Token, the
// frozen base branch, and a fresh work branch.
credentials := env.credentialSnapshot()
ctx = e.withPreparationProgress(ctx, run.ID)
ctx = withRunGitCredentials(ctx, credentials)
ctx = gitwork.WithSSHKeys(ctx, credentials.GitSSHKeys)
return env.prepareRepositoriesOnce(func() error {
if err := prepareRepositories(ctx, env.Workspace(), run.RepositoryContexts()); err != nil {
return explainGitAuthFailure(fmt.Errorf("repository checkout: %w", err), len(credentials.GitHTTPCredentials) > 0)
}
return nil
})
}
// runnerMaterializeBusyWait bounds how long opening a workspace waits for a
// concurrent preparation or snapshot export on the runner.
var runnerMaterializeBusyWait = 30 * time.Second
// A runner only checks repositories out for a prepare request, so a task that
// never ran a command has no directory there yet. Refreshing is disabled: this
// creates missing checkouts and never replaces one that may already be in use.
func (e *Engine) materializeRunnerConversationWorkspace(ctx context.Context, run domain.Run) error {
request, err := e.preparedWorkspaceRequest(run, domain.RunnerWorkspaceRequest{RunID: run.ID, Operation: domain.RunnerWorkspaceOperationPrepare})
if err != nil {
return err
}
request.RefreshCleanRepositories = false
deadline := time.Now().Add(runnerMaterializeBusyWait)
for {
response, err := e.requestWorkspaceCheckpoint(ctx, run, request)
if err != nil {
return err
}
if response.Error == "" {
return nil
}
if !runnerWorkspaceBusy(response) || time.Now().After(deadline) {
return explainGitAuthFailure(fmt.Errorf("%s", response.Error), len(e.resolveRunCredentials(run).GitHTTPCredentials) > 0)
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(300 * time.Millisecond):
}
}
}
// PrepareConversationWorkspace also repairs a legacy checkout that predates
// snapshots. A real snapshot stays authoritative, including intentional deletes.
func (e *Engine) PrepareConversationWorkspace(ctx context.Context, id, token string) ([]string, error) {
if err := e.ensureConversationWorkspace(ctx, id, token); err != nil {
return nil, err
}
meta, _ := e.repo.WorkspaceCheckpoint(id)
if meta.SavedAt != nil {
return nil, nil
}
run, ok := e.repo.RunState(id)
if !ok {
return nil, fmt.Errorf("运行不存在")
}
// Readers stay available during execution; never inject prior outputs while
// a node is currently expected to generate its own fresh deliverables.
if run.HasActiveNode() || !run.IsWorkspace() && (run.Status == "running" || run.Status == "queued") {
return nil, nil
}
return e.restoreArchivedWorkspaceFiles(ctx, run)
}
func (e *Engine) ensureConversationWorkspace(ctx context.Context, id, token string) error {
if run, ok := e.repo.RunState(id); ok && sharesWorkspace(run) {
return fmt.Errorf("子 SOP 共享父任务工作区,请从父任务操作")
}
meta, ok := e.repo.WorkspaceCheckpoint(id)
if !ok || meta.LeaseToken != token || meta.LeaseUntil == nil || meta.LeaseUntil.Before(time.Now()) {
return repo.ErrWorkspaceInUse
}
if err := e.repo.MarkWorkspaceFollowUp(id, token, time.Now().UTC()); err != nil {
return err
}
run, ok := e.repo.RunState(id)
if !ok {
return fmt.Errorf("运行不存在")
}
runner := e.defaultExecutionRunner(run)
if runner == "" && e.foreignWorkspaceBind(run) {
if run.WorkspaceCleanedAt.IsZero() {
return nil
}
if err := e.restoreOrReuseRemoteConversationCheckpoint(ctx, run); err != nil {
return err
}
return e.repo.RestoreCheckpointWorkspaceBinding(id, token, "", "", time.Now().UTC())
}
if runner == "" && run.WorkspaceCleanedAt.IsZero() {
root, reason := e.LocalRunWorkspace(run)
if reason != "" {
return fmt.Errorf("%s", reason)
}
if info, err := os.Lstat(root); err == nil {
if !info.IsDir() {
return fmt.Errorf("工作区路径不是目录")
}
return nil
} else if !os.IsNotExist(err) {
return err
}
} else if runner != "" && run.WorkspaceCleanedAt.IsZero() {
return nil
}
var err error
if runner != "" {
if task, exists := e.repo.RunnerTask("rt-cleanup-" + safeRunnerPath(id)); exists {
if task.Status == domain.RunnerTaskQueued {
if err = e.repo.CancelRunnerTask(task.ID, time.Now().UTC()); err != nil {
return err
}
// A claim may have raced cancellation. In that case cancellation
// only requests a stop; wait for the cleanup to actually finish.
task, _ = e.repo.RunnerTask(task.ID)
}
if task.Status == domain.RunnerTaskLeased || task.Status == domain.RunnerTaskRunning {
return fmt.Errorf("工作区尚在回收,请稍后恢复")
}
}
if err = e.restoreOrReuseRemoteConversationCheckpoint(ctx, run); err != nil {
return err
}
return e.repo.RestoreCheckpointWorkspaceBinding(id, token, "", "", time.Now().UTC())
}
root := e.sandboxes.RunWorkspace(id)
if info, statErr := os.Lstat(root); statErr == nil && info.IsDir() && !e.foreignWorkspaceBind(run) {
hasContent, contentErr := runtime.WorkspaceDirHasContent(root)
if contentErr != nil {
return contentErr
}
if hasContent {
if err = e.verifyStoredCheckpoint(ctx, id); err != nil {
return err
}
return e.repo.RestoreCheckpointWorkspaceBinding(id, token, e.workspaceHost, e.sandboxes.Root(), time.Now().UTC())
}
// An empty or .agentflow-only directory created before restoration
// holds no user files; tear down any container bound to its old inode
// so restoreConversationCheckpoint can atomically replace it and the
// next container bind sees the restored files.
if err = e.releaseRunEnvironment(id, false); err != nil {
return err
}
e.sandboxes.DiscardRunContainer(ctx, id)
}
if err = e.restoreConversationCheckpoint(ctx, run, root); err != nil {
return err
}
return e.repo.RestoreCheckpointWorkspaceBinding(id, token, e.workspaceHost, e.sandboxes.Root(), time.Now().UTC())
}
// claimIdleConversationWorkspace returns managed=true even on save failure, so
// legacy immediate cleanup can never delete files that were not persisted.
func (e *Engine) claimIdleConversationWorkspace(run domain.Run, now time.Time) (managed, claimed bool, release func(), err error) {
if sharesWorkspace(run) {
return false, false, nil, nil
}
meta, exists := e.repo.WorkspaceCheckpoint(run.ID)
session, shared := e.repo.SharedAssistantSession(repo.RunAssistantSessionID(run.ID))
if !exists && !run.IsWorkspace() && run.Status != "succeeded" && run.Status != "cancelled" && (!shared || session.Archived) {
return false, false, nil, nil
}
if !run.WorkspaceCleanedAt.IsZero() {
return false, false, nil, nil
}
if !e.conversationWorkspaceIdle(run) || e.conversationGenerationRunning(run) {
return true, false, nil, nil
}
// Recent conversation/IDE activity must not compete with maintenance for
// the lease. Foreground completion saves changes; reclamation waits an hour.
idleSinceBeforeLease := meta.LastUsedAt
if run.UpdatedAt.After(idleSinceBeforeLease) {
idleSinceBeforeLease = run.UpdatedAt
}
released := exists && meta.Released()
retryDue := meta.SaveRetryAt != nil && !now.Before(*meta.SaveRetryAt)
if exists && meta.FollowUp && !retryDue && !released && now.Before(idleSinceBeforeLease.Add(ConversationWorkspaceIdleTTL)) {
return true, false, nil, nil
}
token, err := e.repo.BeginWorkspaceCheckpointSave(run.ID, now)
if err != nil {
return true, false, nil, nil
}
meta, _ = e.repo.WorkspaceCheckpoint(run.ID)
idleSince := meta.LastUsedAt
if !exists {
idleSince = run.UpdatedAt
if idleSince.IsZero() {
idleSince = now
}
}
if run.UpdatedAt.After(idleSince) {
idleSince = run.UpdatedAt
}
// An idle group is still an ongoing conversation, even if Main Bot has
// only delegated and never invoked a workspace tool. Give it the same hour.
if run.IsWorkspace() && !meta.FollowUp {
if err := e.repo.MarkWorkspaceFollowUp(run.ID, token, idleSince); err != nil {
_ = e.repo.FinishWorkspaceCheckpointSave(run.ID, token, idleSince, now, err.Error())
return true, false, nil, err
}
meta.FollowUp = true
}
saveError := ""
finish := func() {
attemptedAt := time.Now().UTC()
if now.After(attemptedAt) {
attemptedAt = now
}
_ = e.repo.FinishWorkspaceCheckpointSave(run.ID, token, idleSince, attemptedAt, saveError)
}
defer func() {
if !claimed {
finish()
}
}()
if retryDue || released || meta.SavedAt == nil || run.UpdatedAt.After(*meta.SavedAt) || meta.Error != "" || !now.Before(idleSince.Add(ConversationWorkspaceIdleTTL)) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
captureErr := e.saveConversationWorkspace(ctx, run, token)
if captureErr != nil {
saveError = captureErr.Error()
return true, false, nil, captureErr
}
}
if meta.FollowUp && !released && now.Before(idleSince.Add(ConversationWorkspaceIdleTTL)) {
return true, false, nil, nil
}
// Revision and node checks serialize the deletion claim against SOP or group dispatch.
claimed, err = e.repo.ClaimCheckpointWorkspaceCleanup(run.ID, token, run.Revision, now)
if claimed {
release = finish
}
return true, claimed, release, err
}
func (e *Engine) restoreCheckpointBeforeExecution(ctx context.Context, run domain.Run) error {
if sharesWorkspace(run) {
return e.RestoreConversationWorkspaceIfNeeded(ctx, workspaceRunID(run))
}
checkpoint, exists := e.repo.WorkspaceCheckpoint(run.ID)
// A Main Bot turn or IDE heartbeat also creates this metadata row. Old
// reclaimed runs may therefore have a lease but no saved archive. Execution
// must still be able to prepare repositories and archived upstream inputs;
// EnsureConversationWorkspace remains strict for exact history restoration.
if !exists {
return nil
}
if checkpoint.SavedAt == nil {
if checkpoint.LeaseUntil != nil && checkpoint.LeaseUntil.After(time.Now()) && checkpoint.LeaseToken != ConversationWorkspaceToken(ctx, run.ID) {
return repo.ErrWorkspaceInUse
}
return nil
}
token := ConversationWorkspaceToken(ctx, run.ID)
owned := token != ""
if !owned {
var err error
token, err = e.beginWorkspaceUseAfterCleanup(ctx, run.ID)
if err != nil {
return err
}
// Reconnecting reuses immutable saved files; recapturing them here would
// add another full archive/upload before the retry can even start.
defer func() { _ = e.repo.FinishWorkspaceUse(run.ID, token, time.Now().UTC(), nil, "") }()
}
return e.EnsureConversationWorkspace(ctx, run.ID, token)
}
// restoreCheckpointForExecution waits for a concurrent workspace operation at
// actual node startup. A chat turn may activate a legacy run before releasing
// its lease; that temporary contention must not consume a node attempt as a
// failure. API activation preflights keep their bounded wait instead.
func (e *Engine) restoreCheckpointForExecution(ctx context.Context, run domain.Run, onBusy func()) error {
notified := false
for {
if err := ctx.Err(); err != nil {
return err
}
err := e.restoreCheckpointBeforeExecution(ctx, run)
if !errors.Is(err, repo.ErrWorkspaceInUse) {
return err
}
if !notified && onBusy != nil {
onBusy()
notified = true
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(100 * time.Millisecond):
}
}
}
// RestoreConversationWorkspaceIfNeeded is called before activation changes a
// task's status or revision. A generation may reuse its existing workspace lease.
func (e *Engine) RestoreConversationWorkspaceIfNeeded(ctx context.Context, id string) error {
run, ok := e.repo.RunState(id)
if !ok {
return nil // normal activation validation reports the missing run
}
if run.WorkspaceCleanedAt.IsZero() {
return nil
}
return e.restoreCheckpointBeforeExecution(ctx, run)
}
func (e *Engine) beginWorkspaceUseAfterCleanup(ctx context.Context, id string) (string, error) {
var token string
var err error
// Reclamation marks the directory before physical deletion; retry once its
// short-lived save/delete lease completes instead of racing the deletion.
for attempt := 0; attempt < 50; attempt++ {
token, err = e.BeginConversationWorkspaceUse(id)
if !errors.Is(err, repo.ErrWorkspaceInUse) {
return token, err
}
select {
case <-ctx.Done():
return "", ctx.Err()
case <-time.After(100 * time.Millisecond):
}
}
return "", err
}