Self-hosted workflows for coding-agent delivery. Orchestrate CodeBuddy, Codex and Claude Code with scripts and human approval gates. Includes Local workflows for coding assistants.
JavaScript agent-orchestrationagentic-workflowai-agentsai-codingclaude-codecode-reviewcodebuddycodexcoding-agentscursordeveloper-toolshuman-on-the-loopllmmcpmulti-agentsdlcself-hostedsopworkflow-automationworkflow-engine
package engine
import (
"agentflow/internal/domain"
"agentflow/internal/repo"
"context"
"errors"
"fmt"
"log"
"strings"
"time"
)
func (e *Engine) ownsRun(r domain.Run) bool {
e.mu.Lock()
instanceID, enabled := e.instanceID, e.executeRuns
e.mu.Unlock()
return enabled && r.OwnerInstanceID == instanceID && r.Generation > 0 && r.LeaseUntil.After(time.Now().UTC())
}
// foreignWorkspaceBind reports whether the run is already bound to a
// (host, workspace root) pair this instance cannot serve. While that host is
// alive the run stays there: an in-flight node may hold changes that are not
// on the remote work branch yet. See workspaceBlocked for when the bind may
// be taken over.
func (e *Engine) foreignWorkspaceBind(r domain.Run) bool {
host, root := e.workspaceBind(r)
boundRoot := strings.TrimSpace(root)
if boundRoot == "" {
return false
}
return boundRoot != e.sandboxes.Root() || host != e.workspaceHost
}
// workspaceBlocked reports whether a foreign bind must still be honoured: the
// bound host is this host (another process under a different root) or some
// live worker still advertises it. A bind to a host that is gone protects
// nothing; see workspaceLost for what that means for the run.
func (e *Engine) workspaceBlocked(r domain.Run) bool {
if !e.foreignWorkspaceBind(r) {
return false
}
host, _ := e.workspaceBind(r)
if host == e.workspaceHost || e.repo.WorkspaceHostAlive(host) {
return true
}
return e.defaultExecutionRunner(r) != ""
}
// workspaceLost reports a foreign bind to a host that is gone while the
// workspace held code changes. Nodes never push intermediate commits, so
// those changes went with the host: continuing elsewhere would silently drop
// them. Runs without a repository may still be taken over.
func (e *Engine) workspaceLost(r domain.Run) bool {
return e.foreignWorkspaceBind(r) && !e.workspaceBlocked(r) && hasCodeWorkspace(r)
}
func (e *Engine) workspaceLostMessage(r domain.Run) string {
host, root := e.workspaceBind(r)
if sharesWorkspace(r) {
return fmt.Sprintf("父流程工作区所在机器 %s:%s 已下线,之前节点的代码改动随之丢失,请从头重跑父流程", host, root)
}
return fmt.Sprintf("工作区所在机器 %s:%s 已下线,之前节点的代码改动随之丢失,请从头重跑", host, root)
}
func (e *Engine) acquireLease(r domain.Run) (domain.Run, error) {
e.mu.Lock()
instanceID, ttl, enabled, draining := e.instanceID, e.leaseTTL, e.executeRuns, e.draining
e.mu.Unlock()
if !enabled || draining {
return domain.Run{}, ErrExecutionDisabled
}
if err := e.repo.TryAcquireRunLease(r.ID, instanceID, r.Generation, ttl); err != nil {
return domain.Run{}, err
}
current, ok := e.repo.RunState(r.ID)
if !ok {
return domain.Run{}, fmt.Errorf("run not found")
}
return current, nil
}
func (e *Engine) renewRunLease(runID string, generation int) error {
e.mu.Lock()
instanceID, ttl := e.instanceID, e.leaseTTL
e.mu.Unlock()
var last error
for attempt := 0; attempt < 3; attempt++ {
if attempt > 0 {
time.Sleep(time.Second)
}
last = e.repo.RenewRunLease(runID, instanceID, generation, ttl)
if last == nil {
return nil
}
if errors.Is(last, repo.ErrRunLeaseLost) || errors.Is(last, repo.ErrRunLeaseHeld) {
return last
}
}
return last
}
// ResumeInterruptedRuns re-attaches scheduler goroutines to runs left in a live
// state by a previous process. In-flight nodes are reset to pending so execution
// can continue from the last durable checkpoint instead of being marked failed.
// Runs parked on a human gate are left alone: Approve/Retry can still resume them.
func (e *Engine) ResumeInterruptedRuns() []string {
if !e.executes() {
return nil
}
var resumed []string
// Promotion is per bucket; a failing bucket still returns the others and
// must not stop interrupted-run recovery below.
promoted, err := e.repo.PromoteQueuedRuns()
if err != nil {
log.Printf("promote queued runs: %v", err)
}
for _, id := range promoted {
e.publish(id)
}
for _, listed := range e.repo.Runs() {
if listed.IsWorkspace() || listed.HistoricalGroup() {
continue
}
// Approval and operator-paused Runs normally have no scheduler. A cancel
// request that raced with the last worker write must therefore be claimed
// and settled by the recovery loop instead of waiting forever.
if listed.Status == "waiting" && !listed.CancelRequested && alignRunWaitingApproval(listed) {
r, ok := e.repo.RunState(listed.ID)
if !ok || r.Status != "waiting" || r.CancelRequested || !alignRunWaitingApproval(r) {
continue
}
if r.OwnerInstanceID != "" && r.LeaseUntil.After(time.Now().UTC()) {
continue
}
if _, err := e.acquireLease(r); err != nil {
continue
}
e.updateRun(r.ID, func(current *domain.Run) {
if autoSucceedAlignApprovals(current, time.Now().UTC()) {
e.event(current, "", "gate.auto_approved", "基线对齐无需人工确认,已自动通过")
}
})
e.resume(r.ID)
resumed = append(resumed, r.ID)
continue
}
if listed.Status == "waiting" && listed.CancelRequested {
r, ok := e.repo.RunState(listed.ID)
if !ok || r.Status != "waiting" || !r.CancelRequested {
continue
}
if r.OwnerInstanceID != "" && r.LeaseUntil.After(time.Now().UTC()) {
continue
}
acquired, err := e.acquireLease(r)
if err != nil {
continue
}
_ = e.cancelRunnerTasksForRun(r.ID)
e.settleOrphan(r.ID, "运行已被手动终止")
_ = e.repo.ReleaseRunLease(r.ID, e.instanceID, acquired.Generation)
continue
}
if listed.Status == "running" {
fresh, ok := e.repo.RunState(listed.ID)
if !ok || fresh.Status != "running" {
continue
}
status, msg, done := settleStatusForCompletedGraph(workflowForRun(e.repo, fresh), fresh)
// A scheduled run still has to push into the integration branch after
// the last node succeeds. Settling here deletes that checkout mid-push.
if done && !(status == "succeeded" && scheduledIntegratePending(fresh)) {
e.finishAt(fresh.ID, fresh.Generation, status, msg)
continue
}
}
if listed.Status != "running" {
continue
}
r, ok := e.repo.RunState(listed.ID)
if !ok || r.Status != "running" {
continue
}
if r.OwnerInstanceID != "" && r.LeaseUntil.After(time.Now().UTC()) {
continue
}
e.mu.Lock()
_, live := e.running[r.ID]
e.mu.Unlock()
if live {
continue
}
// A run whose workspace is still served by another live host can never
// be resumed here, but settling a cancel or pause only writes run
// state, and cleanupWorkspace already refuses foreign binds. Claiming
// those two still has to happen or the run stays "running" forever: the
// API process has execution disabled, so Engine.Cancel cannot settle it
// either. A bind to a host that is gone is taken over and rebuilt.
blocked := e.workspaceBlocked(r)
if blocked && !r.CancelRequested && !r.PauseRequested {
continue
}
previousOwner := r.OwnerInstanceID
acquired, err := e.acquireLease(r)
if err != nil {
continue
}
if r.CancelRequested || acquired.CancelRequested {
_ = e.cancelRunnerTasksForRun(r.ID)
e.settleOrphan(r.ID, "运行已被手动终止")
_ = e.repo.ReleaseRunLease(r.ID, e.instanceID, acquired.Generation)
continue
}
if r.PauseRequested || acquired.PauseRequested {
_ = e.cancelRunnerTasksForRun(r.ID)
e.pauseAt(r.ID, acquired.Generation, "运行已由用户暂停")
_ = e.repo.ReleaseRunLease(r.ID, e.instanceID, acquired.Generation)
continue
}
if blocked {
// The control request was withdrawn between the check above and the
// lease. Hand the run back instead of resuming another host's bind.
_ = e.repo.ReleaseRunLease(r.ID, e.instanceID, acquired.Generation)
continue
}
if e.workspaceLost(acquired) {
message := e.workspaceLostMessage(acquired)
// Move the bind here so a rerun starts from a fresh checkout
// instead of failing on the dead host again. A child follows its
// parent's bind, which the parent's own recovery moves.
if !sharesWorkspace(acquired) {
if err := e.rebindWorkspaceHere(acquired, true); err != nil {
log.Printf("rebind lost workspace %s: %v", r.ID, err)
}
}
e.settleOrphan(r.ID, message)
_ = e.repo.ReleaseRunLease(r.ID, e.instanceID, acquired.Generation)
continue
}
e.updateRun(r.ID, func(current *domain.Run) {
resetInFlightNodesForResume(current, "执行实例退出,等待恢复")
current.Status = "running"
e.event(current, "", "run.resumed", fmt.Sprintf("服务恢复后继续执行(generation=%d, previousOwner=%s)", acquired.Generation, previousOwner))
})
e.observe(r.ID, "", domain.TelemetryEvent{Category: "orchestration", Name: "run.resumed", Phase: "start", Message: "过期执行租约已接管", Attributes: map[string]any{"generation": acquired.Generation, "previousOwner": previousOwner}})
e.resume(r.ID)
resumed = append(resumed, r.ID)
}
return resumed
}
// RecoverInterruptedRuns keeps the old name for callers/tests that still refer to it.
func (e *Engine) RecoverInterruptedRuns() []string {
return e.ResumeInterruptedRuns()
}
func (e *Engine) ActiveRunCount() int {
e.mu.Lock()
defer e.mu.Unlock()
return len(e.running)
}
// Drain stops claiming work and parks in-flight runs so a replacement worker
// can resume them. Waiting for CodeBuddy nodes to finish would outlive
// systemd TimeoutStopSec and get the process SIGKILL'd mid-exec.
func (e *Engine) Drain(ctx context.Context) error {
e.mu.Lock()
e.draining = true
instanceID := e.instanceID
cancels := make([]context.CancelFunc, 0, len(e.running))
for _, cancel := range e.running {
if cancel != nil {
cancels = append(cancels, cancel)
}
}
e.mu.Unlock()
_ = e.repo.DeleteEngineInstance(instanceID)
for _, cancel := range cancels {
cancel()
}
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
for {
if e.ActiveRunCount() == 0 {
e.detachAllRunEnvironments()
return nil
}
select {
case <-ctx.Done():
e.detachAllRunEnvironments()
return ctx.Err()
case <-ticker.C:
}
}
}
func (e *Engine) claimRunning(runID string, cancel context.CancelFunc) bool {
e.mu.Lock()
defer e.mu.Unlock()
if _, ok := e.running[runID]; ok {
return false
}
e.running[runID] = cancel
return true
}
func (e *Engine) clearRunning(runID string) {
e.mu.Lock()
delete(e.running, runID)
e.mu.Unlock()
}
func (e *Engine) resume(runID string) {
current, ok := e.repo.RunState(runID)
if !ok || current.Status != "running" || !e.ownsRun(current) {
return
}
ctx, cancel := context.WithCancel(context.Background())
if !e.claimRunning(runID, cancel) {
cancel()
// A control decision can reopen a run immediately after cancellation,
// while the cancelled scheduler is still unwinding. Retry once that
// scheduler releases its slot so the persisted running state is not left
// without an owner.
go e.resumeAfterSchedulerRelease(runID)
return
}
go e.execute(ctx, runID)
}
func (e *Engine) resumeAfterSchedulerRelease(runID string) {
ticker := time.NewTicker(20 * time.Millisecond)
defer ticker.Stop()
timeout := time.NewTimer(e.leaseTTL)
defer timeout.Stop()
for {
select {
case <-ticker.C:
if e.isRunSchedulerActive(runID) {
continue
}
current, ok := e.repo.RunState(runID)
if ok && current.Status == "running" {
if current.IsWorkspace() && !e.ownsRun(current) {
if _, err := e.acquireLease(current); err != nil {
return
}
}
if e.ownsRun(current) {
e.resume(runID)
}
}
return
case <-timeout.C:
return
}
}
}
func (e *Engine) runControlRequested(runID string) bool {
return e.repo.RunControlRequested(runID)
}