Skip to content

Self-hosted workflows for coding-agent delivery. Orchestrate CodeBuddy, Codex and Claude Code with scripts and human approval gates. Includes Local workflows for coding assistants.

15.7 KB
Raw Blame
package runtime

import (
	"agentflow/internal/domain"
	"context"
	"fmt"
	"os"
	"path/filepath"
	goruntime "runtime"
	"strings"
	"sync"
)

type Manager struct {
	root  string
	slots chan struct{}
}

func NewManager(root string) *Manager { return NewManagerWithLimit(root, 8) }

func NewManagerWithLimit(root string, concurrency int) *Manager {
	if concurrency < 1 {
		concurrency = 1
	}
	return &Manager{root: root, slots: make(chan struct{}, concurrency)}
}

func (m *Manager) Root() string {
	if m == nil {
		return ""
	}
	return m.root
}

// RunWorkspace returns the host path reserved for a run. It does not create
// the directory; callers must still check whether the run has materialized it.
func (m *Manager) RunWorkspace(runID string) string {
	if m == nil || strings.TrimSpace(m.root) == "" {
		return ""
	}
	return filepath.Join(m.root, safeID(runID))
}

func (m *Manager) Create(ctx context.Context, runID, nodeID string, profile domain.SandboxProfile) (Sandbox, error) {
	id := safeID(runID + "-" + nodeID)
	return m.open(ctx, id, id, profile, true, true, SandboxRequirements{}, RunCredentials{}, SandboxResources{})
}

// AcquirePersistent reuses a run/issue workspace across SOP nodes. An existing
// git checkout is kept so later nodes continue the same branch.
func (m *Manager) AcquirePersistent(ctx context.Context, persistID string, profile domain.SandboxProfile) (Sandbox, error) {
	id := "persist-" + safeID(persistID)
	return m.open(ctx, id, id, profile, false, false, SandboxRequirements{}, RunCredentials{}, SandboxResources{})
}

// AcquireRun opens or reconstructs the one sandbox/workspace owned by a
// workflow run. Requirements must be aggregated across all workflow providers
// before this call so Docker mounts cannot depend on whichever node runs first.
func (m *Manager) AcquireRun(ctx context.Context, runID string, profile domain.SandboxProfile, requirements SandboxRequirements, credentials RunCredentials, resources SandboxResources) (Sandbox, error) {
	return m.AcquireRunIn(ctx, runID, runID, profile, requirements, credentials, resources)
}

// AcquireRunIn opens the sandbox of runID on the workspace of workspaceRunID.
// A child SOP keeps its own sandbox (lock, container, environment install)
// but works in the checkout of the run that owns the workspace.
func (m *Manager) AcquireRunIn(ctx context.Context, runID, workspaceRunID string, profile domain.SandboxProfile, requirements SandboxRequirements, credentials RunCredentials, resources SandboxResources) (Sandbox, error) {
	if strings.TrimSpace(workspaceRunID) == "" {
		workspaceRunID = runID
	}
	return m.open(ctx, safeID(runID), safeID(workspaceRunID), profile, false, false, requirements, credentials, resources)
}

func (m *Manager) open(ctx context.Context, id, dir string, profile domain.SandboxProfile, wipe, ephemeral bool, requirements SandboxRequirements, credentials RunCredentials, resources SandboxResources) (Sandbox, error) {
	lock, err := m.acquireLock(ctx, id)
	if err != nil {
		return nil, fmt.Errorf("sandbox lock: %w", err)
	}
	fail := func(err error) (Sandbox, error) {
		_ = unlockFile(lock)
		return nil, err
	}
	if profile.Driver, err = domain.ResolveSandboxDriver(profile.Driver, goruntime.GOOS); err != nil {
		return fail(err)
	}
	workspace := filepath.Join(m.root, dir)
	if wipe {
		if err := os.RemoveAll(workspace); err != nil {
			return fail(fmt.Errorf("clean sandbox workspace: %w", err))
		}
	}
	if err := os.MkdirAll(workspace, 0o700); err != nil {
		return fail(err)
	}
	activated, err := prepareSandboxEnvironment(workspace, profile, resources.Environment)
	if err != nil {
		return fail(err)
	}
	profile.Environment = overlayEnvironment(profile.Environment, activated.Overlay)
	profile, resources.Dockerfile = isolateLocalDriver(profile, resources.Dockerfile)
	sandbox, err := createDriverSandbox(ctx, "sandbox", id, workspace, profile, requirements, resources.Dockerfile)
	if err != nil {
		return fail(err)
	}
	managed := &managedSandbox{Sandbox: sandbox, slots: m.slots, lock: lock, ephemeral: ephemeral, workspace: workspace, credentials: credentials}
	if err := runEnvironmentInstall(ctx, managed, activated); err != nil {
		_ = managed.Close(ctx)
		return nil, err
	}
	return managed, nil
}

// DestroyRun deletes the host workspace for a finished SOP run: the persistent
// git checkout and any per-node directories. Artifacts must already be in the
// store before calling this.
func (m *Manager) DestroyRun(runID string) error {
	if m == nil || strings.TrimSpace(m.root) == "" {
		return nil
	}
	id := safeID(runID)
	if id == "" {
		return nil
	}
	// Removing the directory alone leaves the container running against a bind
	// that no longer exists. Docker reports such a container as active, so no
	// prune can reclaim its writable layer afterwards.
	removeDockerSandbox(context.Background(), dockerContainerName(id), tempDockerImageName(id))
	_, err := m.removeMatching(func(name string) bool {
		return name == id || strings.HasPrefix(name, id+"-") || name == "persist-"+id || strings.HasPrefix(name, "persist-"+id+"-")
	})
	return err
}

// DiscardRunContainer drops a run's sandbox container and image but keeps its
// workspace. A rerun needs the existing git checkout; it does not need the
// writable layer the previous attempt filled with installs and caches.
func (m *Manager) DiscardRunContainer(ctx context.Context, runID string) {
	if m == nil {
		return
	}
	if id := safeID(runID); id != "" {
		removeDockerSandbox(ctx, dockerContainerName(id), tempDockerImageName(id))
	}
}

// RunsWithIdleDockerSandbox narrows runIDs to those that still have a sandbox
// container on this host and whose workspace no process currently holds open.
func (m *Manager) RunsWithIdleDockerSandbox(ctx context.Context, runIDs []string) ([]string, error) {
	if m == nil || strings.TrimSpace(m.root) == "" || len(runIDs) == 0 {
		return nil, nil
	}
	found, err := dockerSandboxesUnder(ctx, m.root)
	if err != nil || len(found) == 0 {
		return nil, err
	}
	workspaces := make(map[string]struct{}, len(found))
	for _, workspace := range found {
		workspaces[workspace] = struct{}{}
	}
	var out []string
	for _, runID := range runIDs {
		id := safeID(runID)
		if id == "" {
			continue
		}
		// A child SOP's container is named after the child but binds the
		// workspace of the run owning the checkout; its lock is the child's.
		if _, ok := found[dockerContainerName(id)]; ok {
			if !m.workspaceInUse(id) {
				out = append(out, runID)
			}
			continue
		}
		for _, workspace := range []string{id, "persist-" + id} {
			if _, ok := workspaces[workspace]; ok && !m.workspaceInUse(workspace) {
				out = append(out, runID)
				break
			}
		}
	}
	return out, nil
}

// ReclaimDockerSandboxes removes sandbox containers left behind by runs that no
// longer own a workspace. It reads the live set from Docker instead of from an
// in-process map, which is the only way to reach containers inherited from an
// earlier process: a deploy restart drops every sandbox handle while the
// containers keep running.
func (m *Manager) ReclaimDockerSandboxes(ctx context.Context, liveRunIDs []string) (int, error) {
	if m == nil || strings.TrimSpace(m.root) == "" {
		return 0, nil
	}
	found, err := dockerSandboxesUnder(ctx, m.root)
	if err != nil || len(found) == 0 {
		return 0, err
	}
	var removed int
	for container, workspace := range staleSandboxContainers(found, liveRunIDs) {
		// The flock is held for as long as a sandbox is open, including by
		// another process on this host. Reclaiming then would docker rm a
		// container with an agent running inside it.
		if m.workspaceInUse(workspace) {
			continue
		}
		removeDockerSandbox(ctx, container, tempDockerImageName(workspace))
		removed++
	}
	return removed, nil
}

// ReclaimCachedDockerImages drops content-hashed images that no catalog
// Dockerfile still produces and that no container is using. Live recipes and
// in-flight sandboxes keep their tags so the next run does not rebuild.
func (m *Manager) ReclaimCachedDockerImages(ctx context.Context, dockerfiles []domain.DockerfileDefinition) (int, error) {
	if m == nil || !dockerAvailable() {
		return 0, nil
	}
	keep := make(map[string]struct{}, len(dockerfiles))
	for _, dockerfile := range dockerfiles {
		if strings.TrimSpace(dockerfile.Content) == "" {
			continue
		}
		keep[cachedDockerImageName(dockerfile)] = struct{}{}
	}
	return reclaimCachedDockerImages(ctx, keep)
}

// staleSandboxContainers narrows container -> workspace pairs to those whose
// workspace no longer belongs to a run that is allowed to keep one.
func staleSandboxContainers(found map[string]string, liveRunIDs []string) map[string]string {
	keep := make(map[string]struct{}, len(liveRunIDs)*2)
	for _, runID := range liveRunIDs {
		if id := safeID(runID); id != "" {
			keep[id] = struct{}{}
			keep["persist-"+id] = struct{}{}
		}
	}
	stale := map[string]string{}
	for container, workspace := range found {
		if _, live := keep[workspace]; !live {
			stale[container] = workspace
		}
	}
	return stale
}

// ReclaimExcept deletes workspaces that do not belong to the given live run IDs.
// Waiting/running checkouts are kept so an approval or resume can continue.
func (m *Manager) ReclaimExcept(liveRunIDs []string) (int, error) {
	if m == nil || strings.TrimSpace(m.root) == "" {
		return 0, nil
	}
	keep := make(map[string]struct{}, len(liveRunIDs)*2)
	for _, runID := range liveRunIDs {
		id := safeID(runID)
		if id == "" {
			continue
		}
		keep[id] = struct{}{}
		keep["persist-"+id] = struct{}{}
	}
	return m.removeMatching(func(name string) bool {
		for prefix := range keep {
			if name == prefix || strings.HasPrefix(name, prefix+"-") {
				return false
			}
		}
		return true
	})
}

func (m *Manager) removeMatching(match func(name string) bool) (int, error) {
	root, err := filepath.Abs(m.root)
	if err != nil {
		return 0, err
	}
	entries, err := os.ReadDir(root)
	if err != nil {
		if os.IsNotExist(err) {
			return 0, nil
		}
		return 0, err
	}
	var first error
	var removed int
	for _, entry := range entries {
		name := entry.Name()
		if name == sandboxLockDir || strings.HasPrefix(name, ".") {
			continue
		}
		if !match(name) {
			continue
		}
		dir := filepath.Join(root, name)
		rel, relErr := filepath.Rel(root, dir)
		if relErr != nil || rel == "." || strings.HasPrefix(rel, "..") {
			continue
		}
		// A live Exec has this directory as cwd. Removing it makes Node throw
		// "ENOENT: no such file or directory, uv_cwd" instead of a Git error.
		if m.workspaceInUse(name) {
			continue
		}
		if err := os.RemoveAll(dir); err != nil {
			if first == nil {
				first = err
			}
			continue
		}
		removed++
	}
	return removed, first
}

type managedSandbox struct {
	Sandbox
	once          sync.Once
	slots         chan struct{}
	lock          *os.File
	ephemeral     bool
	workspace     string
	credentials   RunCredentials
	execMu        sync.Mutex
	credentialsMu sync.RWMutex
}

// Unwrap returns the underlying driver sandbox so wrappers remain transparent
// to runtime helpers such as browser provisioning and conversation CLI setup.
func (s *managedSandbox) Unwrap() Sandbox {
	if s == nil {
		return nil
	}
	return s.Sandbox
}

func (s *managedSandbox) Driver() string {
	if value, ok := s.Sandbox.(interface{ Driver() string }); ok {
		return value.Driver()
	}
	return ""
}

type isolationEnvSource interface {
	IsolationEnv() map[string]string
}

func sandboxIsolationEnv(sb Sandbox) map[string]string {
	if src, ok := sb.(isolationEnvSource); ok {
		return src.IsolationEnv()
	}
	return nil
}

func (s *managedSandbox) IsolationEnv() map[string]string {
	if src, ok := s.Sandbox.(isolationEnvSource); ok {
		return src.IsolationEnv()
	}
	return nil
}

func (s *processSandbox) IsolationEnv() map[string]string { return s.profile.Environment }

func (s *localSandbox) IsolationEnv() map[string]string { return s.environment }

func (s *dockerSandbox) IsolationEnv() map[string]string { return s.profile.Environment }

func (s *managedSandbox) Close(ctx context.Context) error {
	var err error
	s.once.Do(func() {
		err = s.Sandbox.Close(ctx)
		if s.ephemeral && s.workspace != "" {
			_ = os.RemoveAll(s.workspace)
		}
		_ = unlockFile(s.lock)
	})
	return err
}

// Detach drops the workspace flock without docker rm. Deploy drain parks the
// run so the next worker can reuse the same bind-mounted container.
func (s *managedSandbox) Detach() error {
	s.once.Do(func() {
		_ = unlockFile(s.lock)
	})
	return nil
}

func (s *managedSandbox) Recreate(ctx context.Context) error {
	if rec, ok := s.Sandbox.(interface{ Recreate(context.Context) error }); ok {
		return rec.Recreate(ctx)
	}
	return nil
}

func (s *managedSandbox) BindWorkspace(ctx context.Context) error {
	if bind, ok := s.Sandbox.(interface{ BindWorkspace(context.Context) error }); ok {
		return bind.BindWorkspace(ctx)
	}
	return s.Recreate(ctx)
}

// SetCredentials replaces the token snapshot used by later Exec calls.
// The workspace and container stay as they are.
func (s *managedSandbox) SetCredentials(credentials RunCredentials) {
	s.credentialsMu.Lock()
	defer s.credentialsMu.Unlock()
	s.credentials = credentials
}

// CredentialSnapshot returns the current credential snapshot. Read-only use:
// preparation code validating a node's declared credentials up front.
func (s *managedSandbox) CredentialSnapshot() RunCredentials {
	s.credentialsMu.RLock()
	defer s.credentialsMu.RUnlock()
	return s.credentials
}

// sandboxCredentialSnapshot lets runtime preparation code read a sandbox's
// credentials without widening the public Sandbox interface.
type sandboxCredentialSnapshot interface{ CredentialSnapshot() RunCredentials }

func (s *managedSandbox) Exec(ctx context.Context, command CommandSpec, sink EventSink) (string, error) {
	select {
	case s.slots <- struct{}{}:
	case <-ctx.Done():
		return "", ctx.Err()
	}
	defer func() { <-s.slots }()
	s.credentialsMu.RLock()
	credentials := s.credentials
	s.credentialsMu.RUnlock()
	command, closeGit, err := prepareRunGitBinding(ctx, s.Sandbox, command, credentials)
	if err != nil {
		return "", err
	}
	defer closeGit()
	if credentials.GitHTTPCredentials != nil {
		credentials.GitHTTPHosts = runGitHosts(credentials)
	}
	// Binding writes the credential helpers into the workspace; parallel
	// branches then run their commands concurrently.
	s.execMu.Lock()
	bound, err := bindCommand(s.Sandbox, s.workspace, command, credentials)
	s.execMu.Unlock()
	if err != nil {
		return "", err
	}
	cleanup, err := bindCommandGitSSH(s.Sandbox, s.workspace, bound, credentials)
	if err != nil {
		return "", err
	}
	defer cleanup()
	defer scrubRunGitCredentials(ctx, s.Sandbox, s.workspace, bound, credentials)
	out, err := s.Sandbox.Exec(ctx, bound, runGitEventSink(sink, credentials))
	return redactRunGitSecrets(out, credentials), err
}

// overlayEnvironment layers an activated environment's overlay under env,
// except PATH: the overlay's PATH wins so its tools stay first on the path.
// An empty overlay returns env unchanged.
func overlayEnvironment(env, overlay map[string]string) map[string]string {
	if len(overlay) == 0 {
		return env
	}
	merged := mergeEnvironmentMaps(overlay, env)
	if path := overlay["PATH"]; path != "" {
		merged["PATH"] = path
	}
	return merged
}

// createDriverSandbox builds the sandbox for profile.Driver. kind only names
// the caller in the unsupported-driver error.
func createDriverSandbox(ctx context.Context, kind, id, workspace string, profile domain.SandboxProfile, requirements SandboxRequirements, dockerfile domain.DockerfileDefinition) (Sandbox, error) {
	switch strings.TrimSpace(profile.Driver) {
	case "", "local":
		return &localSandbox{id: id, workspace: workspace, environment: profile.Environment}, nil
	case "seatbelt":
		return createSeatbelt(id, workspace, profile, requirements)
	case "bwrap":
		return createBwrap(id, workspace, profile, requirements)
	case "docker":
		return createDocker(ctx, id, workspace, profile, requirements, dockerfile)
	default:
		return nil, fmt.Errorf("unsupported %s driver %q", kind, profile.Driver)
	}
}