Skip to content

Self-hosted workflows for coding-agent delivery. Orchestrate CodeBuddy, Codex and Claude Code with scripts and human approval gates. Includes Local workflows for coding assistants.

12.7 KB
Raw Blame
package engine

import (
	"agentflow/internal/domain"
	"agentflow/internal/repo"
	"agentflow/internal/runtime"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"time"
)

func (e *Engine) saveConversationWorkspace(ctx context.Context, run domain.Run, token string) error {
	file, err := os.CreateTemp("", "agentflow-checkpoint-save-")
	if err != nil {
		return err
	}
	defer func() { file.Close(); os.Remove(file.Name()) }()
	if e.defaultExecutionRunner(run) != "" || e.foreignWorkspaceBind(run) {
		if err = e.downloadConversationCheckpoint(ctx, run, file); err != nil {
			return err
		}
		if _, err = file.Seek(0, io.SeekStart); err != nil {
			return err
		}
		if err = runtime.ValidateWorkspaceCheckpointFrom(workspaceCheckpointContextReader{ctx, file}); err != nil {
			return err
		}
	} else {
		root, reason := e.LocalRunWorkspace(run)
		if reason != "" {
			return fmt.Errorf("%s", reason)
		}
		err = runtime.CreateWorkspaceCheckpointToContext(ctx, root, workspaceCheckpointContextWriter{ctx, file})
		if os.IsNotExist(err) && run.WorkspaceHost == "" {
			if err = file.Truncate(0); err != nil {
				return err
			}
			if _, err = file.Seek(0, io.SeekStart); err != nil {
				return err
			}
			_, err = file.Write(runtime.EmptyWorkspaceCheckpoint())
		}
		if err != nil {
			return err
		}
	}
	if _, err = file.Seek(0, io.SeekStart); err != nil {
		return err
	}
	return e.repo.SaveWorkspaceCheckpointStream(run.ID, token, time.Now().UTC(), workspaceCheckpointContextReader{ctx, file})
}

func (e *Engine) closeCheckpointTransfer(ctx context.Context, run domain.Run, id string) {
	if id == "" {
		return
	}
	budget := 5 * time.Second
	if ctx.Err() != nil {
		budget = time.Second
	}
	ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), budget)
	defer cancel()
	_, _ = e.requestWorkspaceCheckpoint(ctx, run, domain.RunnerWorkspaceRequest{RunID: run.ID, WorkspaceLayout: run.ExecutionContext.RunnerWorkspaceLayout, Operation: domain.RunnerWorkspaceOperationCheckpointClose, CheckpointTransfer: &domain.WorkspaceCheckpointTransfer{ID: id}})
}

// legacyCheckpointTerminalActive is the message runners sent before checkpoint
// responses carried WorkspaceErrorTerminalActive.
const legacyCheckpointTerminalActive = "workspace terminal is still active"

func workspaceCheckpointResponseError(response domain.RunnerWorkspaceResponse) error {
	if response.Error == "" && response.Status < 400 {
		return nil
	}
	// An open terminal is an execution fence that waiting cannot clear: fail
	// fast as workspace preparation does, never as retryable lease contention.
	if response.Code == domain.WorkspaceErrorTerminalActive || response.Error == legacyCheckpointTerminalActive {
		return fmt.Errorf("%w(%s)", errWorkspaceTerminalActive, response.Error)
	}
	if runnerWorkspaceBusy(response) {
		return repo.ErrWorkspaceInUse
	}
	if response.Code == domain.WorkspaceErrorMissing || response.Status == 404 {
		return fmt.Errorf("%w: %s", os.ErrNotExist, response.Error)
	}
	return fmt.Errorf("workspace checkpoint request failed (HTTP %d): %s", response.Status, response.Error)
}
func (e *Engine) checkpointTransferRequest(ctx context.Context, run domain.Run, op string, transfer *domain.WorkspaceCheckpointTransfer) (domain.WorkspaceCheckpointTransfer, error) {
	response, err := e.requestWorkspaceCheckpoint(ctx, run, domain.RunnerWorkspaceRequest{RunID: run.ID, RunStatus: run.Status, WorkspaceLayout: run.ExecutionContext.RunnerWorkspaceLayout, Operation: op, CheckpointTransfer: transfer})
	if err != nil {
		return domain.WorkspaceCheckpointTransfer{}, err
	}
	if err = workspaceCheckpointResponseError(response); err != nil {
		return domain.WorkspaceCheckpointTransfer{}, err
	}
	var result domain.WorkspaceCheckpointTransfer
	if err = json.Unmarshal(response.Payload, &result); err != nil {
		return result, err
	}
	return result, nil
}
func (e *Engine) beginCheckpointExport(ctx context.Context, run domain.Run) (domain.RunnerWorkspaceResponse, error) {
	response, err := e.requestWorkspaceCheckpoint(ctx, run, domain.RunnerWorkspaceRequest{RunID: run.ID, RunStatus: run.Status, WorkspaceLayout: run.ExecutionContext.RunnerWorkspaceLayout, Operation: domain.RunnerWorkspaceOperationCheckpointExport})
	if err != nil {
		return response, err
	}
	// Compatibility with runners predating chunked transfers. They can still
	// save small workspaces; a large archive is never retried as a huge RPC.
	if response.Status == 400 || strings.Contains(response.Error, "unsupported") {
		response, err = e.requestWorkspaceCheckpoint(ctx, run, domain.RunnerWorkspaceRequest{RunID: run.ID, RunStatus: run.Status, WorkspaceLayout: run.ExecutionContext.RunnerWorkspaceLayout, Operation: domain.RunnerWorkspaceOperationCheckpoint})
		if err != nil {
			return response, err
		}
	}
	return response, nil
}
func (e *Engine) downloadConversationCheckpoint(ctx context.Context, run domain.Run, destination io.Writer) error {
	response, err := e.beginCheckpointExport(ctx, run)
	if err != nil {
		return err
	}
	if err = workspaceCheckpointResponseError(response); err != nil {
		return err
	}
	var manifest domain.WorkspaceCheckpointTransfer
	if err = json.Unmarshal(response.Payload, &manifest); err != nil {
		return err
	}
	if manifest.ID == "" {
		var legacy struct {
			Checkpoint []byte `json:"checkpoint"`
		}
		if err = json.Unmarshal(response.Payload, &legacy); err != nil {
			return err
		}
		if len(legacy.Checkpoint) == 0 || len(legacy.Checkpoint) > runtime.WorkspaceCheckpointMaxBytes {
			return fmt.Errorf("执行机器返回无效工作区快照;请更新执行器以支持分块快照")
		}
		_, err = destination.Write(legacy.Checkpoint)
		return err
	}
	defer e.closeCheckpointTransfer(ctx, run, manifest.ID)
	if manifest.Size <= 0 || manifest.Size > runtime.WorkspaceCheckpointArchiveMaxBytes || len(manifest.SHA256) != 64 {
		return fmt.Errorf("invalid checkpoint export manifest")
	}
	hash := sha256.New()
	writer := workspaceCheckpointContextWriter{ctx, io.MultiWriter(destination, hash)}
	for offset := int64(0); offset < manifest.Size; {
		chunk, err := e.checkpointTransferRequest(ctx, run, domain.RunnerWorkspaceOperationCheckpointRead, &domain.WorkspaceCheckpointTransfer{ID: manifest.ID, Offset: offset})
		if err != nil {
			return err
		}
		if chunk.ID != manifest.ID || chunk.Offset != offset || int64(len(chunk.Data)) != min(int64(runtime.WorkspaceCheckpointChunkBytes), manifest.Size-offset) {
			return fmt.Errorf("invalid checkpoint download chunk")
		}
		if _, err = writer.Write(chunk.Data); err != nil {
			return err
		}
		offset += int64(len(chunk.Data))
	}
	if hex.EncodeToString(hash.Sum(nil)) != manifest.SHA256 {
		return fmt.Errorf("checkpoint download checksum mismatch")
	}
	return nil
}

// Reclamation is recorded before deleting files, and a successful remote
// restore can lose its reply before the binding is updated. Probe the owner
// without exporting the entire checkout. Only a confirmed missing directory
// permits an upload; busy/offline owners must never trigger another restore.
func (e *Engine) restoreOrReuseRemoteConversationCheckpoint(ctx context.Context, run domain.Run) error {
	probe, err := e.requestWorkspaceCheckpoint(ctx, run, domain.RunnerWorkspaceRequest{RunID: run.ID, RunStatus: run.Status, WorkspaceLayout: run.ExecutionContext.RunnerWorkspaceLayout, Operation: domain.RunnerWorkspaceOperationCheckpointStatus})
	if err != nil {
		return err
	}
	if probe.Status == 400 || strings.Contains(probe.Error, "unsupported") {
		// Compatibility with workers/runners deployed before the cheap probe.
		probe, err = e.beginCheckpointExport(ctx, run)
		if err != nil {
			return err
		}
		if err = workspaceCheckpointResponseError(probe); err == nil {
			var transfer domain.WorkspaceCheckpointTransfer
			if err = json.Unmarshal(probe.Payload, &transfer); err != nil {
				return err
			}
			e.closeCheckpointTransfer(ctx, run, transfer.ID)
		} else if legacyCheckpointMissingPath(probe.Error) {
			if e.defaultExecutionRunner(run) != "" {
				// Old transfer handlers returned 409 for all filesystem errors.
				// Their read-only info operation still returns an actual 404 for
				// a missing root; an individual file disappearing during export
				// must not cause the whole checkout to be restored again.
				info, infoErr := e.requestWorkspaceCheckpoint(ctx, run, domain.RunnerWorkspaceRequest{RunID: run.ID, WorkspaceLayout: run.ExecutionContext.RunnerWorkspaceLayout, Operation: domain.RunnerWorkspaceOperationInfo})
				if infoErr != nil {
					return infoErr
				}
				if missing := workspaceCheckpointResponseError(info); errors.Is(missing, os.ErrNotExist) {
					err = missing
				}
			} else {
				// A legacy Worker publishes its workspace root. Match that root
				// exactly rather than treating missing nested files as reclamation.
				root := filepath.Join(run.WorkspaceRoot, safeRunnerPath(workspaceRunID(run)))
				if strings.HasPrefix(probe.Error, "lstat "+root+": ") || strings.HasPrefix(probe.Error, "stat "+root+": ") {
					err = os.ErrNotExist
				}
			}
		}
	} else {
		err = workspaceCheckpointResponseError(probe)
		if err == nil {
			var status struct {
				Empty bool `json:"empty"`
			}
			if len(probe.Payload) > 0 && json.Unmarshal(probe.Payload, &status) == nil && status.Empty {
				err = os.ErrNotExist
			}
		}
	}
	if err == nil {
		return e.verifyStoredCheckpoint(ctx, run.ID)
	}
	if !errors.Is(err, os.ErrNotExist) {
		return err
	}
	return e.restoreConversationCheckpoint(ctx, run, "")
}

func legacyCheckpointMissingPath(message string) bool {
	return strings.HasSuffix(message, ": no such file or directory") || strings.HasSuffix(message, ": The system cannot find the path specified.") || strings.HasSuffix(message, ": The system cannot find the file specified.")
}

func (e *Engine) restoreConversationCheckpoint(ctx context.Context, run domain.Run, root string) error {
	source, err := e.repo.OpenWorkspaceCheckpointArchive(run.ID)
	if err != nil {
		return err
	}
	defer source.Close()
	sourceReader := workspaceCheckpointContextReader{ctx, source}
	if e.defaultExecutionRunner(run) == "" && !e.foreignWorkspaceBind(run) {
		return runtime.RestoreWorkspaceCheckpointFrom(root, sourceReader)
	}
	meta, _ := e.repo.WorkspaceCheckpoint(run.ID)
	if meta.ArchiveBytes <= runtime.WorkspaceCheckpointMaxBytes {
		archive, err := io.ReadAll(io.LimitReader(sourceReader, runtime.WorkspaceCheckpointMaxBytes+1))
		if err != nil {
			return err
		}
		if len(archive) > runtime.WorkspaceCheckpointMaxBytes {
			return fmt.Errorf("invalid inline checkpoint size")
		}
		response, err := e.requestWorkspaceCheckpoint(ctx, run, domain.RunnerWorkspaceRequest{RunID: run.ID, RunStatus: run.Status, WorkspaceLayout: run.ExecutionContext.RunnerWorkspaceLayout, Operation: domain.RunnerWorkspaceOperationRestore, Checkpoint: archive})
		if err != nil {
			return err
		}
		return workspaceCheckpointResponseError(response)
	}
	manifest, err := e.checkpointTransferRequest(ctx, run, domain.RunnerWorkspaceOperationCheckpointUpload, &domain.WorkspaceCheckpointTransfer{Size: meta.ArchiveBytes, SHA256: meta.SHA256})
	if err != nil {
		return err
	}
	defer e.closeCheckpointTransfer(ctx, run, manifest.ID)
	if manifest.ID == "" || manifest.Size != meta.ArchiveBytes || manifest.SHA256 != meta.SHA256 {
		return fmt.Errorf("invalid checkpoint upload response")
	}
	buffer := make([]byte, runtime.WorkspaceCheckpointChunkBytes)
	for offset := int64(0); offset < meta.ArchiveBytes; {
		size := min(int64(len(buffer)), meta.ArchiveBytes-offset)
		n, err := io.ReadFull(sourceReader, buffer[:size])
		if err != nil {
			return err
		}
		ack, err := e.checkpointTransferRequest(ctx, run, domain.RunnerWorkspaceOperationCheckpointAppend, &domain.WorkspaceCheckpointTransfer{ID: manifest.ID, Offset: offset, Data: buffer[:n]})
		if err != nil {
			return err
		}
		offset += int64(n)
		if ack.ID != manifest.ID || ack.Offset != offset {
			return fmt.Errorf("invalid checkpoint upload acknowledgement")
		}
	}
	_, err = e.checkpointTransferRequest(ctx, run, domain.RunnerWorkspaceOperationCheckpointCommit, &domain.WorkspaceCheckpointTransfer{ID: manifest.ID})
	return err
}

func (e *Engine) verifyStoredCheckpoint(ctx context.Context, id string) error {
	source, err := e.repo.OpenWorkspaceCheckpointArchive(id)
	if err != nil {
		return err
	}
	defer source.Close()
	_, err = io.Copy(io.Discard, workspaceCheckpointContextReader{ctx, source})
	return err
}

type workspaceCheckpointContextReader struct {
	ctx    context.Context
	source io.Reader
}

func (r workspaceCheckpointContextReader) Read(data []byte) (int, error) {
	if err := r.ctx.Err(); err != nil {
		return 0, err
	}
	return r.source.Read(data)
}

type workspaceCheckpointContextWriter struct {
	ctx         context.Context
	destination io.Writer
}

func (w workspaceCheckpointContextWriter) Write(data []byte) (int, error) {
	if err := w.ctx.Err(); err != nil {
		return 0, err
	}
	return w.destination.Write(data)
}