Skip to content

Self-hosted workflows for coding-agent delivery. Orchestrate CodeBuddy, Codex and Claude Code with scripts and human approval gates. Includes Local workflows for coding assistants.

13.8 KB
Raw Blame
package engine

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"agentflow/internal/domain"
	"agentflow/internal/repo"
	"agentflow/internal/runtime"
)

func remoteWorkspaceRestoreFixture(t *testing.T, backend string) (*Engine, *Engine, string) {
	t.Helper()
	var store *repo.Store
	var err error
	if backend == "mysql" {
		dsn := os.Getenv("AGENTFLOW_TEST_MYSQL_DSN")
		if dsn == "" {
			t.Skip("isolated local Docker MySQL required")
		}
		store, err = repo.Open(dsn)
	} else {
		store, err = repo.OpenTest()
	}
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(store.Close)
	id := fmt.Sprintf("restore-retry-%d", time.Now().UnixNano())
	t.Cleanup(func() {
		store.DB.Table("runs").Where("id = ?", id).Update("status", "cancelled")
		if err := store.DeleteRun(id); err != nil {
			t.Error(err)
		}
	})
	worker := New(store, runtime.NewManager(t.TempDir()))
	worker.ConfigureExecution("restore-worker-"+id, "restore-worker-"+id, time.Minute, false)
	worker.controlToken = "restore-retry-control"
	t.Cleanup(worker.checkpointTransfers.Close)
	workflow := domain.WorkflowDefinition{ID: id, SandboxProfileID: "local", Nodes: []domain.WorkflowNode{
		{ID: "prepare", Type: "command"}, {ID: "close", Type: "agent", MaxAttempts: 1},
	}, Edges: []domain.WorkflowEdge{{Source: "prepare", Target: "close", When: "success"}}}
	run := domain.Run{ID: id, WorkflowID: id, WorkflowSnapshot: workflow, Status: "succeeded",
		ExecutionContext: domain.TelemetryContext{ExecutionMode: domain.ExecutionModeCloud},
		Nodes: map[string]*domain.NodeRun{
			"prepare": {NodeID: "prepare", Status: "succeeded", Attempt: 1, Output: "upstream result"},
			"close":   {NodeID: "close", Status: "succeeded", Attempt: 1},
		},
	}
	if err := store.PutRun(run); err != nil {
		t.Fatal(err)
	}
	if err := store.DB.Table("runs").Where("id = ?", id).Updates(map[string]any{
		"workspace_host": worker.workspaceHost, "workspace_root": worker.sandboxes.Root(),
	}).Error; err != nil {
		t.Fatal(err)
	}
	root := worker.RunWorkspace(id)
	if err := os.MkdirAll(root, 0700); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(root, "README.md"), []byte("saved changes"), 0600); err != nil {
		t.Fatal(err)
	}
	worker.reconcileTerminalWorkspaceCleanup()
	run, _ = store.RunState(id)
	if run.WorkspaceCleanedAt.IsZero() {
		t.Fatal("fixture workspace was not reclaimed")
	}
	run.Status = "failed"
	run.Nodes["close"].Status = "failed"
	if err := store.PutRun(run); err != nil {
		t.Fatal(err)
	}
	controller := New(store, runtime.NewManager(t.TempDir()))
	controller.ConfigureExecution("restore-api-"+id, "restore-api-"+id, time.Minute, false)
	controller.controlToken = worker.controlToken
	return worker, controller, id
}

func serveWorkspaceRestoreWorker(t *testing.T, worker *Engine, handler http.Handler) {
	t.Helper()
	server := httptest.NewServer(handler)
	t.Cleanup(server.Close)
	if err := worker.repo.HeartbeatEngineInstance(repo.EngineInstance{
		InstanceID: worker.workspaceHost, Role: "worker", WorkspaceHost: worker.workspaceHost, ControlURL: server.URL,
	}, time.Minute); err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() {
		worker.repo.DB.Table("engine_instances").Where("instance_id = ?", worker.workspaceHost).Delete(nil)
	})
}

func TestConversationWorkspaceRetryAfterLostRestoreReply(t *testing.T) {
	t.Parallel()
	for _, backend := range []string{"sqlite", "mysql"} {
		t.Run(backend, func(t *testing.T) {
			worker, controller, id := remoteWorkspaceRestoreFixture(t, backend)
			var lostReply atomic.Bool
			serveWorkspaceRestoreWorker(t, worker, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				body, err := io.ReadAll(r.Body)
				if err != nil {
					t.Error(err)
					http.Error(w, err.Error(), 500)
					return
				}
				r.Body = io.NopCloser(bytes.NewReader(body))
				var request domain.RunnerWorkspaceRequest
				if err := json.Unmarshal(body, &request); err != nil {
					t.Error(err)
					http.Error(w, err.Error(), 400)
					return
				}
				if request.Operation == domain.RunnerWorkspaceOperationRestore && lostReply.CompareAndSwap(false, true) {
					recorder := httptest.NewRecorder()
					worker.HandleWorkspaceCheckpoint(recorder, r)
					var response domain.RunnerWorkspaceResponse
					if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil || recorder.Code != 200 || response.Error != "" {
						t.Errorf("worker restore failed: %d %s", recorder.Code, recorder.Body.String())
					}
					http.Error(w, "lost restore reply", http.StatusBadGateway)
					return
				}
				worker.HandleWorkspaceCheckpoint(w, r)
			}))
			if err := controller.RestoreConversationWorkspaceIfNeeded(context.Background(), id); err == nil || !strings.Contains(err.Error(), "HTTP 502") {
				t.Fatalf("first restore should lose its reply: %v", err)
			}
			run, _ := worker.repo.RunState(id)
			if run.WorkspaceCleanedAt.IsZero() {
				t.Fatal("cleanup marker cleared without acknowledgement")
			}
			root := worker.RunWorkspace(id)
			if err := os.Remove(filepath.Join(root, "README.md")); err != nil {
				t.Fatal(err)
			}
			if err := os.WriteFile(filepath.Join(root, "newer.md"), []byte("newer surviving changes"), 0600); err != nil {
				t.Fatal(err)
			}
			// Repeat the HTTP preflight and then the actual retry, including context.
			if err := controller.RestoreConversationWorkspaceIfNeeded(context.Background(), id); err != nil {
				t.Fatalf("retry could not reuse the restored workspace: %v", err)
			}
			if err := controller.RetryContext(context.Background(), id, "", "继续", "保留代码,重新核验收尾"); err != nil {
				t.Fatal(err)
			}
			run, _ = worker.repo.RunState(id)
			if run.Status != "running" || !run.WorkspaceCleanedAt.IsZero() || run.WorkspaceHost != worker.workspaceHost || run.WorkspaceRoot != worker.sandboxes.Root() {
				t.Fatalf("retry lost workspace binding: %+v", run)
			}
			if run.Nodes["prepare"].Status != "succeeded" || run.Nodes["prepare"].Attempt != 1 || run.Nodes["prepare"].Output != "upstream result" || run.Nodes["close"].Status != "pending" || !strings.Contains(run.Nodes["close"].Feedback, "保留代码,重新核验收尾") {
				t.Fatalf("retry lost upstream results or human context: %+v", run.Nodes)
			}
			if _, err := os.Stat(filepath.Join(root, "README.md")); !os.IsNotExist(err) {
				t.Fatal("retry resurrected an intentionally deleted file")
			}
			if data, err := os.ReadFile(filepath.Join(root, "newer.md")); err != nil || string(data) != "newer surviving changes" {
				t.Fatalf("retry overwrote surviving changes: %q %v", data, err)
			}
		})
	}
}

func TestConversationWorkspaceRunnerReusesSurvivingDirectory(t *testing.T) {
	t.Parallel()
	for _, scenario := range []string{"no-cleanup", "queued-cleanup", "completed-cleanup", "leased-cleanup", "running-cleanup", "corrupt-snapshot", "legacy-runner", "legacy-chunked-runner"} {
		t.Run(scenario, func(t *testing.T) {
			worker, controller, id := remoteWorkspaceRestoreFixture(t, "sqlite")
			run, _ := worker.repo.RunState(id)
			run.ExecutionContext.ExecutionMode = domain.ExecutionModeRunner
			run.ExecutionContext.ExecutionRunnerID = "restore-runner"
			if err := worker.repo.PutRun(run); err != nil {
				t.Fatal(err)
			}
			cleanupStatus := map[string]string{"queued-cleanup": domain.RunnerTaskQueued, "completed-cleanup": domain.RunnerTaskSuccess, "leased-cleanup": domain.RunnerTaskLeased, "running-cleanup": domain.RunnerTaskRunning}[scenario]
			if cleanupStatus != "" {
				if err := worker.repo.PutRunnerTask(domain.RunnerTask{ID: "rt-cleanup-" + id, RunnerID: "restore-runner", RunID: id, Provider: "terminal", Action: domain.RunnerTaskActionCleanupRun, Status: cleanupStatus}); err != nil {
					t.Fatal(err)
				}
			}
			root := worker.RunWorkspace(id)
			if err := os.MkdirAll(root, 0700); err != nil {
				t.Fatal(err)
			}
			if err := os.WriteFile(filepath.Join(root, "newer.md"), []byte("surviving changes"), 0600); err != nil {
				t.Fatal(err)
			}
			if scenario == "corrupt-snapshot" {
				if err := worker.repo.DB.Model(&repo.WorkspaceCheckpoint{}).Where("run_id = ?", id).Update("sha256", "invalid").Error; err != nil {
					t.Fatal(err)
				}
			}
			calls, restores, exports, closes := 0, 0, 0, 0
			controller.WorkspaceCheckpointTransport = func(ctx context.Context, _ domain.Run, request domain.RunnerWorkspaceRequest) (domain.RunnerWorkspaceResponse, error) {
				calls++
				if request.Operation == domain.RunnerWorkspaceOperationCheckpointStatus {
					if strings.HasPrefix(scenario, "legacy-") {
						return domain.RunnerWorkspaceResponse{Status: 400, Error: "unsupported operation"}, nil
					}
					return runtime.WorkspaceCheckpointStatus(root), nil
				}
				if scenario == "legacy-runner" && request.Operation == domain.RunnerWorkspaceOperationCheckpointExport {
					return domain.RunnerWorkspaceResponse{Status: 400, Error: "unsupported operation"}, nil
				}
				if request.Operation == domain.RunnerWorkspaceOperationCheckpointClose {
					closes++
				}
				if domain.IsWorkspaceCheckpointTransferOperation(request.Operation) {
					response := worker.checkpointTransfers.Handle(ctx, root, request)
					if request.Operation == domain.RunnerWorkspaceOperationCheckpointExport && response.Error == "" {
						exports++
					}
					return response, nil
				}
				if request.Operation == domain.RunnerWorkspaceOperationCheckpoint {
					archive, err := runtime.CreateWorkspaceCheckpoint(root)
					if err != nil {
						return domain.RunnerWorkspaceResponse{Error: err.Error()}, nil
					}
					payload, _ := json.Marshal(map[string][]byte{"checkpoint": archive})
					return domain.RunnerWorkspaceResponse{Payload: payload}, nil
				}
				restores++
				return domain.RunnerWorkspaceResponse{Error: "unexpected restore"}, nil
			}
			err := controller.RestoreConversationWorkspaceIfNeeded(context.Background(), id)
			blocked := scenario == "leased-cleanup" || scenario == "running-cleanup"
			if blocked || scenario == "corrupt-snapshot" {
				if err == nil || blocked && !strings.Contains(err.Error(), "尚在回收") || !blocked && !strings.Contains(err.Error(), "校验失败") {
					t.Fatalf("unsafe workspace was reused: %v", err)
				}
			} else if err != nil {
				t.Fatal(err)
			}
			got, _ := worker.repo.RunState(id)
			if got.WorkspaceCleanedAt.IsZero() != (err == nil) || restores != 0 || blocked && calls != 0 {
				t.Fatalf("cleanup marker or restore calls invalid: marker=%v calls=%d restores=%d", got.WorkspaceCleanedAt, calls, restores)
			}
			if !blocked && (closes != exports || scenario == "legacy-chunked-runner" && exports == 0 || !strings.HasPrefix(scenario, "legacy-") && exports != 0) {
				t.Fatalf("checkpoint probe transfer leaked: exports=%d closes=%d", exports, closes)
			}
			if scenario == "queued-cleanup" {
				cleanup, _ := worker.repo.RunnerTask("rt-cleanup-" + id)
				if cleanup.Status != domain.RunnerTaskCanceled {
					t.Fatal("queued cleanup was not canceled")
				}
			}
			if data, err := os.ReadFile(filepath.Join(root, "newer.md")); err != nil || string(data) != "surviving changes" {
				t.Fatalf("surviving changes lost: %q %v", data, err)
			}
		})
	}
}

func TestConversationWorkspaceRemoteAndRunnerRestoreOverEmptyDirectory(t *testing.T) {
	t.Parallel()
	for _, target := range []string{"runner-empty", "runner-agentflow-only", "foreign-worker-empty", "foreign-worker-agentflow-only"} {
		t.Run(target, func(t *testing.T) {
			worker, controller, id := remoteWorkspaceRestoreFixture(t, "sqlite")
			root := worker.RunWorkspace(id)
			if err := os.MkdirAll(root, 0o700); err != nil {
				t.Fatal(err)
			}
			if strings.HasSuffix(target, "agentflow-only") {
				if err := os.MkdirAll(filepath.Join(root, ".agentflow", "hooks"), 0o700); err != nil {
					t.Fatal(err)
				}
			}
			restores := 0
			if strings.HasPrefix(target, "runner-") {
				run, _ := worker.repo.RunState(id)
				run.ExecutionContext.ExecutionMode = domain.ExecutionModeRunner
				run.ExecutionContext.ExecutionRunnerID = "restore-runner"
				if err := worker.repo.PutRun(run); err != nil {
					t.Fatal(err)
				}
				controller.WorkspaceCheckpointTransport = func(_ context.Context, _ domain.Run, request domain.RunnerWorkspaceRequest) (domain.RunnerWorkspaceResponse, error) {
					if request.Operation == domain.RunnerWorkspaceOperationCheckpointStatus {
						return runtime.WorkspaceCheckpointStatus(root), nil
					}
					if request.Operation == domain.RunnerWorkspaceOperationRestore {
						restores++
						if err := runtime.RestoreWorkspaceCheckpoint(root, request.Checkpoint); err != nil {
							return domain.RunnerWorkspaceResponse{Error: err.Error()}, nil
						}
						return domain.RunnerWorkspaceResponse{Payload: json.RawMessage(`{"restored":true}`)}, nil
					}
					return domain.RunnerWorkspaceResponse{Error: "unexpected operation " + request.Operation}, nil
				}
			} else {
				server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					if r.Method == http.MethodPost {
						var body domain.RunnerWorkspaceRequest
						raw, _ := io.ReadAll(r.Body)
						_ = json.Unmarshal(raw, &body)
						if body.Operation == domain.RunnerWorkspaceOperationRestore || body.Operation == domain.RunnerWorkspaceOperationCheckpointCommit {
							restores++
						}
						r.Body = io.NopCloser(bytes.NewReader(raw))
					}
					worker.HandleWorkspaceCheckpoint(w, r)
				}))
				defer server.Close()
				if err := worker.repo.HeartbeatEngineInstance(repo.EngineInstance{InstanceID: worker.workspaceHost, Role: "worker", WorkspaceHost: worker.workspaceHost, ControlURL: server.URL}, time.Minute); err != nil {
					t.Fatal(err)
				}
			}
			if err := controller.RestoreConversationWorkspaceIfNeeded(context.Background(), id); err != nil {
				t.Fatal(err)
			}
			if restores != 1 {
				t.Fatalf("restores = %d, want 1", restores)
			}
			body, err := os.ReadFile(filepath.Join(root, "README.md"))
			if err != nil || string(body) != "saved changes" {
				t.Fatalf("remote restore lost files: %q (%v)", body, err)
			}
		})
	}
}