Skip to content

Self-hosted workflows for coding-agent delivery. Orchestrate CodeBuddy, Codex and Claude Code with scripts and human approval gates. Includes Local workflows for coding assistants.

39.6 KB
Raw Blame
package runner

import (
	"context"
	"encoding/json"
	"net/http/httptest"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"testing"
	"time"

	"agentflow/internal/domain"
	"agentflow/internal/engine"
	"agentflow/internal/httpapi"
	"agentflow/internal/repo"
	"agentflow/internal/runtime"
	"agentflow/internal/tracker"
)

func TestBindInvocationCredentials(t *testing.T) {
	t.Run("local login leaves codex invocation unchanged", func(t *testing.T) {
		argv := []string{"codex", "exec", "prompt"}
		gotArgv, env, err := bindInvocationCredentials(&domain.RunnerInvocation{CredentialSource: domain.RuntimeAuthLocal}, []runtime.CredentialBinding{runtime.BindCodex}, argv)
		if err != nil || len(env) != 0 || strings.Join(gotArgv, "|") != strings.Join(argv, "|") {
			t.Fatalf("argv=%v env=%v err=%v", gotArgv, env, err)
		}
	})
	t.Run("personal codex injects key and provider config", func(t *testing.T) {
		inv := &domain.RunnerInvocation{CredentialSource: domain.RuntimeAuthPersonal, CodexAPIKey: "codex-secret", CodexBaseURL: "https://gateway.example/v1"}
		argv, env, err := bindInvocationCredentials(inv, []runtime.CredentialBinding{runtime.BindCodex}, []string{"codex", "exec", "prompt"})
		if err != nil {
			t.Fatal(err)
		}
		joined := strings.Join(argv, " ")
		if env["CODEX_API_KEY"] != "codex-secret" || !strings.Contains(joined, "gateway.example/v1") || argv[len(argv)-1] != "prompt" {
			t.Fatalf("argv=%v env=%v", argv, env)
		}
	})
	t.Run("personal claude injects key and base URL", func(t *testing.T) {
		inv := &domain.RunnerInvocation{CredentialSource: domain.RuntimeAuthPersonal, ClaudeCodeAPIKey: "claude-secret", ClaudeCodeBaseURL: "https://claude.example"}
		_, env, err := bindInvocationCredentials(inv, []runtime.CredentialBinding{runtime.BindClaudeCode}, []string{"claude", "-p", "prompt"})
		if err != nil || env["ANTHROPIC_API_KEY"] != "claude-secret" || env["ANTHROPIC_BASE_URL"] != "https://claude.example" {
			t.Fatalf("env=%v err=%v", env, err)
		}
	})
}

func TestBindSandboxPrivateHomeUsesContainerPaths(t *testing.T) {
	workspace := t.TempDir()
	env, err := bindSandboxPrivateHome(workspace, "docker", "codex", map[string]string{"CODEX_API_KEY": "secret"})
	if err != nil {
		t.Fatal(err)
	}
	if env["HOME"] != "/workspace/.agentflow/private-home/codex" || env["CODEX_HOME"] != "/workspace/.agentflow/private-home/codex/.codex" {
		t.Fatalf("container private home = %+v", env)
	}
	if env["CODEX_API_KEY"] != "secret" {
		t.Fatal("credential environment was lost")
	}
}

func TestWorkspaceContainsPath(t *testing.T) {
	root := filepath.Join(t.TempDir(), "runs", "run-1")
	if !workspaceContainsPath(root, filepath.Join(root, "node-a")) {
		t.Fatal("node workspace should belong to run workspace")
	}
	if !workspaceContainsPath(root, root) {
		t.Fatal("run workspace should contain itself")
	}
	if workspaceContainsPath(root, filepath.Join(filepath.Dir(root), "run-10", "node-a")) {
		t.Fatal("sibling run workspace must not match by string prefix")
	}
}

func TestWriteRunnerHumanContext(t *testing.T) {
	workspace := t.TempDir()
	if err := writeRunnerHumanContext(workspace, "operator fact"); err != nil {
		t.Fatal(err)
	}
	raw, err := os.ReadFile(filepath.Join(workspace, "human-context.md"))
	if err != nil || !strings.Contains(string(raw), "operator fact") {
		t.Fatalf("human context file=%q err=%v", raw, err)
	}
	if err := writeRunnerHumanContext(workspace, ""); err != nil {
		t.Fatal(err)
	}
	raw, err = os.ReadFile(filepath.Join(workspace, "human-context.md"))
	if err != nil || !strings.Contains(string(raw), "operator fact") {
		t.Fatalf("empty context should not delete a workspace file, file=%q err=%v", raw, err)
	}
}

// newDaemonTestServer starts a real control-plane handler backed by the
// in-memory store, with runner registration enabled.
func newDaemonTestServer(t *testing.T) (*httptest.Server, *repo.Store, *engine.Engine) {
	t.Helper()
	root := t.TempDir()
	s, err := repo.OpenTest()
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { s.Close() })
	e := engine.New(s, runtime.NewManager(filepath.Join(root, "workspaces")))
	h := httpapi.New(s, e, tracker.NewService(), "", "")
	server := httptest.NewServer(h)
	t.Cleanup(server.Close)
	t.Setenv("AGENTFLOW_RUNNER_BOOTSTRAP_TOKEN", "boot-secret")
	return server, s, e
}

func putMockDeveloperPlaybook(t *testing.T, s *repo.Store) {
	t.Helper()
	agent, ok := s.Agent("developer")
	if !ok {
		t.Fatal("developer role missing")
	}
	agent.ExecutorID = "mock"
	agent.Prompt = "{{input}}"
	if err := s.PutAgent(agent); err != nil {
		t.Fatal(err)
	}
}

// waitFor polls until cond is true or the deadline expires.
func waitFor(t *testing.T, timeout time.Duration, what string, cond func() bool) {
	t.Helper()
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		if cond() {
			return
		}
		time.Sleep(50 * time.Millisecond)
	}
	t.Fatalf("timed out waiting for %s", what)
}

func TestDeleteFailedRunCleansRunnerWorkspace(t *testing.T) {
	server, s, e := newDaemonTestServer(t)
	workspaceRoot := filepath.Join(t.TempDir(), "runner-workspace")
	daemon, err := New(Config{
		ServerURL:            server.URL,
		BootstrapToken:       "boot-secret",
		StatePath:            filepath.Join(t.TempDir(), "state.json"),
		Name:                 "delete-cleanup-runner",
		MaxConcurrency:       1,
		HeartbeatInterval:    time.Second,
		LongPollTimeout:      2 * time.Second,
		DefaultWorkspaceRoot: workspaceRoot,
	})
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	t.Cleanup(cancel)
	go func() { _ = daemon.Run(ctx) }()
	waitFor(t, 5*time.Second, "runner registration", func() bool { return len(s.Runners()) == 1 })
	runnerID := s.Runners()[0].ID
	run := domain.Run{
		ID:     "failed-delete-cleanup",
		Status: "failed",
		ExecutionContext: domain.TelemetryContext{
			ExecutionMode:     domain.ExecutionModeRunner,
			ExecutionRunnerID: runnerID,
		},
	}
	if err := s.PutRun(run); err != nil {
		t.Fatal(err)
	}
	workspace := filepath.Join(workspaceRoot, "runs", run.ID)
	if err := os.MkdirAll(workspace, 0o700); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(workspace, "partial.txt"), []byte("failed output"), 0o600); err != nil {
		t.Fatal(err)
	}

	if err := e.DeleteRun(run.ID); err != nil {
		t.Fatal(err)
	}
	if _, ok := s.Run(run.ID); ok {
		t.Fatal("run record was not deleted")
	}
	waitFor(t, 10*time.Second, "runner workspace cleanup after delete", func() bool {
		task, ok := s.RunnerTask("rt-cleanup-" + run.ID)
		if !ok || task.Status != domain.RunnerTaskSuccess {
			return false
		}
		_, statErr := os.Stat(workspace)
		return os.IsNotExist(statErr)
	})
}

func TestWorkflowScriptsShareRunnerRunWorkspace(t *testing.T) {
	server, s, e := newDaemonTestServer(t)
	repository := t.TempDir()
	if err := os.MkdirAll(filepath.Join(repository, "vendor", "tracked"), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(repository, "vendor", "tracked", "dependency.php"), []byte("baseline\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	for _, args := range [][]string{
		{"init", "-q", "-b", "master"},
		{"config", "user.email", "[email protected]"},
		{"config", "user.name", "Runner Test"},
		{"add", "vendor/tracked/dependency.php"},
		{"commit", "-qm", "baseline"},
	} {
		cmd := exec.Command("git", append([]string{"-C", repository}, args...)...)
		if output, err := cmd.CombinedOutput(); err != nil {
			t.Fatalf("git %v: %v: %s", args, err, output)
		}
	}
	daemon, err := New(Config{
		ServerURL:            server.URL,
		BootstrapToken:       "boot-secret",
		StatePath:            filepath.Join(t.TempDir(), "state.json"),
		Name:                 "shared-script-runner",
		MaxConcurrency:       1,
		HeartbeatInterval:    time.Second,
		LongPollTimeout:      2 * time.Second,
		DefaultWorkspaceRoot: filepath.Join(t.TempDir(), "workspace"),
	})
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	t.Cleanup(cancel)
	go func() { _ = daemon.Run(ctx) }()
	waitFor(t, 5*time.Second, "runner registration", func() bool { return len(s.Runners()) == 1 })
	runner := s.Runners()[0]
	runner.OwnerUserID = 42
	if err := s.PutRunner(runner); err != nil {
		t.Fatal(err)
	}
	if err := s.SetDefaultRunner(42, runner.ID, time.Now().UTC()); err != nil {
		t.Fatal(err)
	}
	wf := domain.WorkflowDefinition{
		ID: "runner-shared-scripts", Name: "Runner shared scripts", SandboxProfileID: "local",
		Nodes: []domain.WorkflowNode{
			{ID: "prepare", Name: "Prepare", Type: "script", ScriptLanguage: "shell", ExecutionScope: domain.ExecutionScopeRepositories, Script: "test \"$(git branch --show-current)\" = fix/shared-runner-workspace && mkdir -p .git/info vendor/new && printf '/vendor/\\n' >> .git/info/exclude && git ls-files -z -- vendor/ | git update-index --skip-worktree -z --stdin && printf 'changed\\n' > vendor/tracked/dependency.php && printf 'new\\n' > vendor/new/dependency.php && printf 'shared-ready\\n' > workspace-marker", MaxAttempts: 1},
			{ID: "consume", Name: "Consume", Type: "script", ScriptLanguage: "python", ExecutionScope: domain.ExecutionScopeRepositories, Script: "from pathlib import Path\nimport subprocess\nassert subprocess.check_output(['git', 'status', '--short', '--', 'vendor']).decode() == ''\nprint(Path(\"workspace-marker\").read_text().strip() + \" it's visible\")", MaxAttempts: 1},
		},
		Edges: []domain.WorkflowEdge{{ID: "prepare-consume", Source: "prepare", Target: "consume", When: "success"}},
	}
	if err := s.PutWorkflow(wf); err != nil {
		t.Fatal(err)
	}
	run, err := e.StartWithExecutionContext(wf.ID, "", domain.RepositoryContext{RepositoryID: "repo", CloneURL: repository, BaseBranch: "master", WorkBranch: "fix/shared-runner-workspace"}, domain.TelemetryContext{ResponsibleUserID: 42})
	if err != nil {
		t.Fatal(err)
	}
	waitFor(t, 15*time.Second, "shared runner script workflow", func() bool {
		current, ok := s.Run(run.ID)
		return ok && (current.Status == "succeeded" || current.Status == "failed")
	})
	final, _ := s.Run(run.ID)
	if final.Status != "succeeded" {
		var events []domain.RunnerTaskEvent
		for _, task := range s.RunnerTasks() {
			if task.RunID == run.ID {
				events = append(events, s.RunnerTaskEvents(task.ID)...)
			}
		}
		t.Fatalf("script workflow status = %s, prepare error = %q, consume error = %q, events = %+v", final.Status, final.Nodes["prepare"].Error, final.Nodes["consume"].Error, events)
	}
	if final.ExecutionContext.RunnerWorkspaceLayout != domain.RunnerWorkspaceLayoutShared {
		t.Fatalf("runner workspace layout was not frozen: %+v", final.ExecutionContext)
	}
	if output := final.Nodes["consume"].Output; !strings.Contains(output, "shared-ready it's visible") {
		t.Fatalf("downstream script did not see shared workspace: %q", output)
	}
	wantWorkspace := filepath.ToSlash(filepath.Join("runs", run.ID, "workspace"))
	var scriptTasks int
	for _, task := range s.RunnerTasks() {
		if task.RunID != run.ID || task.Action != "" {
			continue
		}
		scriptTasks++
		if task.Provider != "terminal" || filepath.ToSlash(task.Workspace) != wantWorkspace {
			t.Fatalf("script task did not use shared runner workspace: %+v", task)
		}
	}
	if scriptTasks != 2 {
		t.Fatalf("runner script task count = %d, want 2", scriptTasks)
	}
}

func TestConfigureRepositoryGitIdentityOverridesStaleLocalConfig(t *testing.T) {
	repository := t.TempDir()
	for _, args := range [][]string{
		{"init", "-q", "-b", "master"},
		{"config", "--local", "user.name", "Stale User"},
		{"config", "--local", "user.email", "[email protected]"},
	} {
		if output, err := exec.Command("git", append([]string{"-C", repository}, args...)...).CombinedOutput(); err != nil {
			t.Fatalf("git %v: %v: %s", args, err, output)
		}
	}
	if err := configureRepositoryGitIdentity(context.Background(), repository, "example-admin", "[email protected]"); err != nil {
		t.Fatal(err)
	}
	for key, want := range map[string]string{"user.name": "example-admin", "user.email": "[email protected]"} {
		output, err := exec.Command("git", "-C", repository, "config", "--local", key).Output()
		if err != nil || strings.TrimSpace(string(output)) != want {
			t.Fatalf("%s = %q, want %q (err=%v)", key, output, want, err)
		}
	}
}

func TestCheckoutInvocationRepositoriesLeavesAgentConfigVisible(t *testing.T) {
	root := t.TempDir()
	invocation := &domain.RunnerInvocation{}
	for _, name := range []string{"frontend", "backend"} {
		repository := filepath.Join(root, name+"-source")
		if err := os.MkdirAll(filepath.Join(repository, ".codebuddy"), 0o755); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(filepath.Join(repository, "AGENTS.md"), []byte("project instructions\n"), 0o644); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(filepath.Join(repository, ".codebuddy", "settings.json"), []byte("{}\n"), 0o644); err != nil {
			t.Fatal(err)
		}
		for _, args := range [][]string{
			{"init", "-q", "-b", "master"},
			{"config", "user.email", "[email protected]"},
			{"config", "user.name", "Runner Test"},
			{"add", "AGENTS.md", ".codebuddy/settings.json"},
			{"commit", "-qm", "baseline"},
		} {
			cmd := exec.Command("git", append([]string{"-C", repository}, args...)...)
			if output, err := cmd.CombinedOutput(); err != nil {
				t.Fatalf("git %s %v: %v: %s", name, args, err, output)
			}
		}
		invocation.Repositories = append(invocation.Repositories, domain.RunnerRepository{CloneURL: repository, Ref: "master", Dir: name})
	}
	workspace := filepath.Join(root, "workspace")
	if err := os.MkdirAll(workspace, 0o755); err != nil {
		t.Fatal(err)
	}
	if err := checkoutInvocationRepositories(context.Background(), workspace, invocation); err != nil {
		t.Fatal(err)
	}
	for _, name := range []string{"frontend", "backend"} {
		repository := filepath.Join(workspace, name)
		cmd := exec.Command("git", "-C", repository, "ls-files", "-v", "--", "AGENTS.md", ".codebuddy/settings.json")
		output, err := cmd.CombinedOutput()
		if err != nil {
			t.Fatalf("git ls-files %s: %v: %s", name, err, output)
		}
		for _, line := range strings.Split(strings.TrimSpace(string(output)), "\n") {
			if !strings.HasPrefix(line, "H ") {
				t.Fatalf("%s repository instruction file was hidden from Git: %q", name, line)
			}
		}
		exclude, err := os.ReadFile(filepath.Join(repository, ".git", "info", "exclude"))
		if err != nil || strings.Contains(string(exclude), ".agents/") {
			t.Fatalf("%s repository instruction excludes: %v %q", name, err, exclude)
		}
	}
}

func TestDaemonExecutesMockTaskEndToEnd(t *testing.T) {
	server, s, _ := newDaemonTestServer(t)

	statePath := filepath.Join(t.TempDir(), "state.json")
	daemon, err := New(Config{
		ServerURL:            server.URL,
		BootstrapToken:       "boot-secret",
		StatePath:            statePath,
		Name:                 "test-daemon",
		MaxConcurrency:       1,
		HeartbeatInterval:    time.Second,
		LongPollTimeout:      2 * time.Second,
		DefaultWorkspaceRoot: filepath.Join(t.TempDir(), "workspace"),
	})
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go func() { _ = daemon.Run(ctx) }()

	// The runner registers with its mock runtime.
	waitFor(t, 5*time.Second, "runner registration", func() bool {
		runners := s.Runners()
		return len(runners) == 1 && runners[0].Status == domain.RunnerStatusOnline
	})

	// Create a task through the management API.
	task := domain.RunnerTask{
		ID: "rt-e2e", Provider: mockProviderKind, Prompt: "hello end to end",
		Status: domain.RunnerTaskQueued, MaxAttempts: 1,
	}
	if err := s.PutRunnerTask(task); err != nil {
		t.Fatal(err)
	}

	// The daemon claims it, executes the mock provider, and completes it.
	waitFor(t, 15*time.Second, "task completion", func() bool {
		final, ok := s.RunnerTask("rt-e2e")
		return ok && final.Status == domain.RunnerTaskSuccess
	})
	final, _ := s.RunnerTask("rt-e2e")
	if final.Result != "MOCK hello end to end" {
		t.Fatalf("unexpected result: %q", final.Result)
	}
	if final.ExitCode == nil || *final.ExitCode != 0 {
		t.Fatalf("unexpected exit code: %+v", final.ExitCode)
	}
	// Execution events were reported with seq keys.
	events := s.RunnerTaskEvents("rt-e2e")
	if len(events) == 0 {
		t.Fatal("no execution events reported")
	}

	// State was persisted so a restart reuses the same runner id.
	raw, err := os.ReadFile(statePath)
	if err != nil {
		t.Fatal(err)
	}
	var st struct {
		RunnerID string `json:"runnerId"`
	}
	if err := json.Unmarshal(raw, &st); err != nil {
		t.Fatal(err)
	}
	if st.RunnerID == "" {
		t.Fatal("state file missing runner id")
	}
	cancel()
}

func TestDaemonRestartImmediatelyRestoresPresence(t *testing.T) {
	server, s, _ := newDaemonTestServer(t)
	cfg := Config{
		ServerURL: server.URL, BootstrapToken: "boot-secret",
		StatePath:         filepath.Join(t.TempDir(), "state.json"),
		HeartbeatInterval: time.Hour,
	}
	first, err := New(cfg)
	if err != nil {
		t.Fatal(err)
	}
	if err := first.ensureRegistered(context.Background()); err != nil {
		t.Fatal(err)
	}
	first.disconnect(context.Background())
	before, _ := s.Runner(first.runnerID())
	if before.Status != domain.RunnerStatusOffline {
		t.Fatal("daemon shutdown did not mark runner offline")
	}
	restarted, err := New(cfg)
	if err != nil {
		t.Fatal(err)
	}
	if err := restarted.ensureRegistered(context.Background()); err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan struct{})
	go func() { defer close(done); restarted.heartbeatLoop(ctx) }()
	t.Cleanup(func() { cancel(); <-done })
	waitFor(t, 2*time.Second, "immediate restart heartbeat", func() bool {
		current, _ := s.Runner(first.runnerID())
		return current.Status == domain.RunnerStatusOnline
	})
	if restarted.runnerID() != first.runnerID() || restarted.clientSession() != first.clientSession() || len(s.Runners()) != 1 {
		t.Fatal("restart must retain the runner identity and session")
	}
}

func TestDaemonExecuteProcessRunsRealCommand(t *testing.T) {
	server, s, _ := newDaemonTestServer(t)
	_ = server

	daemon, err := New(Config{
		ServerURL:            server.URL,
		BootstrapToken:       "boot-secret",
		StatePath:            filepath.Join(t.TempDir(), "state.json"),
		DefaultWorkspaceRoot: filepath.Join(t.TempDir(), "workspace"),
	})
	if err != nil {
		t.Fatal(err)
	}

	// Register the daemon so its client has a session, then drive
	// executeProcess directly with a provider that definitely exists: the
	// test binary itself invoked through `go`... simpler: use /bin/echo via a
	// synthetic runtime entry and a custom provider through ExtraBinaries.
	daemon, err = New(Config{
		ServerURL:            server.URL,
		BootstrapToken:       "boot-secret",
		StatePath:            filepath.Join(t.TempDir(), "state.json"),
		DefaultWorkspaceRoot: filepath.Join(t.TempDir(), "workspace"),
		ExtraBinaries:        map[string]string{"echo": "echo"},
	})
	if err != nil {
		t.Fatal(err)
	}
	ctx := context.Background()
	if err := daemon.ensureRegistered(ctx); err != nil {
		t.Fatal(err)
	}

	task := domain.RunnerTask{
		ID: "rt-echo", Provider: "echo", Prompt: "direct exec", Status: domain.RunnerTaskQueued, MaxAttempts: 1,
	}
	if err := s.PutRunnerTask(task); err != nil {
		t.Fatal(err)
	}
	claimed, ok, err := s.ClaimRunnerTask(daemon.runnerID(), []string{"echo"}, time.Now().UTC(), time.Minute)
	if err != nil || !ok {
		t.Fatalf("claim: ok=%v err=%v", ok, err)
	}

	// "echo" is a registered ExtraBinaries provider with no template, so
	// executeTask will report "provider not supported"; drive the real
	// process path directly with an explicit argv instead, then complete the
	// task the same way executeTask would.
	argv := []string{"echo", "hello-process"}
	result := daemon.executeProcess(ctx, claimed, argv)
	if result.status != domain.RunnerTaskSuccess {
		t.Fatalf("unexpected status %q err %q", result.status, result.errMsg)
	}
	if result.exitCode != 0 {
		t.Fatalf("exit code = %d", result.exitCode)
	}
	if !strings.Contains(result.result, "hello-process") {
		t.Fatalf("missing command stdout in result %q", result.result)
	}
	if strings.Contains(result.result, "file already closed") {
		t.Fatalf("stream error leaked into result %q", result.result)
	}
	if err := s.CompleteRunnerTask("rt-echo", daemon.runnerID(), domain.RunnerTaskSuccess, &result.exitCode, result.result, result.errMsg, nil, nil, time.Now().UTC()); err != nil {
		t.Fatal(err)
	}
	final, _ := s.RunnerTask("rt-echo")
	if final.Status != domain.RunnerTaskSuccess {
		t.Fatalf("final task status = %s", final.Status)
	}
	events := s.RunnerTaskEvents("rt-echo")
	if len(events) == 0 {
		t.Fatal("no events streamed for real process")
	}
}

func TestExecuteProcessCapturesFastCommandStdout(t *testing.T) {
	server, s, _ := newDaemonTestServer(t)
	daemon, err := New(Config{
		ServerURL:            server.URL,
		BootstrapToken:       "boot-secret",
		StatePath:            filepath.Join(t.TempDir(), "state.json"),
		DefaultWorkspaceRoot: filepath.Join(t.TempDir(), "workspace"),
		ExtraBinaries:        map[string]string{"echo": "echo"},
	})
	if err != nil {
		t.Fatal(err)
	}
	ctx := context.Background()
	if err := daemon.ensureRegistered(ctx); err != nil {
		t.Fatal(err)
	}
	for i := 0; i < 20; i++ {
		id := "rt-fast-" + strconv.Itoa(i)
		task := domain.RunnerTask{
			ID: id, Provider: "echo", Prompt: "fast", Status: domain.RunnerTaskQueued, MaxAttempts: 1,
		}
		if err := s.PutRunnerTask(task); err != nil {
			t.Fatal(err)
		}
		claimed, ok, err := s.ClaimRunnerTask(daemon.runnerID(), []string{"echo"}, time.Now().UTC(), time.Minute)
		if err != nil || !ok {
			t.Fatalf("claim %s: ok=%v err=%v", id, ok, err)
		}
		result := daemon.executeProcess(ctx, claimed, []string{"/bin/sh", "-lc", "echo gate-ran-on-runner"})
		if result.status != domain.RunnerTaskSuccess {
			t.Fatalf("iter %d status=%q err=%q result=%q", i, result.status, result.errMsg, result.result)
		}
		if !strings.Contains(result.result, "gate-ran-on-runner") {
			t.Fatalf("iter %d missing stdout: %q", i, result.result)
		}
		if strings.Contains(result.result, "file already closed") {
			t.Fatalf("iter %d leaked stream error into result: %q", i, result.result)
		}
		if err := s.CompleteRunnerTask(id, daemon.runnerID(), domain.RunnerTaskSuccess, &result.exitCode, result.result, result.errMsg, nil, nil, time.Now().UTC()); err != nil {
			t.Fatal(err)
		}
	}
}

func TestExecuteProcessStoresVisibleTextNotStreamJSON(t *testing.T) {
	server, s, _ := newDaemonTestServer(t)
	daemon, err := New(Config{
		ServerURL:            server.URL,
		BootstrapToken:       "boot-secret",
		StatePath:            filepath.Join(t.TempDir(), "state.json"),
		DefaultWorkspaceRoot: filepath.Join(t.TempDir(), "workspace"),
	})
	if err != nil {
		t.Fatal(err)
	}
	if err := daemon.ensureRegistered(context.Background()); err != nil {
		t.Fatal(err)
	}
	task := domain.RunnerTask{
		ID: "rt-visible", Provider: "codebuddy", Status: domain.RunnerTaskQueued, MaxAttempts: 1,
		Invocation: &domain.RunnerInvocation{Executor: domain.ExecutorDefinition{Kind: "codebuddy", Protocol: "stream-json", Model: "test"}},
	}
	if err := s.PutRunnerTask(task); err != nil {
		t.Fatal(err)
	}
	claimed, ok, err := s.ClaimRunnerTask(daemon.runnerID(), []string{"codebuddy"}, time.Now().UTC(), time.Minute)
	if err != nil || !ok {
		t.Fatalf("claim: ok=%v err=%v", ok, err)
	}
	script := `printf '%s\n' '{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"plain answer"}]}}' '{"type":"result","result":"final answer","is_error":false}'`
	result := daemon.executeProcess(context.Background(), claimed, []string{"/bin/sh", "-c", script})
	if result.status != domain.RunnerTaskSuccess {
		t.Fatalf("status=%s err=%s", result.status, result.errMsg)
	}
	if result.result != "final answer" {
		t.Fatalf("node output = %q", result.result)
	}
	if strings.Contains(result.result, `"type":"assistant"`) || strings.Contains(result.result, `\"`) {
		t.Fatalf("stream envelope leaked into node output: %q", result.result)
	}
}

func TestDaemonStreamsAndNormalizesSparseProviderOutput(t *testing.T) {
	server, s, _ := newDaemonTestServer(t)
	daemon, err := New(Config{
		ServerURL:            server.URL,
		BootstrapToken:       "boot-secret",
		StatePath:            filepath.Join(t.TempDir(), "state.json"),
		DefaultWorkspaceRoot: filepath.Join(t.TempDir(), "workspace"),
	})
	if err != nil {
		t.Fatal(err)
	}
	if err := daemon.ensureRegistered(context.Background()); err != nil {
		t.Fatal(err)
	}
	task := domain.RunnerTask{
		ID: "rt-observable", Provider: "codex", Status: domain.RunnerTaskQueued, MaxAttempts: 1,
		Invocation: &domain.RunnerInvocation{Executor: domain.ExecutorDefinition{Kind: "codex", Protocol: "jsonl", Model: "gpt-test"}},
	}
	if err := s.PutRunnerTask(task); err != nil {
		t.Fatal(err)
	}
	claimed, ok, err := s.ClaimRunnerTask(daemon.runnerID(), []string{"codex"}, time.Now().UTC(), time.Minute)
	if err != nil || !ok {
		t.Fatalf("claim: ok=%v err=%v", ok, err)
	}

	done := make(chan execResult, 1)
	line := `{"type":"turn.completed","thread_id":"thread-remote","usage":{"input_tokens":12,"output_tokens":5}}`
	go func() {
		done <- daemon.executeProcess(context.Background(), claimed, []string{"/bin/sh", "-c", "printf '%s\\n' '" + line + "'; printf 'provider warning\\n' >&2; sleep 2"})
	}()

	// Sparse output must be visible while the process is still running; it
	// must not wait for the old 16 KiB threshold or process completion.
	waitFor(t, 1500*time.Millisecond, "incremental runner output", func() bool {
		for _, event := range s.RunnerTaskEvents(task.ID) {
			if event.Type == "stdout" && strings.Contains(event.Data, "turn.completed") {
				return true
			}
		}
		return false
	})
	select {
	case <-done:
		t.Fatal("process completed before incremental-output assertion")
	default:
	}

	result := <-done
	if result.status != domain.RunnerTaskSuccess {
		t.Fatalf("status=%s err=%s", result.status, result.errMsg)
	}
	foundUsage, foundStderr, foundCompleted := false, false, false
	for _, event := range s.RunnerTaskEvents(task.ID) {
		if event.Type != "observation" {
			continue
		}
		var observation domain.TelemetryEvent
		if json.Unmarshal([]byte(event.Data), &observation) == nil && observation.Category == "usage" && observation.Usage != nil && observation.Usage.InputTokens == 12 {
			foundUsage = true
		}
		if observation.Name == "process.stderr" && observation.Message == "provider warning" {
			foundStderr = true
		}
		if observation.Name == "agent.completed" {
			foundCompleted = true
		}
	}
	if !foundUsage {
		t.Fatal("normalized usage observation missing from runner event stream")
	}
	if !foundStderr || !foundCompleted {
		t.Fatalf("terminal observations missing: stderr=%v completed=%v", foundStderr, foundCompleted)
	}
}

// TestWorkflowAgentNodeDispatchesToRunner is the headline integration case:
// a role bound to a runner machine (AgentDefinition.RunnerID) executes its
// workflow Agent node on the runner daemon, and the run finishes successfully
// with the runner result wired back into the node.
func TestWorkflowAgentNodeDispatchesToRunner(t *testing.T) {
	server, s, e := newDaemonTestServer(t)
	agent, found := s.Agent("developer")
	if !found {
		t.Fatal("seed agent developer missing")
	}
	agent.ExecutorID = "mock"
	daemon, err := New(Config{
		ServerURL:            server.URL,
		BootstrapToken:       "boot-secret",
		StatePath:            filepath.Join(t.TempDir(), "state.json"),
		Name:                 "workflow-runner",
		MaxConcurrency:       1,
		HeartbeatInterval:    time.Second,
		LongPollTimeout:      2 * time.Second,
		DefaultWorkspaceRoot: filepath.Join(t.TempDir(), "workspace"),
	})
	if err != nil {
		t.Fatal(err)
	}
	workspaceRoot := daemon.cfg.DefaultWorkspaceRoot
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go func() { _ = daemon.Run(ctx) }()
	waitFor(t, 5*time.Second, "runner registration", func() bool {
		runners := s.Runners()
		return len(runners) == 1 && runners[0].Status == domain.RunnerStatusOnline
	})

	// The run's responsible user selects the runner as their default
	// execution machine: routing is a user-level decision.
	runner := s.Runners()[0]
	runner.OwnerUserID = 42
	if err := s.PutRunner(runner); err != nil {
		t.Fatal(err)
	}
	if err := s.SetDefaultRunner(42, runner.ID, time.Now().UTC()); err != nil {
		t.Fatal(err)
	}
	putMockDeveloperPlaybook(t, s)
	wf := domain.WorkflowDefinition{
		ID: "runner-e2e", Name: "Runner e2e", SandboxProfileID: "local",
		Nodes: []domain.WorkflowNode{{ID: "work", Name: "Work", Type: "agent", AgentID: "developer", ExecutorID: "mock", MaxAttempts: 1}},
	}
	if err := s.PutWorkflow(wf); err != nil {
		t.Fatal(err)
	}
	run, err := e.StartWithExecutionContext(wf.ID, "runner integration input", domain.RepositoryContext{}, domain.TelemetryContext{ResponsibleUserID: 42})
	if err != nil {
		t.Fatal(err)
	}
	waitFor(t, 30*time.Second, "run completion", func() bool {
		current, _ := s.Run(run.ID)
		return current.Status == "succeeded"
	})
	waitFor(t, 10*time.Second, "runner workspace immediate cleanup", func() bool {
		_, statErr := os.Stat(filepath.Join(workspaceRoot, "runs", run.ID))
		current, ok := s.Run(run.ID)
		return ok && os.IsNotExist(statErr) && !current.WorkspaceCleanedAt.IsZero()
	})
	final, _ := s.Run(run.ID)
	node := final.Nodes["work"]
	if node == nil || node.Status != "succeeded" {
		t.Fatalf("unexpected node state: %+v", node)
	}
	if !strings.Contains(node.Output, "runner integration input") {
		t.Fatalf("node output missing runner result: %q", node.Output)
	}
	// The dispatch created a workflow-linked runner task.
	allTasks := s.RunnerTasks()
	tasks := make([]domain.RunnerTask, 0, len(allTasks))
	for _, task := range allTasks {
		if task.Action == "" {
			tasks = append(tasks, task)
		}
	}
	if len(tasks) != 1 {
		t.Fatalf("expected exactly one execution task, got %d (%d including cleanup)", len(tasks), len(allTasks))
	}
	if tasks[0].RunID != run.ID || tasks[0].NodeID != "work" || tasks[0].Status != domain.RunnerTaskSuccess {
		t.Fatalf("unexpected runner task: %+v", tasks[0])
	}
	// The dispatch is visible in the run timeline.
	foundDispatch := false
	for _, ev := range final.Telemetry {
		if ev.Name == "runner.dispatch" {
			foundDispatch = true
		}
	}
	if !foundDispatch {
		t.Fatal("runner.dispatch telemetry missing from run timeline")
	}
	cancel()
}

func TestRunnerPauseWaitsForRemoteStopAndRestartsScriptInSameWorkspace(t *testing.T) {
	server, s, e := newDaemonTestServer(t)
	workspaceRoot := filepath.Join(t.TempDir(), "workspace")
	daemon, err := New(Config{
		ServerURL:            server.URL,
		BootstrapToken:       "boot-secret",
		StatePath:            filepath.Join(t.TempDir(), "state.json"),
		Name:                 "pause-runner",
		MaxConcurrency:       1,
		HeartbeatInterval:    time.Second,
		LongPollTimeout:      2 * time.Second,
		DefaultWorkspaceRoot: workspaceRoot,
	})
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go func() { _ = daemon.Run(ctx) }()
	waitFor(t, 5*time.Second, "runner registration", func() bool {
		return len(s.Runners()) == 1 && s.Runners()[0].Status == domain.RunnerStatusOnline
	})
	waitFor(t, 5*time.Second, "runner websocket control connection", func() bool {
		return daemon.wsOnline.Load()
	})
	runner := s.Runners()[0]
	runner.OwnerUserID = 42
	if err := s.PutRunner(runner); err != nil {
		t.Fatal(err)
	}
	if err := s.SetDefaultRunner(42, runner.ID, time.Now().UTC()); err != nil {
		t.Fatal(err)
	}
	wf := domain.WorkflowDefinition{ID: "runner-pause", Name: "Runner pause", SandboxProfileID: "local", Nodes: []domain.WorkflowNode{{
		ID: "work", Name: "Work", Type: "command", Command: "if test -f .script-started; then printf restarted; else touch .script-started; sleep 30; fi", MaxAttempts: 1,
	}}}
	if err := s.PutWorkflow(wf); err != nil {
		t.Fatal(err)
	}
	run, err := e.StartWithExecutionContext(wf.ID, "pause remote", domain.RepositoryContext{}, domain.TelemetryContext{ResponsibleUserID: 42})
	if err != nil {
		t.Fatal(err)
	}
	waitFor(t, 10*time.Second, "remote task running with persistent marker", func() bool {
		active, _ := s.ActiveRunnerTasksForRun(run.ID)
		if len(active) != 1 || active[0].Status != domain.RunnerTaskRunning {
			return false
		}
		_, statErr := os.Stat(filepath.Join(workspaceRoot, filepath.FromSlash(active[0].Workspace), ".script-started"))
		return statErr == nil
	})
	pauseStarted := time.Now()
	if err := e.Pause(run.ID); err != nil {
		t.Fatal(err)
	}
	waitFor(t, 10*time.Second, "remote process stopped", func() bool {
		current, _ := s.Run(run.ID)
		active, _ := s.ActiveRunnerTasksForRun(run.ID)
		return current.Status == "waiting" && current.PauseRequested && len(active) == 0
	})
	if elapsed := time.Since(pauseStarted); elapsed >= 5*time.Second {
		t.Fatalf("websocket pause took %s; expected server push before HTTP fallback", elapsed)
	}
	if err := e.ResumePaused(run.ID, "this must not reach a script"); err != nil {
		t.Fatal(err)
	}
	waitFor(t, 15*time.Second, "resumed remote run", func() bool {
		current, _ := s.Run(run.ID)
		return current.Status == "succeeded"
	})
	final, _ := s.Run(run.ID)
	if final.HumanContext != "" || final.Nodes["work"].Status != "succeeded" || !strings.Contains(final.Nodes["work"].Output, "restarted") {
		t.Fatalf("resumed remote run=%+v node=%+v", final, final.Nodes["work"])
	}
}

// TestRunnerBoundRoleRunsGatesOnRunnerMachine verifies the terminal
// capability: when a role is bound to a runner, its verification gates also
// execute on that machine, inside the same task workspace as the CLI run.
func TestRunnerBoundRoleRunsGatesOnRunnerMachine(t *testing.T) {
	server, s, e := newDaemonTestServer(t)
	agent, found := s.Agent("developer")
	if !found {
		t.Fatal("seed agent developer missing")
	}
	agent.ExecutorID = "mock"
	agent.Verification = []domain.VerificationGate{{ID: "check", Name: "检查", Mode: "command", Commands: []string{"echo gate-ran-on-runner"}, Required: true, ExecutionScope: domain.ExecutionScopeRun}}
	if err := s.PutAgent(agent); err != nil {
		t.Fatal(err)
	}
	daemon, err := New(Config{
		ServerURL:            server.URL,
		BootstrapToken:       "boot-secret",
		StatePath:            filepath.Join(t.TempDir(), "state.json"),
		Name:                 "gate-runner",
		MaxConcurrency:       1,
		HeartbeatInterval:    time.Second,
		LongPollTimeout:      2 * time.Second,
		DefaultWorkspaceRoot: filepath.Join(t.TempDir(), "workspace"),
	})
	if err != nil {
		t.Fatal(err)
	}
	workspaceRoot := daemon.cfg.DefaultWorkspaceRoot
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go func() { _ = daemon.Run(ctx) }()
	waitFor(t, 5*time.Second, "runner registration", func() bool {
		runners := s.Runners()
		return len(runners) == 1 && runners[0].Status == domain.RunnerStatusOnline
	})
	runner := s.Runners()[0]
	runner.OwnerUserID = 42
	if err := s.PutRunner(runner); err != nil {
		t.Fatal(err)
	}
	if err := s.SetDefaultRunner(42, runner.ID, time.Now().UTC()); err != nil {
		t.Fatal(err)
	}
	putMockDeveloperPlaybook(t, s)
	wf := domain.WorkflowDefinition{
		ID: "runner-gate-e2e", Name: "Runner gate e2e", SandboxProfileID: "local",
		Nodes: []domain.WorkflowNode{{
			ID: "work", Name: "Work", Type: "agent", AgentID: "developer", ExecutorID: "mock",
			MaxAttempts: 1,
		}},
	}
	if err := s.PutWorkflow(wf); err != nil {
		t.Fatal(err)
	}
	run, err := e.StartWithExecutionContext(wf.ID, "gate routing input", domain.RepositoryContext{}, domain.TelemetryContext{ResponsibleUserID: 42})
	if err != nil {
		t.Fatal(err)
	}
	waitFor(t, 30*time.Second, "run completion", func() bool {
		current, _ := s.Run(run.ID)
		return current.Status == "succeeded"
	})
	waitFor(t, 10*time.Second, "runner workspace immediate cleanup", func() bool {
		_, statErr := os.Stat(filepath.Join(workspaceRoot, "runs", run.ID))
		current, ok := s.Run(run.ID)
		return ok && os.IsNotExist(statErr) && !current.WorkspaceCleanedAt.IsZero()
	})
	// Both the CLI invocation and the gate command ran on the runner.
	allTasks := s.RunnerTasks()
	tasks := make([]domain.RunnerTask, 0, len(allTasks))
	for _, task := range allTasks {
		if task.Action == "" {
			tasks = append(tasks, task)
		}
	}
	if len(tasks) != 2 || len(allTasks) != 3 {
		t.Fatalf("expected CLI + gate + cleanup tasks, got %d: %+v", len(allTasks), allTasks)
	}
	providers := map[string]bool{}
	for _, task := range tasks {
		providers[task.Provider] = true
		if task.Status != domain.RunnerTaskSuccess {
			t.Fatalf("task %s (%s) status = %s", task.ID, task.Provider, task.Status)
		}
	}
	if !providers["mock"] || !providers["terminal"] {
		t.Fatalf("expected mock + terminal tasks, got %v", providers)
	}
	// The gate command output came back from the runner machine.
	for _, task := range tasks {
		if task.Provider == "terminal" && !strings.Contains(task.Result, "gate-ran-on-runner") {
			t.Fatalf("gate task result missing command output: %q", task.Result)
		}
	}
	cancel()
}

func TestProviderArgvTemplates(t *testing.T) {
	cases := []struct {
		provider string
		binary   string
		args     []string
		wantHead []string
		wantTail string
	}{
		{"claude-code", "/bin/claude", nil, []string{"/bin/claude", "-p", "--output-format", "text"}, "do it"},
		{"codex", "/bin/codex", nil, []string{"/bin/codex", "exec", "--skip-git-repo-check"}, "do it"},
	}
	for _, tc := range cases {
		argv, err := providerArgv(tc.provider, tc.binary, tc.wantTail, tc.args)
		if err != nil {
			t.Fatalf("%s: %v", tc.provider, err)
		}
		if len(argv) < len(tc.wantHead)+1 {
			t.Fatalf("%s: argv too short: %v", tc.provider, argv)
		}
		for i, want := range tc.wantHead {
			if argv[i] != want {
				t.Fatalf("%s: argv[%d] = %q, want %q (full: %v)", tc.provider, i, argv[i], want, argv)
			}
		}
		if argv[len(argv)-1] != tc.wantTail {
			t.Fatalf("%s: prompt must be last, got %q", tc.provider, argv[len(argv)-1])
		}
	}
	if _, err := providerArgv("unknown", "/bin/x", "p", nil); err == nil {
		t.Fatal("unknown provider accepted")
	}
}

func TestControlFallbackIsFasterThanLeaseRenew(t *testing.T) {
	if controlFallbackInterval > time.Second {
		t.Fatalf("control fallback %s is too slow for pause/cancel", controlFallbackInterval)
	}
	if controlFallbackInterval >= leaseRenewInterval {
		t.Fatalf("control fallback %s must be faster than lease renew %s", controlFallbackInterval, leaseRenewInterval)
	}
}

func TestCloneOutputRedactsToken(t *testing.T) {
	token := "tok-secret-123"
	got := cloneOutput([]byte(strings.Repeat("x", 280)+" https://private:"+token+"@git.example.com/a.git"), token)
	if strings.Contains(got, token) {
		t.Fatalf("clone error leaked the token: %s", got)
	}
	if !strings.Contains(got, "***") {
		t.Fatalf("clone error was not redacted: %s", got)
	}
	if len(got) > 300 {
		t.Fatalf("clone error length = %d, want <= 300", len(got))
	}
}

func TestCleanupDockerWorkspaceSkipsVanishedContainer(t *testing.T) {
	workspace := t.TempDir()
	logPath := filepath.Join(t.TempDir(), "rm.log")
	installFakeDocker(t, `#!/bin/sh
last=""
for a in "$@"; do last="$a"; done
case "$1" in
ps) printf '%s\n' gone keep ;;
inspect)
  case "$last" in
  gone) echo "Error: No such object: gone" >&2; exit 1 ;;
  keep)
    case "$3" in
    *Config.Image*) echo ubuntu ;;
    *) printf '%s\n' "$AF_TEST_WORKSPACE/node" ;;
    esac
    ;;
  *) echo "unexpected $last" >&2; exit 1 ;;
  esac
  ;;
rm) printf '%s\n' "$last" >> "$AF_TEST_RM_LOG" ;;
*) echo "unexpected $1" >&2; exit 1 ;;
esac
`)
	t.Setenv("AF_TEST_WORKSPACE", workspace)
	t.Setenv("AF_TEST_RM_LOG", logPath)
	if err := cleanupDockerWorkspace(context.Background(), workspace); err != nil {
		t.Fatal(err)
	}
	removed, err := os.ReadFile(logPath)
	if err != nil {
		t.Fatal(err)
	}
	if strings.TrimSpace(string(removed)) != "keep" {
		t.Fatalf("removed = %q, want keep", removed)
	}
}

func TestCleanupDockerWorkspaceReturnsRealInspectFailure(t *testing.T) {
	installFakeDocker(t, `#!/bin/sh
case "$1" in
ps) echo bad ;;
*) echo "permission denied" >&2; exit 1 ;;
esac
`)
	err := cleanupDockerWorkspace(context.Background(), t.TempDir())
	if err == nil || !strings.Contains(err.Error(), "permission denied") {
		t.Fatalf("err = %v, want permission denied", err)
	}
}

func installFakeDocker(t *testing.T, script string) {
	t.Helper()
	bin := t.TempDir()
	path := filepath.Join(bin, "docker")
	if err := os.WriteFile(path, []byte(script), 0o755); err != nil {
		t.Fatal(err)
	}
	t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
}