Skip to content

Self-hosted workflows for coding-agent delivery. Orchestrate CodeBuddy, Codex and Claude Code with scripts and human approval gates. Includes Local workflows for coding assistants.

14.5 KB
Raw Blame
package httpapi

import (
	"context"
	"encoding/json"
	"errors"
	"net/http"
	"net/http/httptest"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"agentflow/internal/domain"
	"agentflow/internal/engine"
	"agentflow/internal/repo"
	"agentflow/internal/runtime"

	"github.com/gin-gonic/gin"
)

func quietRunnerHub() *runnerControlHub {
	hub := newRunnerControlHub()
	hub.startupGrace = 0
	hub.reconnectWait = 2 * time.Second
	hub.handoverDrain = time.Second
	return hub
}

func testRunnerConn(runnerID string) *runnerControlConn {
	return &runnerControlConn{runnerID: runnerID, send: make(chan runnerControlMessage, 4), done: make(chan struct{})}
}

// registerTestConn mirrors register without a real websocket.
func registerTestConn(hub *runnerControlHub, conn *runnerControlConn) {
	hub.mu.Lock()
	old := hub.conns[conn.runnerID]
	hub.conns[conn.runnerID] = conn
	close(hub.registered)
	hub.registered = make(chan struct{})
	hub.mu.Unlock()
	if old != nil {
		old.close()
	}
}

// answerOnce serves the next workspace request on conn.
func answerOnce(t *testing.T, hub *runnerControlHub, conn *runnerControlConn, response domain.RunnerWorkspaceResponse) domain.RunnerWorkspaceRequest {
	t.Helper()
	select {
	case message := <-conn.send:
		if message.Type != "workspace.request" || message.WorkspaceRequest == nil {
			t.Fatalf("unexpected message %+v", message)
		}
		hub.resolveWorkspace(message.RequestID, response)
		return *message.WorkspaceRequest
	case <-time.After(3 * time.Second):
		t.Fatal("runner never received the request")
	}
	return domain.RunnerWorkspaceRequest{}
}

type workspaceResult struct {
	response domain.RunnerWorkspaceResponse
	err      error
}

func requestAsync(hub *runnerControlHub, runnerID string, request domain.RunnerWorkspaceRequest) <-chan workspaceResult {
	done := make(chan workspaceResult, 1)
	go func() {
		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
		defer cancel()
		response, err := hub.requestWorkspace(ctx, runnerID, request)
		done <- workspaceResult{response, err}
	}()
	return done
}

// A runner that is simply offline fails immediately instead of stalling the
// caller for the reconnect window.
func TestRunnerWorkspaceOfflineRunnerFailsFast(t *testing.T) {
	hub := quietRunnerHub()
	started := time.Now()
	_, err := hub.requestWorkspace(context.Background(), "runner-a", domain.RunnerWorkspaceRequest{Operation: domain.RunnerWorkspaceOperationTree})
	if !errors.Is(err, errRunnerWorkspaceUnavailable) || time.Since(started) > 500*time.Millisecond {
		t.Fatalf("err=%v after %s", err, time.Since(started))
	}
}

// A socket that just dropped (deploy, network blip) is expected back within
// seconds; the request waits for it instead of failing the tool call.
func TestRunnerWorkspaceWaitsForReconnect(t *testing.T) {
	hub := quietRunnerHub()
	first := testRunnerConn("runner-a")
	registerTestConn(hub, first)
	hub.unregister(first)

	result := requestAsync(hub, "runner-a", domain.RunnerWorkspaceRequest{RunID: "run-a", Operation: domain.RunnerWorkspaceOperationTree})
	time.Sleep(100 * time.Millisecond)
	second := testRunnerConn("runner-a")
	registerTestConn(hub, second)
	answerOnce(t, hub, second, domain.RunnerWorkspaceResponse{Status: 200, Payload: json.RawMessage(`{"ok":true}`)})
	got := <-result
	if got.err != nil || string(got.response.Payload) != `{"ok":true}` {
		t.Fatalf("request across reconnect: %+v err=%v", got.response, got.err)
	}
}

// Once a request reached the socket it may already be running on the runner;
// it must fail rather than run twice (writes, shell commands).
func TestRunnerWorkspaceNeverReplaysDeliveredRequest(t *testing.T) {
	hub := quietRunnerHub()
	first := testRunnerConn("runner-a")
	registerTestConn(hub, first)
	result := requestAsync(hub, "runner-a", domain.RunnerWorkspaceRequest{RunID: "run-a", Operation: domain.RunnerWorkspaceOperationWrite})
	select {
	case <-first.send:
	case <-time.After(3 * time.Second):
		t.Fatal("request not delivered")
	}
	second := testRunnerConn("runner-a")
	registerTestConn(hub, second) // closes first
	got := <-result
	if !errors.Is(got.err, errRunnerWorkspaceUnavailable) {
		t.Fatalf("delivered request must fail on socket loss, got %v", got.err)
	}
	select {
	case message := <-second.send:
		t.Fatalf("delivered request replayed: %+v", message)
	case <-time.After(100 * time.Millisecond):
	}
}

// After a listener handover the old process keeps finishing conversations;
// their workspace calls reach the runner through the successor, including
// the gap before the runner reconnected there.
func TestRunnerWorkspaceRelaysThroughSuccessorAfterHandover(t *testing.T) {
	predecessor, successor := quietRunnerHub(), quietRunnerHub()
	predecessor.successor = successor.requestRelayed
	old := testRunnerConn("runner-a")
	registerTestConn(predecessor, old)

	predecessor.handOver()
	waitClosed(t, old)
	result := requestAsync(predecessor, "runner-a", domain.RunnerWorkspaceRequest{RunID: "run-a", Operation: domain.RunnerWorkspaceOperationTerminal})
	time.Sleep(100 * time.Millisecond)
	moved := testRunnerConn("runner-a")
	registerTestConn(successor, moved)
	request := answerOnce(t, successor, moved, domain.RunnerWorkspaceResponse{Status: 200, Payload: json.RawMessage(`"done"`)})
	if request.RunID != "run-a" || request.Operation != domain.RunnerWorkspaceOperationTerminal {
		t.Fatalf("relayed request changed: %+v", request)
	}
	got := <-result
	if got.err != nil || string(got.response.Payload) != `"done"` {
		t.Fatalf("relayed request: %+v err=%v", got.response, got.err)
	}
}

// A relayed request is served locally only: two processes must never bounce
// a request between each other.
func TestRunnerWorkspaceRelayedRequestIsNotRelayedAgain(t *testing.T) {
	hub := quietRunnerHub()
	hub.reconnectWait = 50 * time.Millisecond
	hub.successor = func(context.Context, string, domain.RunnerWorkspaceRequest) (domain.RunnerWorkspaceResponse, error) {
		t.Error("relayed request was relayed again")
		return domain.RunnerWorkspaceResponse{}, nil
	}
	hub.handOver()
	_, err := hub.requestRelayed(context.Background(), "runner-a", domain.RunnerWorkspaceRequest{Operation: domain.RunnerWorkspaceOperationTree})
	if !errors.Is(err, errRunnerWorkspaceUnavailable) {
		t.Fatalf("err=%v", err)
	}
}

func relayTestEngine(t *testing.T) *engine.Engine {
	t.Helper()
	s, err := repo.OpenTest()
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { s.Close() })
	return engine.New(s, runtime.NewManager(filepath.Join(t.TempDir(), "workspaces")))
}

func TestRunnerWorkspaceRelayEndpointRequiresServiceToken(t *testing.T) {
	gin.SetMode(gin.TestMode)
	hub := quietRunnerHub()
	e := relayTestEngine(t)
	router := gin.New()
	router.POST(runnerWorkspaceRelayPath, handleRunnerWorkspaceRelay(e, hub))
	post := func(token string) int {
		req := httptest.NewRequest(http.MethodPost, runnerWorkspaceRelayPath, strings.NewReader(`{"runnerId":"runner-a","request":{"operation":"tree"}}`))
		if token != "" {
			req.Header.Set(engine.ControlTokenHeader, token)
		}
		rec := httptest.NewRecorder()
		router.ServeHTTP(rec, req)
		return rec.Code
	}
	// No configured token: the endpoint is closed, even for an empty header.
	if code := post(""); code != http.StatusUnauthorized {
		t.Fatalf("unconfigured token must not authorize, got %d", code)
	}
	e.ConfigureControl(engine.ControlConfig{Token: "service-secret"})
	if code := post("wrong"); code != http.StatusUnauthorized {
		t.Fatalf("wrong token: %d", code)
	}
	if code := post(""); code != http.StatusUnauthorized {
		t.Fatalf("missing token: %d", code)
	}
}

// End to end over HTTP: the predecessor relays to the successor's endpoint
// with the configured service token.
func TestRunnerWorkspaceRelayOverHTTP(t *testing.T) {
	gin.SetMode(gin.TestMode)
	successor := quietRunnerHub()
	successorEngine := relayTestEngine(t)
	successorEngine.ConfigureControl(engine.ControlConfig{Token: "service-secret"})
	router := gin.New()
	router.POST(runnerWorkspaceRelayPath, handleRunnerWorkspaceRelay(successorEngine, successor))
	server := httptest.NewServer(router)
	defer server.Close()

	predecessorEngine := relayTestEngine(t)
	predecessorEngine.ConfigureControl(engine.ControlConfig{Token: "service-secret", APIURL: server.URL})
	predecessor := quietRunnerHub()
	predecessor.successor = relayRunnerWorkspaceToSuccessor(predecessorEngine)
	predecessor.handOver()

	moved := testRunnerConn("runner-a")
	registerTestConn(successor, moved)
	result := requestAsync(predecessor, "runner-a", domain.RunnerWorkspaceRequest{RunID: "run-a", Operation: domain.RunnerWorkspaceOperationFile, Path: "README.md"})
	request := answerOnce(t, successor, moved, domain.RunnerWorkspaceResponse{Status: 200, Payload: json.RawMessage(`{"content":"hi"}`)})
	if request.Path != "README.md" {
		t.Fatalf("relayed request: %+v", request)
	}
	got := <-result
	if got.err != nil || got.response.Status != 200 || string(got.response.Payload) != `{"content":"hi"}` {
		t.Fatalf("relay over HTTP: %+v err=%v", got.response, got.err)
	}

	// A successor that rejects the credential surfaces as unavailable, not as
	// a silent empty workspace.
	predecessorEngine.ConfigureControl(engine.ControlConfig{Token: "stale", APIURL: server.URL})
	_, err := predecessor.requestWorkspace(context.Background(), "runner-a", domain.RunnerWorkspaceRequest{Operation: domain.RunnerWorkspaceOperationTree})
	if !errors.Is(err, errRunnerWorkspaceUnavailable) || !strings.Contains(err.Error(), "401") {
		t.Fatalf("rejected relay: %v", err)
	}
}

func TestRunnerWorkspaceRPCIsTransient(t *testing.T) {
	hub := newRunnerControlHub()
	conn := &runnerControlConn{runnerID: "runner-a", send: make(chan runnerControlMessage, 1), done: make(chan struct{})}
	hub.conns[conn.runnerID] = conn

	type result struct {
		response domain.RunnerWorkspaceResponse
		err      error
	}
	completed := make(chan result, 1)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	go func() {
		response, err := hub.requestWorkspace(ctx, conn.runnerID, domain.RunnerWorkspaceRequest{RunID: "run-a", Operation: domain.RunnerWorkspaceOperationTree})
		completed <- result{response: response, err: err}
	}()

	message := <-conn.send
	if message.Type != "workspace.request" || message.RequestID == "" || message.WorkspaceRequest == nil {
		t.Fatalf("unexpected workspace request: %+v", message)
	}
	payload := json.RawMessage(`{"entries":[],"truncated":false}`)
	hub.resolveWorkspace(message.RequestID, domain.RunnerWorkspaceResponse{Payload: payload, Status: 200})
	got := <-completed
	if got.err != nil || string(got.response.Payload) != string(payload) {
		t.Fatalf("unexpected response: %+v err=%v", got.response, got.err)
	}
	if len(hub.pending) != 0 {
		t.Fatalf("completed transient requests retained in memory: %d", len(hub.pending))
	}
}

func waitClosed(t *testing.T, conn *runnerControlConn) {
	t.Helper()
	select {
	case <-conn.done:
	case <-time.After(3 * time.Second):
		t.Fatal("handover must drop runner sockets so runners reconnect to the successor")
	}
}

// A request already running on the runner when the listener is handed over
// (a command, a write) finishes on the old socket instead of failing; new
// requests already go to the successor, and the socket closes once idle.
func TestRunnerWorkspaceHandoverDrainsInFlightRequests(t *testing.T) {
	predecessor, successor := quietRunnerHub(), quietRunnerHub()
	predecessor.handoverDrain = 3 * time.Second
	predecessor.successor = successor.requestRelayed
	old := testRunnerConn("runner-a")
	registerTestConn(predecessor, old)

	inFlight := requestAsync(predecessor, "runner-a", domain.RunnerWorkspaceRequest{RunID: "run-a", Operation: domain.RunnerWorkspaceOperationTerminal})
	var delivered runnerControlMessage
	select {
	case delivered = <-old.send:
	case <-time.After(3 * time.Second):
		t.Fatal("request not delivered")
	}
	predecessor.handOver()
	later := requestAsync(predecessor, "runner-a", domain.RunnerWorkspaceRequest{RunID: "run-a", Operation: domain.RunnerWorkspaceOperationTree})
	time.Sleep(100 * time.Millisecond)
	if old.closed() {
		t.Fatal("socket closed while a request was still running on it")
	}
	select {
	case message := <-old.send:
		t.Fatalf("a request after the handover used the draining socket: %+v", message)
	default:
	}
	predecessor.resolveWorkspace(delivered.RequestID, domain.RunnerWorkspaceResponse{Status: 200, Payload: json.RawMessage(`"step-4"`)})
	if got := <-inFlight; got.err != nil || string(got.response.Payload) != `"step-4"` {
		t.Fatalf("in-flight request across handover: %+v err=%v", got.response, got.err)
	}
	waitClosed(t, old)
	moved := testRunnerConn("runner-a")
	registerTestConn(successor, moved)
	answerOnce(t, successor, moved, domain.RunnerWorkspaceResponse{Status: 200, Payload: json.RawMessage(`[]`)})
	if got := <-later; got.err != nil || string(got.response.Payload) != `[]` {
		t.Fatalf("request after handover: %+v err=%v", got.response, got.err)
	}
}

// A runner that never comes back cannot hold the old process's sockets open
// forever.
func TestRunnerWorkspaceHandoverDrainIsBounded(t *testing.T) {
	hub := quietRunnerHub()
	hub.handoverDrain = 200 * time.Millisecond
	conn := testRunnerConn("runner-a")
	registerTestConn(hub, conn)
	stuck := requestAsync(hub, "runner-a", domain.RunnerWorkspaceRequest{Operation: domain.RunnerWorkspaceOperationTerminal})
	<-conn.send
	hub.handOver()
	waitClosed(t, conn)
	if got := <-stuck; !errors.Is(got.err, errRunnerWorkspaceUnavailable) {
		t.Fatalf("stuck request: %v", got.err)
	}
}

func TestRunnerCheckpointCancellationReachesExecutorOnlyForOneShotRequests(t *testing.T) {
	for _, operation := range []string{domain.RunnerWorkspaceOperationCheckpointExport, "conversation_process"} {
		t.Run(operation, func(t *testing.T) {
			hub := quietRunnerHub()
			conn := testRunnerConn("runner-cancel")
			registerTestConn(hub, conn)
			ctx, cancel := context.WithCancel(context.Background())
			done := make(chan error, 1)
			go func() {
				_, err := hub.requestWorkspace(ctx, conn.runnerID, domain.RunnerWorkspaceRequest{Operation: operation})
				done <- err
			}()
			request := <-conn.send
			cancel()
			select {
			case err := <-done:
				if !errors.Is(err, context.Canceled) {
					t.Fatal(err)
				}
			case <-time.After(time.Second):
				t.Fatal("request cancellation blocked")
			}
			if operation == "conversation_process" {
				select {
				case unexpected := <-conn.send:
					t.Fatalf("persistent CLI lifecycle cancelled by RPC: %+v", unexpected)
				default:
				}
			} else {
				select {
				case stop := <-conn.send:
					if stop.Type != "workspace.cancel" || stop.RequestID != request.RequestID {
						t.Fatalf("wrong cancellation: %+v", stop)
					}
				case <-time.After(time.Second):
					t.Fatal("export cancellation not sent")
				}
			}
		})
	}
}