Self-hosted workflows for coding-agent delivery. Orchestrate CodeBuddy, Codex and Claude Code with scripts and human approval gates. Includes Local workflows for coding assistants.
JavaScript agent-orchestrationagentic-workflowai-agentsai-codingclaude-codecode-reviewcodebuddycodexcoding-agentscursordeveloper-toolshuman-on-the-loopllmmcpmulti-agentsdlcself-hostedsopworkflow-automationworkflow-engine
package httpapi
import (
"github.com/gin-gonic/gin"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"agentflow/internal/domain"
"agentflow/internal/engine"
"agentflow/internal/repo"
"agentflow/internal/runtime"
"agentflow/internal/tracker"
"github.com/go-sql-driver/mysql"
)
type workspaceTurnEngine struct {
begin, prepare, renewals int
ended []string
renewErr, saveErr error
}
func (e *workspaceTurnEngine) BeginConversationWorkspaceUse(id string) (string, error) {
e.begin++
return "token-" + id, nil
}
func (e *workspaceTurnEngine) MaterializeConversationWorkspace(context.Context, string) error {
e.prepare++
return nil
}
func (e *workspaceTurnEngine) EnsureConversationWorkspace(_ context.Context, id, token string) error {
if token != "token-"+id {
return errors.New("wrong workspace restore lease")
}
return nil
}
func (e *workspaceTurnEngine) RenewConversationWorkspaceUse(string, string) error {
e.renewals++
return e.renewErr
}
func (e *workspaceTurnEngine) EndConversationWorkspaceUse(ctx context.Context, id, token string) error {
if err := ctx.Err(); err != nil {
return fmt.Errorf("checkpoint inherited cancellation: %w", err)
}
if token != "token-"+id {
return errors.New("wrong workspace lease")
}
e.ended = append(e.ended, id)
return e.saveErr
}
func (e *workspaceTurnEngine) EndConversationWorkspaceUseAsync(ctx context.Context, id, token string) (<-chan error, error) {
if err := e.EndConversationWorkspaceUse(context.WithoutCancel(ctx), id, token); err != nil {
return nil, err
}
done := make(chan error, 1)
done <- nil
close(done)
return done, nil
}
func TestAssistantWorkspaceTurnReusesAndRenewsLease(t *testing.T) {
en := &workspaceTurnEngine{}
turn := newAssistantWorkspaceTurn(en, nil)
t.Cleanup(func() { _ = turn.close(context.Background()) })
var calls sync.WaitGroup
for range 24 {
calls.Add(1)
go func() {
defer calls.Done()
release := turn.lockOperation()
defer release()
ctx, err := turn.lease(context.Background(), "first")
if err != nil || engine.ConversationWorkspaceToken(ctx, "first") != "token-first" {
t.Errorf("tool did not reuse the turn lease: %v", err)
}
}()
}
calls.Wait()
if err := turn.renew(); err != nil {
t.Fatal(err)
}
if en.begin != 1 || en.prepare != 1 || en.renewals != 1 || len(en.ended) != 0 {
t.Fatalf("tools took new leases or saved checkpoints: %+v", en)
}
if engine.ConversationWorkspaceToken(turn.withLease(context.Background(), "other"), "other") != "" {
t.Fatal("run control borrowed another workspace's lease")
}
if err := turn.checkpoint(context.Background()); err != nil {
t.Fatal(err)
}
if _, err := turn.lease(context.Background(), "second"); err != nil {
t.Fatal(err)
}
cancelled, cancel := context.WithCancel(context.Background())
cancel()
if err := turn.close(cancelled); err != nil {
t.Fatal(err)
}
if err := turn.close(cancelled); err != nil || strings.Join(en.ended, ",") != "first,second" {
t.Fatalf("switch/end must save once each, including cancellation: %+v %v", en, err)
}
if _, err := turn.lease(context.Background(), "second"); err == nil {
t.Fatal("closed turn accepted a late tool")
}
}
func TestAssistantWorkspaceTurnLosesLeaseAndReportsSaveFailure(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
en := &workspaceTurnEngine{renewErr: repo.ErrWorkspaceInUse, saveErr: errors.New("storage unavailable")}
turn := newAssistantWorkspaceTurn(en, cancel)
if _, err := turn.lease(ctx, "first"); err != nil {
t.Fatal(err)
}
if err := turn.renew(); err == nil || ctx.Err() == nil {
t.Fatal("lease loss did not stop model execution")
}
if _, err := turn.lease(context.Background(), "first"); err == nil {
t.Fatal("tools continued after losing the workspace lease")
}
if err := turn.close(ctx); err == nil || !strings.Contains(err.Error(), "租约续期失败") || !strings.Contains(err.Error(), "storage unavailable") {
t.Fatalf("lease/save failures were lost: %v", err)
}
}
// A storage hiccup must not abort a long answer while the 15-minute lease is
// still valid; a lease that is about to lapse still stops the turn.
func TestAssistantWorkspaceTurnRetriesTransientRenewFailure(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
en := &workspaceTurnEngine{renewErr: errors.New("database is locked")}
turn := newAssistantWorkspaceTurn(en, cancel)
t.Cleanup(func() { _ = turn.close(context.Background()) })
if _, err := turn.lease(ctx, "first"); err != nil {
t.Fatal(err)
}
for range 3 {
if err := turn.renew(); err != nil || ctx.Err() != nil {
t.Fatalf("a transient renewal error stopped the turn: %v", err)
}
}
if _, err := turn.lease(ctx, "first"); err != nil {
t.Fatalf("tools stopped after a transient renewal error: %v", err)
}
en.renewErr = nil
turn.mu.Lock()
turn.renewedAt = time.Now().Add(-time.Hour)
turn.mu.Unlock()
if err := turn.renew(); err != nil {
t.Fatal(err)
}
turn.mu.Lock()
renewed := time.Since(turn.renewedAt) < time.Minute
turn.mu.Unlock()
if !renewed {
t.Fatal("a successful renewal did not restart the retry window")
}
en.renewErr = errors.New("database is locked")
turn.mu.Lock()
turn.renewedAt = time.Now().Add(-assistantWorkspaceRenewGrace)
turn.mu.Unlock()
if err := turn.renew(); err == nil || ctx.Err() == nil {
t.Fatal("a lease about to lapse did not stop the turn")
}
if _, err := turn.lease(context.Background(), "first"); err == nil {
t.Fatal("tools continued after the lease could not be kept")
}
}
// turnSandbox is the environment of a workspaceTurnFixture workspace.
func turnSandbox(runID string) string {
if strings.HasSuffix(runID, "-second") {
return runID + "-sandbox"
}
return strings.TrimSuffix(runID, "-first") + "-sandbox"
}
func workspaceTurnFixture(t *testing.T) (*assistantHandlers, domain.User, *AgentToolSession, *agentToolGateway, []string) {
t.Helper()
t.Setenv("AGENTFLOW_ASSISTANT_SESSION_DIR", t.TempDir())
var st *repo.Store
var err error
if dsn := os.Getenv("AGENTFLOW_WORKSPACE_TURN_TEST_MYSQL_DSN"); dsn != "" {
cfg, parseErr := mysql.ParseDSN(dsn)
if parseErr != nil || cfg.Net != "tcp" || !strings.HasPrefix(cfg.Addr, "127.0.0.1:") || !strings.HasPrefix(cfg.DBName, "agentflow_workspace_turn_test") {
t.Fatal("workspace turn tests require a disposable loopback MySQL database")
}
st, err = repo.Open(dsn)
} else {
st, err = repo.OpenTest()
}
if err != nil {
t.Fatal(err)
}
t.Cleanup(st.Close)
id := fmt.Sprintf("turn-%d", time.Now().UnixNano())
if err := st.PutUser(domain.User{LoginName: id, CodeBuddyAPIKey: "test-only"}); err != nil {
t.Fatal(err)
}
user, _ := st.UserByLoginName(id)
if err := st.PutProject(domain.Project{ID: id, Name: "Workspace turn test", OwnerID: user.ID}); err != nil {
t.Fatal(err)
}
root := t.TempDir()
en := engine.New(st, runtime.NewManager(root))
h := &assistantHandlers{deps: &deps{repo: st, engine: en}}
profile := domain.SandboxProfile{ID: id + "-sandbox", ProjectID: id, Name: "Workspace turn test", Driver: "local"}
if err := st.PutSandbox(profile); err != nil {
t.Fatal(err)
}
// Each workspace runs in its own environment: the conversation switches
// between them by environment.
if err := st.PutSandbox(domain.SandboxProfile{ID: id + "-second-sandbox", ProjectID: id, Name: "Workspace turn test 2", Driver: "local"}); err != nil {
t.Fatal(err)
}
ids := []string{id + "-first", id + "-second"}
for _, rid := range ids {
run := domain.Run{ID: rid, Kind: domain.RunKindWorkspace, Status: "waiting", WorkflowSnapshot: domain.WorkflowDefinition{SandboxProfileID: turnSandbox(rid)}, ExecutionContext: domain.TelemetryContext{ProjectID: id}, Nodes: map[string]*domain.NodeRun{}}
if err := st.PutRun(run); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(filepath.Join(root, rid), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(root, rid, "note.txt"), []byte("before"), 0o600); err != nil {
t.Fatal(err)
}
}
if err := st.PutAssistantSession(user.ID, domain.AssistantSession{ID: id, ProjectID: id, TitleEdited: true, WorkspaceRunID: ids[0], OwnWorkspaceRunIDs: ids}); err != nil {
t.Fatal(err)
}
for _, rid := range []string{ids[1], ids[0]} {
if err := st.SetAssistantSessionWorkspaceRun(user.ID, id, id, rid, true); err != nil {
t.Fatal(err)
}
}
s := &AgentToolSession{Mode: AgentToolAssistant, ProjectID: id, UserID: user.ID, User: user, ChatSessionID: id, ChatOwnerID: user.ID, workspaceRunID: ids[0], workspaceTurn: newAssistantWorkspaceTurn(en, nil)}
t.Cleanup(func() {
_ = s.workspaceTurn.close(context.Background())
// Wait only for this fixture's saves. Global inflightTasks also contains
// unrelated title summaries, which can legitimately take much longer.
deadline := time.Now().Add(5 * time.Second)
for _, runID := range ids {
for {
meta, ok := st.WorkspaceCheckpoint(runID)
if !ok || meta.LeaseToken == "" {
break
}
if time.Now().After(deadline) {
t.Error("background save did not finish", runID)
return
}
time.Sleep(10 * time.Millisecond)
}
}
})
gateway := &agentToolGateway{deps: h.deps}
gateway.api = New(st, en, tracker.NewService(), "", "")
return h, user, s, gateway, ids
}
// workspaceTestEdit edits a file the way a workspace CLI does: within the
// turn's lease on the bound workspace, straight on disk.
func workspaceTestEdit(ctx context.Context, gateway *agentToolGateway, s *AgentToolSession, path, content string) error {
if _, err := gateway.callMainBotWorkspace(ctx, s, "workbench_workspace_tree", map[string]any{}); err != nil {
return err
}
run, _ := gateway.repo.RunState(s.workspaceRun())
root, reason := gateway.engine.LocalRunWorkspace(run)
if reason != "" {
return errors.New(reason)
}
return os.WriteFile(filepath.Join(root, path), []byte(content), 0o600)
}
func workspaceTurnCheckpointText(t *testing.T, h *assistantHandlers, id string) string {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for {
meta, exists := h.repo.WorkspaceCheckpoint(id)
if exists && meta.LeaseToken == "" && meta.SavedAt != nil {
break
}
if time.Now().After(deadline) {
t.Fatal("background checkpoint did not finish", id)
}
time.Sleep(10 * time.Millisecond)
}
archive, err := h.repo.WorkspaceCheckpointArchive(id)
if err != nil {
t.Fatal(err)
}
root := filepath.Join(t.TempDir(), "restored")
if err := runtime.RestoreWorkspaceCheckpoint(root, archive); err != nil {
t.Fatal(err)
}
data, err := os.ReadFile(filepath.Join(root, "note.txt"))
if err != nil {
t.Fatal(err)
}
return string(data)
}
func TestAssistantWorkspaceTurnCheckpointsOnlyOnSwitchAndEnd(t *testing.T) {
h, _, s, gateway, ids := workspaceTurnFixture(t)
call := func(name string, args map[string]any) {
t.Helper()
if _, err := gateway.callMainBotWorkspace(context.Background(), s, "workbench_workspace_"+name, args); err != nil {
t.Fatal(err)
}
}
for _, text := range []string{"edit-one", "edit-two", "final-first"} {
call("tree", map[string]any{})
if err := workspaceTestEdit(context.Background(), gateway, s, "note.txt", text); err != nil {
t.Fatal(err)
}
meta, _ := h.repo.WorkspaceCheckpoint(ids[0])
if meta.LeaseToken == "" || meta.SavedAt != nil {
t.Fatalf("tool released its lease or took a checkpoint: %+v", meta)
}
}
// A switch waits for operations already using the old workspace.
release := s.workspaceTurn.lockOperation()
switched := make(chan error, 1)
go func() {
_, err := gateway.bindConversationWorkspace(context.Background(), s, workspaceOpenChoice{SandboxID: turnSandbox(ids[1])})
switched <- err
}()
select {
case err := <-switched:
t.Fatalf("workspace switched while a tool still used it: %v", err)
case <-time.After(30 * time.Millisecond):
}
release()
if err := <-switched; err != nil {
t.Fatal(err)
}
if got := workspaceTurnCheckpointText(t, h, ids[0]); got != "final-first" {
t.Fatalf("switch lost the latest edits: %q", got)
}
oldScope := context.WithValue(context.Background(), mainBotWorkspaceKey{}, ids[0])
if _, err := gateway.callMainBotWorkspace(oldScope, s, "workbench_workspace_tree", nil); err == nil {
t.Fatal("an open racing another switch read the wrong workspace")
}
if err := workspaceTestEdit(context.Background(), gateway, s, "note.txt", "final-second"); err != nil {
t.Fatal(err)
}
meta, _ := h.repo.WorkspaceCheckpoint(ids[1])
if meta.SavedAt != nil {
t.Fatal("tool in the new workspace took a premature checkpoint")
}
if _, err := gateway.bindConversationWorkspace(context.Background(), s, workspaceOpenChoice{SandboxID: turnSandbox(ids[1])}); err != nil {
t.Fatal(err)
}
meta, _ = h.repo.WorkspaceCheckpoint(ids[1])
if meta.SavedAt != nil {
t.Fatal("reopening the same workspace took a checkpoint")
}
if err := s.workspaceTurn.close(context.Background()); err != nil {
t.Fatal(err)
}
if got := workspaceTurnCheckpointText(t, h, ids[1]); got != "final-second" {
t.Fatalf("turn end lost the latest edits: %q", got)
}
meta, _ = h.repo.WorkspaceCheckpoint(ids[1])
if meta.LeaseToken != "" {
t.Fatal("turn end did not release its lease")
}
}
func TestAssistantWorkspaceTurnGenerationAlwaysCheckpoints(t *testing.T) {
for _, outcome := range []string{"completed", "failed", "cancelled", "panic"} {
t.Run(outcome, func(t *testing.T) {
h, user, fixtureSession, gateway, ids := workspaceTurnFixture(t)
h.generate = func(ctx context.Context, _ string, _ domain.User) (string, error) {
var session *AgentToolSession
agentToolSessions.mu.Lock()
for _, candidate := range agentToolSessions.sessions {
if candidate.ChatSessionID == fixtureSession.ChatSessionID {
session = candidate
}
}
agentToolSessions.mu.Unlock()
if session == nil || session.workspaceTurn == nil {
return "", errors.New("generation did not own its workspace lifetime")
}
if err := workspaceTestEdit(ctx, gateway, session, "note.txt", outcome); err != nil {
return "", err
}
meta, _ := h.repo.WorkspaceCheckpoint(ids[0])
if meta.SavedAt != nil {
return "", errors.New("tool saved before the turn ended")
}
switch outcome {
case "failed":
return "", errors.New("model failed")
case "cancelled":
g, _ := h.repo.AssistantGeneration(user.ID, fixtureSession.ChatSessionID)
if err := h.repo.CancelAssistantGeneration(user.ID, g.SessionID, g.ID); err != nil {
return "", err
}
<-ctx.Done()
return "", ctx.Err()
case "panic":
panic("model panic")
}
return "done", nil
}
body, _ := json.Marshal(assistantRequest{SessionID: fixtureSession.ChatSessionID, ProjectID: fixtureSession.ProjectID, Messages: []assistantMessage{{Role: "user", Content: "edit"}}})
c, rec := workbenchContext(user, "POST", "/api/assistant/generations", bytes.NewBuffer(body))
h.startGeneration(c)
if rec.Code != 202 {
t.Fatalf("generation start: %d %s", rec.Code, rec.Body)
}
deadline := time.Now().Add(10 * time.Second)
for {
g, ok := h.repo.AssistantGeneration(user.ID, fixtureSession.ChatSessionID)
if ok && g.Status != "running" {
want := outcome
if want == "panic" {
want = "failed"
}
if g.Status != want || g.WorkspaceWarning != "" {
t.Fatalf("generation outcome: %+v", g)
}
break
}
if time.Now().After(deadline) {
t.Fatal("generation did not finish")
}
time.Sleep(10 * time.Millisecond)
}
if got := workspaceTurnCheckpointText(t, h, ids[0]); got != outcome {
t.Fatalf("%s lost edits: %q", outcome, got)
}
})
}
}
type workspaceTurnCheckpointBarrier struct {
assistantWorkspaceLeaseEngine
entered chan struct{}
release chan struct{}
err error
}
func (e *workspaceTurnCheckpointBarrier) EndConversationWorkspaceUseAsync(ctx context.Context, id, token string) (<-chan error, error) {
if e.entered == nil {
// Simulate an immediate handoff failure; an actual background failure is
// reported only in workspace metadata, never in a completed generation.
return nil, errors.Join(e.assistantWorkspaceLeaseEngine.EndConversationWorkspaceUse(context.WithoutCancel(ctx), id, token), e.err)
}
result, err := e.assistantWorkspaceLeaseEngine.EndConversationWorkspaceUseAsync(ctx, id, token)
if err != nil {
return nil, err
}
done := make(chan error, 1)
go func() {
defer close(done)
close(e.entered)
<-e.release
done <- errors.Join(<-result, e.err)
}()
return done, nil
}
// A workspace that cannot be saved (too large, or slower than the save
// timeout) must not strand the turn: the switch proceeds and the old
// workspace keeps its files for the idle sweep, which saves before reclaiming.
func TestAssistantWorkspaceTurnSwitchesDespiteFailedCheckpoint(t *testing.T) {
h, _, s, gateway, ids := workspaceTurnFixture(t)
if _, err := gateway.callMainBotWorkspace(context.Background(), s, "workbench_workspace_tree", map[string]any{}); err != nil {
t.Fatal(err)
}
s.workspaceTurn.mu.Lock()
s.workspaceTurn.engine = &workspaceTurnCheckpointBarrier{assistantWorkspaceLeaseEngine: h.engine, err: errors.New("checkpoint exceeds size limit")}
s.workspaceTurn.mu.Unlock()
bound, err := gateway.bindConversationWorkspace(context.Background(), s, workspaceOpenChoice{SandboxID: turnSandbox(ids[1])})
if err != nil || bound.Previous != ids[0] || !strings.Contains(bound.CheckpointError, "exceeds size limit") {
t.Fatalf("a failed save blocked or hid the switch: %+v %v", bound, err)
}
conv, _ := h.repo.AssistantSession(s.ChatOwnerID, s.ChatSessionID)
if s.workspaceRun() != ids[1] || conv.WorkspaceRunID != ids[1] {
t.Fatal("the switch did not bind the new workspace")
}
meta, _ := h.repo.WorkspaceCheckpoint(ids[0])
if meta.LeaseToken != "" || !meta.Released() {
t.Fatalf("the old workspace was not handed to the idle sweep: %+v", meta)
}
s.workspaceTurn.mu.Lock()
s.workspaceTurn.engine = h.engine
_, pending := s.workspaceTurn.saveErrors[ids[0]]
s.workspaceTurn.mu.Unlock()
if !pending {
t.Fatal("the unsaved old workspace would not be reported at turn end")
}
if _, err := gateway.callMainBotWorkspace(context.Background(), s, "workbench_workspace_tree", map[string]any{}); err != nil {
t.Fatalf("the new workspace is unusable after the switch: %v", err)
}
// Switching back and saving it successfully resolves that warning.
if _, err := gateway.bindConversationWorkspace(context.Background(), s, workspaceOpenChoice{SandboxID: turnSandbox(ids[0])}); err != nil {
t.Fatal(err)
}
if _, err := gateway.callMainBotWorkspace(context.Background(), s, "workbench_workspace_tree", map[string]any{}); err != nil {
t.Fatal(err)
}
if err := s.workspaceTurn.close(context.Background()); err != nil {
t.Fatalf("a successful save left a stale checkpoint warning: %v", err)
}
}
// Opening a new workspace after using one this turn is the path the
// missingRepositories hint steers Main Bot to. A save that keeps failing must
// neither block it nor leave behind a created but unbound workspace run.
func TestAssistantWorkspaceTurnNewWorkspaceSurvivesFailedSave(t *testing.T) {
h, owner, s := chatSandboxFixture(t)
gateway := &agentToolGateway{deps: h.deps}
en := &workspaceTurnEngine{saveErr: errors.New("workspace checkpoint exceeds size limit")}
s.workspaceTurn = newAssistantWorkspaceTurn(en, nil)
t.Cleanup(func() { _ = s.workspaceTurn.close(context.Background()) })
workspaceRuns := func() []string {
var ids []string
for _, run := range h.repo.Runs() {
if run.IsWorkspace() && run.ExecutionContext.ProjectID == s.ProjectID {
ids = append(ids, run.ID)
}
}
return ids
}
first, err := gateway.bindConversationWorkspace(context.Background(), s, workspaceOpenChoice{})
if err != nil {
t.Fatal(err)
}
if _, err := s.workspaceTurn.lease(context.Background(), first.Run.ID); err != nil {
t.Fatal(err)
}
second, err := gateway.bindConversationWorkspace(context.Background(), s, workspaceOpenChoice{SandboxID: "business"})
if err != nil || !second.Created || second.Previous != first.Run.ID || !strings.Contains(second.CheckpointError, "exceeds size limit") {
t.Fatalf("a failed save blocked opening a new workspace: %+v %v", second, err)
}
stored, _ := h.repo.AssistantSession(owner.ID, s.ChatSessionID)
if stored.WorkspaceRunID != second.Run.ID || len(stored.OwnWorkspaceRunIDs) != 2 {
t.Fatalf("the new workspace was not bound: %+v", stored)
}
if runs := workspaceRuns(); len(runs) != 2 {
t.Fatalf("workspace runs = %v, want only the two bound ones", runs)
}
if strings.Join(en.ended, ",") != first.Run.ID {
t.Fatalf("checkpoints = %v, want one for the workspace switched away from", en.ended)
}
if err := s.workspaceTurn.close(context.Background()); err == nil || !strings.Contains(err.Error(), first.Run.ID) {
t.Fatalf("the unsaved workspace was not reported at turn end: %v", err)
}
if _, err := gateway.bindConversationWorkspace(context.Background(), s, workspaceOpenChoice{SandboxID: "shared"}); !errors.Is(err, errAssistantTurnClosed) {
t.Fatalf("a finished turn opened a workspace: %v", err)
}
if runs := workspaceRuns(); len(runs) != 2 {
t.Fatalf("a refused open left a workspace run behind: %v", runs)
}
}
func TestAssistantWorkspaceTurnClosedRefusesSwitch(t *testing.T) {
h, _, s, gateway, ids := workspaceTurnFixture(t)
if err := s.workspaceTurn.close(context.Background()); err != nil {
t.Fatal(err)
}
if _, err := gateway.bindConversationWorkspace(context.Background(), s, workspaceOpenChoice{SandboxID: turnSandbox(ids[1])}); !errors.Is(err, errAssistantTurnClosed) {
t.Fatalf("a finished turn switched workspaces: %v", err)
}
conv, _ := h.repo.AssistantSession(s.ChatOwnerID, s.ChatSessionID)
if s.workspaceRun() != ids[0] || conv.WorkspaceRunID != ids[0] {
t.Fatal("a refused switch changed the binding")
}
}
func TestAssistantWorkspaceTurnGenerationDoesNotWaitForCheckpoint(t *testing.T) {
h, user, fixtureSession, gateway, ids := workspaceTurnFixture(t)
barrier := &workspaceTurnCheckpointBarrier{assistantWorkspaceLeaseEngine: h.engine, entered: make(chan struct{}), release: make(chan struct{}), err: errors.New("test checkpoint warning")}
var release sync.Once
unblock := func() { release.Do(func() { close(barrier.release) }) }
defer unblock()
h.generate = func(ctx context.Context, _ string, _ domain.User) (string, error) {
var session *AgentToolSession
agentToolSessions.mu.Lock()
for _, candidate := range agentToolSessions.sessions {
if candidate.ChatSessionID == fixtureSession.ChatSessionID {
session = candidate
}
}
agentToolSessions.mu.Unlock()
if session == nil || session.workspaceTurn == nil {
return "", errors.New("generation has no workspace lifetime")
}
session.workspaceTurn.mu.Lock()
session.workspaceTurn.engine = barrier
session.workspaceTurn.mu.Unlock()
err := workspaceTestEdit(ctx, gateway, session, "note.txt", "latest edit")
return "done", err
}
body, _ := json.Marshal(assistantRequest{SessionID: fixtureSession.ChatSessionID, ProjectID: fixtureSession.ProjectID, Messages: []assistantMessage{{Role: "user", Content: "edit"}}})
c, rec := workbenchContext(user, "POST", "/api/assistant/generations", bytes.NewBuffer(body))
h.startGeneration(c)
if rec.Code != 202 {
t.Fatalf("generation start: %d %s", rec.Code, rec.Body)
}
select {
case <-barrier.entered:
case <-time.After(10 * time.Second):
t.Fatal("turn never reached its final checkpoint")
}
deadline := time.Now().Add(time.Second)
var g domain.AssistantGeneration
for {
g, _ = h.repo.AssistantGeneration(user.ID, fixtureSession.ChatSessionID)
if g.Status != "running" {
break
}
if time.Now().After(deadline) {
t.Fatal("generation waited for background save")
}
time.Sleep(10 * time.Millisecond)
}
if g.Status != "completed" || g.WorkspaceWarning != "" {
t.Fatalf("outcome: %+v", g)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
defer cancel()
if err := inflightTasks.Wait(ctx); !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("deployment drain did not retain the background save: %v", err)
}
unblock()
if got := workspaceTurnCheckpointText(t, h, ids[0]); got != "latest edit" {
t.Fatalf("checkpoint lost edits: %q", got)
}
}
func TestAssistantCancelledGenerationDoesNotWaitForCheckpoint(t *testing.T) {
h, user, fixtureSession, gateway, ids := workspaceTurnFixture(t)
barrier := &workspaceTurnCheckpointBarrier{assistantWorkspaceLeaseEngine: h.engine, entered: make(chan struct{}), release: make(chan struct{})}
var release sync.Once
unblock := func() { release.Do(func() { close(barrier.release) }) }
defer unblock()
edited := make(chan struct{})
h.generate = func(ctx context.Context, _ string, _ domain.User) (string, error) {
agentToolSessions.mu.Lock()
var session *AgentToolSession
for _, candidate := range agentToolSessions.sessions {
if candidate.ChatSessionID == fixtureSession.ChatSessionID {
session = candidate
}
}
agentToolSessions.mu.Unlock()
if session == nil || session.workspaceTurn == nil {
return "", errors.New("missing workspace turn")
}
session.workspaceTurn.mu.Lock()
session.workspaceTurn.engine = barrier
session.workspaceTurn.mu.Unlock()
if err := workspaceTestEdit(ctx, gateway, session, "note.txt", "edit before stop"); err != nil {
return "", err
}
close(edited)
<-ctx.Done()
return "", ctx.Err()
}
body, _ := json.Marshal(assistantRequest{SessionID: fixtureSession.ChatSessionID, ProjectID: fixtureSession.ProjectID, Messages: []assistantMessage{{Role: "user", Content: "edit then wait"}}})
c, rec := workbenchContext(user, "POST", "/api/assistant/generations", bytes.NewBuffer(body))
h.startGeneration(c)
if rec.Code != 202 {
t.Fatalf("start: %d %s", rec.Code, rec.Body)
}
select {
case <-edited:
case <-time.After(5 * time.Second):
t.Fatal("edit did not start")
}
g, _ := h.repo.AssistantGeneration(user.ID, fixtureSession.ChatSessionID)
began := time.Now()
c, rec = workbenchContext(user, "POST", "/api/assistant/sessions/"+g.SessionID+"/generation/cancel?generationId="+g.ID, bytes.NewBufferString(`{}`))
c.Params = append(c.Params, gin.Param{Key: "id", Value: g.SessionID})
h.generation(c)
if rec.Code != 200 {
t.Fatalf("cancel: %d %s", rec.Code, rec.Body)
}
select {
case <-barrier.entered:
case <-time.After(time.Second):
t.Fatal("stop did not hand off checkpoint")
}
for {
g, _ = h.repo.AssistantGeneration(user.ID, g.SessionID)
if g.Status != "running" {
break
}
if time.Since(began) > time.Second {
t.Fatal("stop waited for background checkpoint")
}
time.Sleep(5 * time.Millisecond)
}
if g.Status != "cancelled" || g.WorkspaceWarning != "" {
t.Fatalf("stop outcome: %+v", g)
}
run, _ := h.repo.RunState(ids[0])
root, _ := h.engine.LocalRunWorkspace(run)
content, err := os.ReadFile(filepath.Join(root, "note.txt"))
if err != nil || string(content) != "edit before stop" {
t.Fatalf("stop lost edits: %q %v", content, err)
}
token, err := h.engine.BeginConversationWorkspaceUse(ids[0])
if err != nil {
t.Fatal("next turn cannot acquire workspace", err)
}
if err = h.engine.EndConversationWorkspaceUse(context.Background(), ids[0], token); err != nil {
t.Fatal(err)
}
unblock()
}
func TestAssistantWorkspaceTurnRestoresIdleWorkspaceBeforeNativeExecution(t *testing.T) {
h, _, session, _, ids := workspaceTurnFixture(t)
id := ids[0]
token, err := h.engine.BeginConversationWorkspaceUse(id)
if err != nil {
t.Fatal(err)
}
if err = h.engine.EnsureConversationWorkspace(context.Background(), id, token); err != nil {
t.Fatal(err)
}
if err = h.engine.EndConversationWorkspaceUse(context.Background(), id, token); err != nil {
t.Fatal(err)
}
root := h.engine.RunWorkspace(id)
// Simulate the idle sweep retaining its snapshot while reclaiming the files.
if err = h.repo.DB.Table("runs").Where("id = ?", id).Update("workspace_cleaned_at", time.Now().UTC()).Error; err != nil {
t.Fatal(err)
}
if err = os.RemoveAll(root); err != nil {
t.Fatal(err)
}
if _, err = session.workspaceTurn.lease(context.Background(), id); err != nil {
t.Fatal(err)
}
got, err := os.ReadFile(filepath.Join(root, "note.txt"))
if err != nil || string(got) != "before" {
t.Fatalf("native CLI would see an empty recreated workspace: %q %v", got, err)
}
run, _ := h.repo.RunState(id)
meta, _ := h.repo.WorkspaceCheckpoint(id)
if !run.WorkspaceCleanedAt.IsZero() || !meta.FollowUp {
t.Fatal("restored workspace stayed marked reclaimed")
}
}