Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

28.0 KB
Raw Blame
package core

import (
	"context"
	"errors"
	"fmt"
	"math/rand"
	"runtime"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"ragflow/internal/harness/core/schema"
)

// ============================================================================
// Stress/Defect-Discovery Test Suite for workflow.go, agent_loop.go, flow.go
//
// These tests are designed NOT to pass trivially. They target specific known
// weaknesses and edge cases to expose bugs or design flaws.
// ============================================================================

// ---- Bug #1: drainEvents drops error events (workflow.go:263) ----
//
// drainEvents sends the error to gen but returns nil, causing runSeq to
// continue to the next sub-agent instead of terminating the workflow.
// Expected: a failing sub-agent should stop the entire sequential workflow.
// Actual (current): the error is sent to the event stream, but runSeq
// continues, resulting in partial execution after the failing node.
func TestWorkflow_ErrorInSubAgent_ShouldStopNotContinue(t *testing.T) {
	var execOrder []string
	var mu sync.Mutex

	agents := make([]Agent, 5)
	for i := 0; i < 5; i++ {
		i := i
		nodeID := fmt.Sprintf("node_%02d", i)
		if i == 2 {
			// Node 2 fails
			agents[i] = newErrorAgent(nodeID + "_error")
		} else {
			tool := workflowNodeTool(nodeID, &execOrder, &mu)
			model := &forcedToolModel{
				toolCalls: []schema.ToolCall{{ID: fmt.Sprintf("c%d", i), Function: schema.ToolCallFunction{Name: tool.Name(), Arguments: "{}"}}},
				finalResp: fmt.Sprintf("final from %s", nodeID),
				firstCall: true,
			}
			agents[i] = NewReActAgent(&ReActConfig[*schema.Message]{Model: model, Tools: []Tool{tool}}).WithName(nodeID)
		}
	}

	wf, err := NewSequential(t.Context(), &SequentialConfig{
		Name: "error_stop_test", Description: "5 nodes with middle failure",
		SubAgents: agents,
	})
	if err != nil {
		t.Fatal(err)
	}

	ctx := t.Context()
	iter := wf.Run(ctx, &AgentInput{Messages: []Message{schema.UserMessage("run")}})

	var gotError bool
	for {
		ev, ok := iter.Next()
		if !ok {
			break
		}
		if ev.Err != nil {
			gotError = true
			t.Logf("Got expected error: %v", ev.Err)
		}
	}

	if !gotError {
		t.Error("BUG: expected an error from the failing sub-agent, but none was received")
	}

	// The bug: node_03 and node_04 should NOT have executed because node_02 failed.
	// If they did, drainEvents is masking the error.
	mu.Lock()
	execCount := len(execOrder)
	mu.Unlock()

	if execCount > 2 {
		t.Errorf("BUG: node_02 failed, but %d nodes after it still executed. "+
			"drainEvents returns nil after error, so runSeq continues. "+
			"Expected ≤2 tool executions, got %d (executed: %v)",
			2, execCount, execOrder)
	}
	t.Logf("Executed %d tools before error stopped workflow (expected ≤2)", execCount)
}

// errorAgent returns an error immediately on Run.
type errorAgent struct {
	name string
}

func newErrorAgent(name string) Agent {
	return &errorAgent{name: name}
}

func (a *errorAgent) Name(_ context.Context) string        { return a.name }
func (a *errorAgent) Description(_ context.Context) string { return a.name + " error" }
func (a *errorAgent) GetType() string                      { return "ErrorAgent" }
func (a *errorAgent) Run(_ context.Context, _ *AgentInput, _ ...RunOption) *AsyncIterator[*AgentEvent] {
	it, gen := NewAsyncIteratorPair[*AgentEvent]()
	gen.Send(&AgentEvent{Err: errors.New("intentional agent failure")})
	gen.Close()
	return it
}

// ---- Bug #2: cancelTransition uses Interrupted instead of CancelError (workflow.go:106) ----
//
// cancelTransition() creates an Interrupted action (business interrupt) when a cancel
// is requested, rather than a CancelError. This means cancellation in a sequential
// workflow is indistinguishable from a business interrupt. The upper AgentLoop treats
// them differently — CancelError causes clean exit, Interrupted saves checkpoint.
// This test verifies the distinction is correct.
func TestWorkflow_SequentialCancel_ShouldNotTriggerInterruptCheckpoint(t *testing.T) {
	var execOrder []string
	var mu sync.Mutex

	wf, err := buildSequentialWorkflow(6, &execOrder, &mu)
	if err != nil {
		t.Fatal(err)
	}

	store := newConcurrentStore()
	ctx := t.Context()
	opt, cancel := WithCancel()
	cpID := "cancel_no_checkpoint_test"

	runner := NewTypedRunner(RunnerConfig[*schema.Message]{
		Agent:           wf,
		CheckPointStore: store,
	})
	evIter := runner.Run(ctx, []*schema.Message{schema.UserMessage("cancel test")},
		WithCheckPointID(cpID), opt)

	// Cancel after node 3
	for {
		ev, ok := evIter.Next()
		if !ok {
			break
		}
		_ = ev
		mu.Lock()
		orderLen := len(execOrder)
		mu.Unlock()
		if orderLen >= 3 {
			cancel()
			break
		}
	}
	// Drain remaining
	for {
		_, ok := evIter.Next()
		if !ok {
			break
		}
	}

	// After cancel, checkpoint should NOT have been saved for interrupt purposes.
	// The cancelTransition creates an Interrupted action, which may trigger checkpoint save.
	_, found, err := store.Get(ctx, cpID)
	if err != nil {
		t.Fatal(err)
	}

	// This assertion exposes the bug: if found == true, the cancel was incorrectly
	// treated as an interrupt, saving an unnecessary checkpoint.
	if found {
		t.Errorf("BUG: cancel in sequential workflow saved a checkpoint. " +
			"cancelTransition uses Interrupted action instead of CancelError, " +
			"so the upper AgentLoop treats cancel as a business interrupt and saves checkpoint. " +
			"Cancel should NOT produce an interrupt checkpoint.")
	}
	t.Logf("Cancel checkpoint found=%v (expected false if cancel is clean)", found)
}

// ---- Bug #3: AgentLoop goroutine leak detection ----
//
// Tests that calling Stop() does not leak goroutines. The AgentLoop starts
// multiple goroutines (run, handleEvents, watchPreempt, watchStop, proxyGen).
// This test uses runtime.NumGoroutine to detect leaks.
func TestAgentLoop_GoroutineLeak(t *testing.T) {
	initial := runtime.NumGoroutine()

	for i := 0; i < 20; i++ {
		loop := NewAgentLoop[*schema.Message](AgentLoopConfig[*schema.Message]{
			GenInput: func(_ context.Context, _ *AgentLoop[*schema.Message], items []*schema.Message) (*GenInputResult[*schema.Message], error) {
				return &GenInputResult[*schema.Message]{
					Input: &AgentInput{Messages: items}, Consumed: items,
				}, nil
			},
			PrepareAgent: func(_ context.Context, _ *AgentLoop[*schema.Message], _ []*schema.Message) (Agent, error) {
				m := &mockModel{}
				m.addResp("ok")
				return NewReActAgent(&ReActConfig[*schema.Message]{Model: m}).WithName("leak_test"), nil
			},
		})
		loop.Push(schema.UserMessage("leak"))
		loop.Run(context.Background())
		loop.Stop()
		_ = loop.Wait()
		time.Sleep(time.Millisecond * 5)
	}

	// Give goroutines time to clean up
	time.Sleep(time.Millisecond * 50)
	after := runtime.NumGoroutine()

	leaked := after - initial
	if leaked > 5 {
		t.Errorf("BUG: possible goroutine leak: started with %d goroutines, ended with %d "+
			"(diff=%d, expected <5)", initial, after, leaked)
	}
	t.Logf("Goroutine check: initial=%d, after=%d, diff=%d", initial, after, leaked)
}

// ---- Bug #4: flow.go transfer loop — context cancel doesn't stop infinite loop ----
//
// flow.runLoop (line 213-218) handles transfer by calling next.Run().Next() in a loop.
// If the context is canceled, the loop should terminate but doesn't check for it.
func TestFlow_TransferLoop_ContextCancel(t *testing.T) {
	agentA := NewReActAgent(&ReActConfig[*schema.Message]{
		Model: &forcedToolModel{
			toolCalls: []schema.ToolCall{{ID: "c1", Function: schema.ToolCallFunction{Name: "dummy_tool", Arguments: "{}"}}},
			finalResp: "done",
			firstCall: true,
		},
		Tools: []Tool{&mockTool{name: "dummy_tool", desc: "dummy"}},
	}).WithName("agent_a")

	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan struct{})

	go func() {
		defer close(done)
		iter := agentA.Run(ctx, &AgentInput{Messages: []Message{schema.UserMessage("start")}})
		for {
			ev, ok := iter.Next()
			if !ok {
				break
			}
			_ = ev
		}
	}()

	// Cancel after a short delay
	time.Sleep(time.Millisecond * 50)
	cancel()

	select {
	case <-done:
		// Agent terminated cleanly
	case <-time.After(time.Second * 5):
		t.Errorf("BUG: agent did not terminate within 5s after context cancel. " +
			"flow.go runLoop may not check context cancellation, causing goroutine leak")
	}
}

// ---- Bug #5: Concurrent Push + Stop race ----
//
// Push items while concurrently calling Stop(). The AgentLoop has lateItems and
// buffer that could race. This test tries to trigger the race.
func TestAgentLoop_ConcurrentPushStop_Race(t *testing.T) {
	const iterations = 50

	for i := 0; i < iterations; i++ {
		loop := NewAgentLoop[*schema.Message](AgentLoopConfig[*schema.Message]{
			GenInput: func(_ context.Context, _ *AgentLoop[*schema.Message], items []*schema.Message) (*GenInputResult[*schema.Message], error) {
				return &GenInputResult[*schema.Message]{
					Input: &AgentInput{Messages: items}, Consumed: items,
				}, nil
			},
			PrepareAgent: func(_ context.Context, _ *AgentLoop[*schema.Message], _ []*schema.Message) (Agent, error) {
				m := &mockModel{}
				m.addResp("ok")
				return NewReActAgent(&ReActConfig[*schema.Message]{Model: m}).WithName("race_test"), nil
			},
		})

		var wg sync.WaitGroup
		wg.Add(3)

		// Push items concurrently
		go func() {
			defer wg.Done()
			for j := 0; j < 10; j++ {
				loop.Push(schema.UserMessage(fmt.Sprintf("item_%d_%d", i, j)))
				time.Sleep(time.Microsecond * time.Duration(rand.Intn(100)))
			}
		}()

		// Start loop concurrently
		go func() {
			defer wg.Done()
			time.Sleep(time.Microsecond * 50)
			loop.Run(context.Background())
		}()

		// Stop concurrently
		go func() {
			defer wg.Done()
			time.Sleep(time.Microsecond * time.Duration(50+rand.Intn(100)))
			loop.Stop()
		}()

		wg.Wait()
		_ = loop.Wait()
	}
}

// ---- Bug #6: closing a paired iterator must not make runner cleanup panic ----
func TestRunner_HandleIter_PanicSafety(t *testing.T) {
	ai, inputGen := NewAsyncIteratorPair[*TypedAgentEvent[*schema.Message]]()
	outputIter, outputGen := NewAsyncIteratorPair[*TypedAgentEvent[*schema.Message]]()
	outputIter.Close()
	inputGen.Send(nil)
	inputGen.Close()

	done := make(chan struct{})
	var panicValue any
	go func() {
		defer close(done)
		defer func() { panicValue = recover() }()
		handleIter(false, nil, t.Context(), ai, outputGen, nil, nil)
	}()

	select {
	case <-done:
		if panicValue != nil {
			t.Fatalf("handleIter panicked after its iterator was closed: %v", panicValue)
		}
	case <-time.After(time.Second * 5):
		t.Fatal("handleIter did not terminate after its iterator was closed")
	}
}

type panicAgent struct {
	name string
}

func (a *panicAgent) Name(_ context.Context) string        { return a.name }
func (a *panicAgent) Description(_ context.Context) string { return "panics" }
func (a *panicAgent) GetType() string                      { return "PanicAgent" }
func (a *panicAgent) Run(_ context.Context, _ *AgentInput, _ ...RunOption) *AsyncIterator[*AgentEvent] {
	_, gen := NewAsyncIteratorPair[*AgentEvent]()
	gen.Send(&AgentEvent{Err: errors.New("before panic")})
	panic("intentional panic in Run()")
}

// ---- Bug #7: workflow runSeq — Exit action does not stop AgentLoop ----
//
// When a sub-agent returns Exit action, runSeq sends the event to gen and returns nil.
// But the AgentLoop's runAgentAndHandleEvents only checks interruptContexts and
// capturedCancelErr — it doesn't check for Exit. So the AgentLoop continues to the
// next iteration instead of stopping.
func TestWorkflow_ExitAction_ShouldStopAgentLoop(t *testing.T) {
	exitAgent := &exitAgent{name: "exit_agent"}

	loop := NewAgentLoop[*schema.Message](AgentLoopConfig[*schema.Message]{
		GenInput: func(_ context.Context, _ *AgentLoop[*schema.Message], items []*schema.Message) (*GenInputResult[*schema.Message], error) {
			return &GenInputResult[*schema.Message]{
				Input: &AgentInput{Messages: items}, Consumed: items,
			}, nil
		},
		PrepareAgent: func(_ context.Context, _ *AgentLoop[*schema.Message], _ []*schema.Message) (Agent, error) {
			return exitAgent, nil
		},
	})

	loop.Push(schema.UserMessage("test exit"))
	loop.Run(context.Background())
	loop.Stop()
	result := loop.Wait()

	// After Exit action, AgentLoop should stop cleanly, not produce more events
	_ = result
	t.Logf("Exit action test completed. Result exit reason: %v", result.ExitReason)
}

type exitAgent struct {
	name string
}

func (a *exitAgent) Name(_ context.Context) string        { return a.name }
func (a *exitAgent) Description(_ context.Context) string { return "returns exit" }
func (a *exitAgent) GetType() string                      { return "ExitAgent" }
func (a *exitAgent) Run(_ context.Context, _ *AgentInput, _ ...RunOption) *AsyncIterator[*AgentEvent] {
	it, gen := NewAsyncIteratorPair[*AgentEvent]()
	gen.Send(&AgentEvent{Action: &AgentAction{Exit: true}})
	gen.Close()
	return it
}

// ---- Bug #8: React loop doesn't call runAfterAgent on max iteration exceeded ----
//
// react_loop.go buildReActRunFunc: when state.RemainingIterations <= 0 (line 93),
// it sends error and returns, skipping runAfterAgent. Middleware cleanup is missed.
func TestReAct_MaxIterationExceeded_SkipsAfterAgent(t *testing.T) {
	var afterAgentCalled atomic.Bool

	// Use a custom middleware that tracks AfterAgent call
	afterAgentMW := &callTrackingMiddleware{
		onAfterAgent: func() { afterAgentCalled.Store(true) },
	}

	// Create a model that always returns tool calls (infinite loop)
	loopModel := &loopToolModel{
		toolCalls: []schema.ToolCall{
			{ID: "c1", Function: schema.ToolCallFunction{Name: "loop_tool", Arguments: "{}"}},
		},
	}

	agent := NewReActAgent(&ReActConfig[*schema.Message]{
		Model:         loopModel,
		MaxIterations: 3,
		Tools:         []Tool{&mockTool{name: "loop_tool", desc: "loops"}},
		Middlewares:   []TypedReActMiddleware[*schema.Message]{afterAgentMW},
	})

	ctx := t.Context()
	iter := agent.Run(ctx, &AgentInput{Messages: []Message{schema.UserMessage("loop test")}})

	var gotMaxIterError bool
	for {
		ev, ok := iter.Next()
		if !ok {
			break
		}
		if ev.Err != nil {
			gotMaxIterError = true
			t.Logf("Got error: %v", ev.Err)
		}
	}

	if !gotMaxIterError {
		t.Fatal("expected max iteration error")
	}

	if !afterAgentCalled.Load() {
		t.Errorf("BUG: AfterAgent middleware not called after max iteration exceeded. " +
			"buildReActRunFunc returns early on line 94, skipping runAfterAgent. " +
			"Middleware cleanup/hooks are missed.")
	} else {
		t.Log("AfterAgent middleware was called (pass)")
	}
}

// callTrackingMiddleware is a simple middleware that tracks calls to AfterAgent.
type callTrackingMiddleware struct {
	TypedReActMiddleware[*schema.Message]
	onAfterAgent func()
}

func (m *callTrackingMiddleware) BeforeAgent(ctx context.Context, rc *ReActAgentContext) (context.Context, *ReActAgentContext, error) {
	return ctx, rc, nil
}
func (m *callTrackingMiddleware) BeforeModelRewrite(ctx context.Context, state *ReActAgentState, mc *ModelContext) (context.Context, *ReActAgentState, error) {
	return ctx, state, nil
}
func (m *callTrackingMiddleware) AfterModelRewrite(ctx context.Context, state *ReActAgentState, mc *ModelContext) (context.Context, *ReActAgentState, error) {
	return ctx, state, nil
}
func (m *callTrackingMiddleware) AfterAgent(ctx context.Context, state *ReActAgentState) (context.Context, error) {
	if m.onAfterAgent != nil {
		m.onAfterAgent()
	}
	return ctx, nil
}
func (m *callTrackingMiddleware) WrapModel(ctx context.Context, model Model[*schema.Message], mc *ModelContext) (Model[*schema.Message], error) {
	return model, nil
}

// ---- Bug #9: workflow runSeq cancelTransition data leakage ----
//
// cancelTransition creates an Interrupted action with msg and state in
// internalInterrupted. This state is used by the upper framework to determine
// if a checkpoint should be saved. If cancel is confused with interrupt,
// the state saved may be incorrect.
func TestWorkflow_SequentialCancel_StateConsistency(t *testing.T) {
	var execOrder []string
	var mu sync.Mutex

	wf, err := buildSequentialWorkflow(8, &execOrder, &mu)
	if err != nil {
		t.Fatal(err)
	}

	ctx := t.Context()
	opt, cancel := WithCancel()

	iter := wf.Run(ctx, &AgentInput{Messages: []Message{schema.UserMessage("consistency")}}, opt)

	var lastEvent *AgentEvent
	for {
		ev, ok := iter.Next()
		if !ok {
			break
		}
		lastEvent = ev
		mu.Lock()
		orderLen := len(execOrder)
		mu.Unlock()
		if orderLen >= 4 {
			cancel()
		}
	}

	if lastEvent != nil && lastEvent.Action != nil && lastEvent.Action.Interrupted != nil {
		t.Logf("Cancel produced Interrupted action (may be a bug): Data=%v", lastEvent.Action.Interrupted.Data)
		// Check if the interrupt data suggests it was treated as a business interrupt
		if msg, ok := lastEvent.Action.Interrupted.Data.(string); ok && msg == "Sequential cancel" {
			t.Errorf("BUG: cancel is represented as Interrupted with Data=%q. "+
				"This causes the framework to save a checkpoint for a cancel, "+
				"which is unnecessary and may confuse resume logic.", msg)
		}
	}
}

// ---- Bug #10: drainEventsChan goroutine leak ----
//
// drainEventsChan starts a goroutine that loops on iter.Next(). If the caller
// breaks out of the for-range loop (e.g., via break), the goroutine is leaked
// because it's blocked on iter.Next().
func TestDrainEventsChan_GoroutineLeak(t *testing.T) {
	initial := runtime.NumGoroutine()

	for i := 0; i < 10; i++ {
		wf, err := buildSequentialWorkflow(3, &[]string{}, &sync.Mutex{})
		if err != nil {
			t.Fatal(err)
		}
		ctx := t.Context()
		iter := wf.Run(ctx, &AgentInput{Messages: []Message{schema.UserMessage("leak test")}})

		ch := drainEventsChan(iter)
		// Read just 1 event then break — goroutine should still be alive
		select {
		case _, ok := <-ch:
			if !ok {
				continue
			}
		case <-time.After(time.Second):
			t.Fatal("timeout waiting for first event")
		}
		// Break early — the drain goroutine is leaked!
		break
	}

	time.Sleep(time.Millisecond * 50)
	after := runtime.NumGoroutine()
	leaked := after - initial

	if leaked > 5 {
		t.Errorf("BUG: drainEventsChan goroutine leak detected: %d goroutines leaked. "+
			"The goroutine blocks on iter.Next() even after the caller breaks the loop.", leaked)
	} else {
		t.Logf("No significant goroutine leak: initial=%d, after=%d", initial, after)
	}
}

// ---- Bug #11: Sequential workflow drainEvents drops non-action last events ----
//
// In workflow.go runSeq, drainEvents returns the last AgentEvent only if it has an Action.
// If the last event is a regular message output without action, drainEvents returns nil
// and runSeq continues to the next sub-agent without propagating the final event.
func TestWorkflow_Sequential_LastEventWithoutActionIsDropped(t *testing.T) {
	// Create an agent that returns a message event without any action
	plainMsgAgent := &plainMessageAgent{name: "plain_msg_agent"}

	wf, err := NewSequential(t.Context(), &SequentialConfig{
		Name: "drop_test", Description: "test last event dropping",
		SubAgents: []Agent{plainMsgAgent},
	})
	if err != nil {
		t.Fatal(err)
	}

	ctx := t.Context()
	iter := wf.Run(ctx, &AgentInput{Messages: []Message{schema.UserMessage("test")}})

	var events []*AgentEvent
	for {
		ev, ok := iter.Next()
		if !ok {
			break
		}
		events = append(events, ev)
	}

	if len(events) == 0 {
		t.Errorf("BUG: no events received from sequential workflow. " +
			"drainEvents in runSeq returns nil when the last event has no Action, " +
			"so the final message output is dropped by runSeq (it's not forwarded to gen).")
	} else {
		t.Logf("Received %d events (first is probably the dropped one)", len(events))
		for i, ev := range events {
			var out string
			if ev.Output != nil && ev.Output.MessageOutput != nil && ev.Output.MessageOutput.Message != nil {
				out = ev.Output.MessageOutput.Message.Content
			}
			t.Logf("  event[%d]: Output=%v, Action=%v, Err=%v", i, out, ev.Action, ev.Err)
		}
	}
}

type plainMessageAgent struct {
	name string
}

func (a *plainMessageAgent) Name(_ context.Context) string        { return a.name }
func (a *plainMessageAgent) Description(_ context.Context) string { return "plain msg" }
func (a *plainMessageAgent) GetType() string                      { return "PlainMsgAgent" }
func (a *plainMessageAgent) Run(_ context.Context, _ *AgentInput, _ ...RunOption) *AsyncIterator[*AgentEvent] {
	it, gen := NewAsyncIteratorPair[*AgentEvent]()
	gen.Send(&AgentEvent{
		AgentName: a.name,
		Output: &AgentOutput{MessageOutput: &TypedMessageVariant[*schema.Message]{
			Message: &schema.Message{Role: schema.RoleAssistant, Content: "hello"},
		}},
	})
	gen.Close()
	return it
}

// ---- Bug #12: Concurrent workflow with shared state race ----
//
// Multiple concurrent agents share the same runContext.Session.Values map.
// If values are modified concurrently, data race occurs.
func TestWorkflow_Parallel_SharedSessionValuesRace(t *testing.T) {
	agents := make([]Agent, 10)
	for i := 0; i < 10; i++ {
		i := i
		nodeID := fmt.Sprintf("parallel_%02d", i)
		m := &mockModel{}
		m.addResp(fmt.Sprintf("result %d", i))
		agents[i] = NewReActAgent(&ReActConfig[*schema.Message]{
			Model: m,
		}).WithName(nodeID)
	}

	wf, err := NewParallel(t.Context(), &ParallelConfig{
		Name: "parallel_race", Description: "test session value race",
		SubAgents: agents,
	})
	if err != nil {
		t.Fatal(err)
	}

	ctx := t.Context()
	iter := wf.Run(ctx, &AgentInput{Messages: []Message{schema.UserMessage("race test")}})
	for range drainEventsChan(iter) {
	}
	t.Log("Parallel shared session values race test completed (run with -race)")
}

// ---- Bug #13: AgentLoop idle timer goroutine leak ----
//
// agent_loop_run.go line 135 starts a goroutine for idle timer.
// If commitStop is called externally while the timer goroutine is running,
// the goroutine may leak or cause double-commit.
func TestAgentLoop_IdleTimerGoroutineLeak(t *testing.T) {
	initial := runtime.NumGoroutine()

	for i := 0; i < 10; i++ {
		loop := NewAgentLoop[*schema.Message](AgentLoopConfig[*schema.Message]{
			GenInput: func(_ context.Context, _ *AgentLoop[*schema.Message], items []*schema.Message) (*GenInputResult[*schema.Message], error) {
				return &GenInputResult[*schema.Message]{
					Input: &AgentInput{Messages: items}, Consumed: items,
				}, nil
			},
			PrepareAgent: func(_ context.Context, _ *AgentLoop[*schema.Message], _ []*schema.Message) (Agent, error) {
				m := &mockModel{}
				m.addResp("ok")
				return NewReActAgent(&ReActConfig[*schema.Message]{Model: m}).WithName("idle_test"), nil
			},
		})
		loop.Push(schema.UserMessage("idle"))
		loop.Run(context.Background())

		// Set idle timeout and immediately stop — should not leak goroutines
		loop.Stop(UntilIdleFor(time.Millisecond * 100))
		loop.Stop() // immediate stop to cancel idle timer
		_ = loop.Wait()
	}

	time.Sleep(time.Millisecond * 200) // wait for any lingering timers
	after := runtime.NumGoroutine()

	leaked := after - initial
	if leaked > 5 {
		t.Errorf("BUG: possible goroutine leak from idle timer: initial=%d, after=%d, diff=%d",
			initial, after, leaked)
	}
	t.Logf("Idle timer goroutine check: initial=%d, after=%d", initial, after)
}

// ---- Bug #14: Checkpoint data race under concurrent resume ----
//
// Multiple concurrent resumes from the same checkpoint should be safe.
// This tests for data races in the checkpoint store.
func TestCheckpoint_ConcurrentResumeRace(t *testing.T) {
	store := newConcurrentStore()

	// Run a workflow, interrupt it, save checkpoint
	var execOrder []string
	var mu sync.Mutex
	wf, err := buildSequentialWorkflow(5, &execOrder, &mu)
	if err != nil {
		t.Fatal(err)
	}

	ctx := t.Context()
	cpID := "concurrent_resume_test"

	runner := NewTypedRunner(RunnerConfig[*schema.Message]{
		Agent:           wf,
		CheckPointStore: store,
	})
	iter := runner.Run(ctx, []*schema.Message{schema.UserMessage("resume test")},
		WithCheckPointID(cpID))

	// Run until interrupted
	for {
		ev, ok := iter.Next()
		if !ok {
			break
		}
		if ev.Action != nil && ev.Action.Interrupted != nil {
			t.Logf("Interrupted at: %v", ev.Action.Interrupted.Data)
			break
		}
	}

	// Now resume concurrently
	var wg sync.WaitGroup
	for i := 0; i < 5; i++ {
		wg.Add(1)
		go func(id int) {
			defer wg.Done()
			r := NewTypedRunner(RunnerConfig[*schema.Message]{
				Agent:           wf,
				CheckPointStore: store,
			})
			it, err := r.Resume(t.Context(), cpID)
			if err != nil {
				t.Logf("Tenant %d resume error: %v", id, err)
				return
			}
			for {
				_, ok := it.Next()
				if !ok {
					break
				}
			}
		}(i)
	}
	wg.Wait()
	t.Log("Concurrent resume test completed (run with -race)")
}

// ---- Bug #15: flowAgent deepCopy loses subAgents after SetSubAgents ----
//
// flow.go SetSubAgents checks if len(fa.subAgents) > 0 but doesn't actually
// set them on the flowAgent. The toFlowAgent deepCopy copies subAgents but
// SetSubAgents doesn't populate them. This means workflow.subAgents and
// flowAgent.subAgents are out of sync.
func TestFlow_SetSubAgents_DoesNotPopulateSubAgents(t *testing.T) {
	agent := NewReActAgent(&ReActConfig[*schema.Message]{
		Model: &mockModel{},
	}).WithName("parent")

	sub1 := NewReActAgent(&ReActConfig[*schema.Message]{
		Model: &mockModel{},
	}).WithName("sub1")

	ctx := t.Context()
	fa := toFlowAgent(ctx, agent)
	_, err := SetSubAgents(ctx, fa, []Agent{sub1})
	if err != nil {
		t.Fatal(err)
	}

	// SetSubAgents returns fa (which is the agent itself), but doesn't
	// populate fa.subAgents — it just validates that none are already set.
	if len(fa.subAgents) > 0 {
		t.Logf("subAgents populated: %d (unexpected but OK)", len(fa.subAgents))
	} else {
		t.Errorf("BUG: SetSubAgents does not populate flowAgent.subAgents. " +
			"The sub-agents are set on the returned ResumableAgent but not on " +
			"the original flowAgent. This causes inconsistency when flowAgent.Run() " +
			"tries to find sub-agents via getAgent().")
	}
}

// ---- Bug #16: runSeq ignores canceled context after drainEvents returns error ----
//
// workflow.go runSeq calls drainEvents, which sends error to gen but returns nil.
// The loop then checks shouldCancel() again but at that point the next sub-agent
// may already have started executing. This is a TOCTOU race.
func TestWorkflow_Sequential_CancelRaceInRunSeq(t *testing.T) {
	var execOrder []string
	var mu sync.Mutex

	agents := make([]Agent, 10)
	for i := 0; i < 10; i++ {
		i := i
		nodeID := fmt.Sprintf("race_node_%02d", i)
		tool := workflowNodeTool(nodeID, &execOrder, &mu)
		// Add small delay in tool to widen the race window
		delayedTool := &delayedTool{
			inner: tool.(*workflowNodeToolImpl),
			delay: time.Millisecond * time.Duration(5+i),
			order: &execOrder,
			mu:    &mu,
		}
		model := &forcedToolModel{
			toolCalls: []schema.ToolCall{{ID: fmt.Sprintf("c%d", i), Function: schema.ToolCallFunction{Name: delayedTool.Name(), Arguments: "{}"}}},
			finalResp: fmt.Sprintf("final from %s", nodeID),
			firstCall: true,
		}
		agents[i] = NewReActAgent(&ReActConfig[*schema.Message]{Model: model, Tools: []Tool{delayedTool}}).WithName(nodeID)
	}

	wf, err := NewSequential(t.Context(), &SequentialConfig{
		Name: "cancel_race", Description: "cancel race in runSeq",
		SubAgents: agents,
	})
	if err != nil {
		t.Fatal(err)
	}

	ctx := t.Context()
	opt, cancel := WithCancel()

	iter := wf.Run(ctx, &AgentInput{Messages: []Message{schema.UserMessage("race")}}, opt)

	// Cancel early — before most agents have run
	time.AfterFunc(time.Millisecond*10, func() {
		cancel()
	})

	for {
		_, ok := iter.Next()
		if !ok {
			break
		}
	}

	mu.Lock()
	count := len(execOrder)
	mu.Unlock()

	// Due to the TOCTOU race, we may have executed more nodes than expected.
	// The cancel check in runSeq happens BEFORE each sub-agent execution,
	// but the cancel itself is async. This is a design limitation.
	t.Logf("Cancel race: executed %d tools (expected ≤~5 due to async cancel delay)", count)
}

type delayedTool struct {
	inner *workflowNodeToolImpl
	delay time.Duration
	order *[]string
	mu    *sync.Mutex
}

func (t *delayedTool) Name() string        { return t.inner.Name() }
func (t *delayedTool) Description() string { return t.inner.Description() }
func (t *delayedTool) Invoke(ctx context.Context, args string, opts ...ToolOption) (string, error) {
	time.Sleep(t.delay)
	t.mu.Lock()
	*t.order = append(*t.order, t.Name())
	t.mu.Unlock()
	return fmt.Sprintf("%s executed", t.Name()), nil
}
func (t *delayedTool) Stream(ctx context.Context, args string, opts ...ToolOption) (*schema.StreamReader[string], error) {
	return schema.StreamReaderFromArray([]string{"stream: " + t.Name()}), nil
}