Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
feat(background): shared background-work runtime (#4579)
nullptr-errore28c68cf552c
diff --git a/internal/agent/background.go b/internal/agent/background.gonew file mode 100644index 000000000..ab9439bb4--- /dev/null+++ b/internal/agent/background.go@@ -0,0 +1,91 @@+package agent++import (+ "context"+ "strings"++ "github.com/Tencent/WeKnora/internal/background"+ "github.com/Tencent/WeKnora/internal/logger"+ "github.com/Tencent/WeKnora/internal/models/chat"+ "github.com/Tencent/WeKnora/internal/types"+)++// BackgroundRun is the run-scoped source of background-work context. The+// engine asks it what to tell the model at the two boundaries steer uses, and+// which finished items tools showed the model during a step.+// *background.RunNotices implements it.+type BackgroundRun interface {+ RoundStart(ctx context.Context) (background.Delivery, error)+ BeforeFinal(ctx context.Context) (background.Delivery, error)+ TakeSeen() []string+}++// SetBackgroundRun wires background-work context for this run. Nil (the+// default) disables it. Must be called before Execute.+func (e *AgentEngine) SetBackgroundRun(run BackgroundRun) {+ e.backgroundRun = run+}++// backgroundNoticesContent wraps background-work context so the model can tell+// it apart from what the user typed.+func backgroundNoticesContent(text string) string {+ return "<background_notices>\n" + strings.TrimSpace(text) + "\n</background_notices>"+}++// injectBackground appends background-work context before the next LLM call+// and queues it for the next step, which is where history replays it. It+// returns how many messages it appended (0 or 1). Failures are logged and+// skipped: the inbox keeps unacknowledged notices for a later boundary.+func (e *AgentEngine) injectBackground(+ ctx context.Context, state *types.AgentState, messagesPtr *[]chat.Message, beforeFinal bool,+) int {+ if e.backgroundRun == nil {+ return 0+ }+ read := e.backgroundRun.RoundStart+ if beforeFinal {+ read = e.backgroundRun.BeforeFinal+ }+ delivery, err := read(ctx)+ if err != nil {+ logger.Warnf(ctx, "[Agent][Round-%d] Background notices unavailable: %v", state.CurrentRound+1, err)+ return 0+ }+ if strings.TrimSpace(delivery.Text) == "" {+ return 0+ }+ content := backgroundNoticesContent(delivery.Text)+ *messagesPtr = append(*messagesPtr, chat.Message{Role: "user", Content: content})+ state.PendingBackground = append(state.PendingBackground, types.BackgroundInjection{+ Content: content, AckIDs: delivery.AckIDs,+ })+ return 1+}++// takeBackgroundSeen returns the finished IDs tools showed the model since the+// previous call.+func (e *AgentEngine) takeBackgroundSeen() []string {+ if e.backgroundRun == nil {+ return nil+ }+ return e.backgroundRun.TakeSeen()+}++// HasPendingInput reports whether a steered user message is waiting for the+// run that owns messageID. Blocking tools poll it so the user is not kept+// waiting behind them. It reads the queue without consuming it.+func (e *AgentEngine) HasPendingInput(ctx context.Context, sessionID, messageID string) bool {+ if e == nil || e.steerSink == nil {+ return false+ }+ events, _, err := e.steerSink.PollSteer(ctx, sessionID, messageID, 0)+ if err != nil {+ return false+ }+ for _, evt := range events {+ if sanitizeSteerContent(getString(evt, "content")) != "" {+ return true+ }+ }+ return false+}diff --git a/internal/agent/background_test.go b/internal/agent/background_test.gonew file mode 100644index 000000000..0bed85851--- /dev/null+++ b/internal/agent/background_test.go@@ -0,0 +1,171 @@+package agent++import (+ "context"+ "strings"+ "testing"+ "time"++ agenttools "github.com/Tencent/WeKnora/internal/agent/tools"+ "github.com/Tencent/WeKnora/internal/background"+ "github.com/Tencent/WeKnora/internal/event"+ "github.com/Tencent/WeKnora/internal/models/chat"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/stretchr/testify/require"+)++// fakeBackgroundRun hands out queued deliveries one call at a time.+type fakeBackgroundRun struct {+ roundStart []background.Delivery+ beforeFinal []background.Delivery+ seen [][]string+}++func (f *fakeBackgroundRun) RoundStart(context.Context) (background.Delivery, error) {+ if len(f.roundStart) == 0 {+ return background.Delivery{}, nil+ }+ d := f.roundStart[0]+ f.roundStart = f.roundStart[1:]+ return d, nil+}++func (f *fakeBackgroundRun) BeforeFinal(context.Context) (background.Delivery, error) {+ if len(f.beforeFinal) == 0 {+ return background.Delivery{}, nil+ }+ d := f.beforeFinal[0]+ f.beforeFinal = f.beforeFinal[1:]+ return d, nil+}++func (f *fakeBackgroundRun) TakeSeen() []string {+ if len(f.seen) == 0 {+ return nil+ }+ s := f.seen[0]+ f.seen = f.seen[1:]+ return s+}++func answer(content string) mockResponse {+ return mockResponse{chunks: []types.StreamResponse{+ {ResponseType: types.ResponseTypeAnswer, Content: content, Done: true, FinishReason: "stop"},+ }}+}++// userMessageContaining returns the first user message sent to the model that+// contains needle. Per-call envelopes may follow it, so position is not fixed.+func userMessageContaining(messages []chat.Message, needle string) string {+ for _, m := range messages {+ if m.Role == "user" && strings.Contains(m.Content, needle) {+ return m.Content+ }+ }+ return ""+}++func TestBackgroundNoticeIsInjectedBeforeTheRoundAndRecordedOnItsStep(t *testing.T) {+ model := &mockChat{responses: []mockResponse{answer("done")}}+ engine := newTestEngine(t, model)+ engine.SetBackgroundRun(&fakeBackgroundRun{roundStart: []background.Delivery{+ {Text: "build (sj_1) succeeded.", AckIDs: []string{"sj_1"}},+ }})++ state := &types.AgentState{}+ _, err := engine.executeLoop(context.Background(), state, "q", emptyMessages(), nil, "sess", "msg")+ require.NoError(t, err)++ want := "<background_notices>\nbuild (sj_1) succeeded.\n</background_notices>"+ require.Contains(t, userMessageContaining(model.calls[0], "<background_notices>"), want)+ require.Len(t, state.RoundSteps, 1)+ require.Equal(t, []types.BackgroundInjection{{Content: want, AckIDs: []string{"sj_1"}}},+ state.RoundSteps[0].BackgroundInjections)+ require.Empty(t, state.PendingBackground)+}++func TestBackgroundReminderKeepsTheLoopGoing(t *testing.T) {+ model := &mockChat{responses: []mockResponse{answer("first"), answer("second")}}+ engine := newTestEngine(t, model)+ engine.SetBackgroundRun(&fakeBackgroundRun{beforeFinal: []background.Delivery{+ {Text: "Background work you started in this turn is still running: build (sj_1)."},+ }})++ state := &types.AgentState{}+ _, err := engine.executeLoop(context.Background(), state, "q", emptyMessages(), nil, "sess", "msg")+ require.NoError(t, err)++ require.Len(t, model.calls, 2)+ require.NotEmpty(t, userMessageContaining(model.calls[1], "build (sj_1)"))+ require.Empty(t, userMessageContaining(model.calls[0], "build (sj_1)"), "the reminder comes only before finishing")+ require.Equal(t, "second", state.FinalAnswer)+ require.Len(t, state.RoundSteps, 2)+ require.True(t, state.RoundSteps[0].IntermediateAnswer)+ require.Len(t, state.RoundSteps[1].BackgroundInjections, 1)+}++func TestBackgroundSeenIDsRideOnTheToolStep(t *testing.T) {+ model := &mockChat{responses: []mockResponse{+ {chunks: []types.StreamResponse{{+ ResponseType: types.ResponseTypeAnswer,+ ToolCalls: []types.LLMToolCall{{+ ID: "call-1", Type: "function",+ Function: types.FunctionCall{Name: "probe", Arguments: "{}"},+ }},+ Done: true, FinishReason: "tool_calls",+ }}},+ answer("done"),+ }}+ engine := newTestEngine(t, model)+ engine.toolRegistry = agenttools.NewToolRegistry()+ engine.toolRegistry.RegisterTool(newCountingTool("probe"))+ engine.SetBackgroundRun(&fakeBackgroundRun{seen: [][]string{{"sj_9"}}})++ state := &types.AgentState{}+ _, err := engine.executeLoop(context.Background(), state, "q", emptyMessages(), nil, "sess", "msg")+ require.NoError(t, err)++ require.Len(t, state.RoundSteps, 2)+ require.Equal(t, []string{"sj_9"}, state.RoundSteps[0].BackgroundSeenIDs)+ require.Empty(t, state.RoundSteps[1].BackgroundSeenIDs)+}++func TestCompletionKeepsUndeliveredBackgroundOnATrailingStep(t *testing.T) {+ engine := newTestEngine(t, &mockChat{})+ engine.eventBus = event.NewEventBus()+ engine.SetBackgroundRun(&fakeBackgroundRun{seen: [][]string{{"sj_2"}}})+ var steps []types.AgentStep+ engine.eventBus.On(event.EventAgentComplete, func(_ context.Context, evt event.Event) error {+ steps, _ = evt.Data.(event.AgentCompleteData).AgentSteps.([]types.AgentStep)+ return nil+ })+ injection := types.BackgroundInjection{+ Content: "<background_notices>\nx\n</background_notices>",+ AckIDs: []string{"sj_1"},+ }+ state := &types.AgentState{PendingBackground: []types.BackgroundInjection{injection}}++ engine.emitCompletionEvent(context.Background(), state, "sess", "msg", time.Now())++ require.Len(t, steps, 1)+ require.Equal(t, []types.BackgroundInjection{injection}, steps[0].BackgroundInjections)+ require.Equal(t, []string{"sj_2"}, steps[0].BackgroundSeenIDs)+}++func TestHasPendingInputPeeksAtTheSteerQueue(t *testing.T) {+ engine := newTestEngine(t, &mockChat{})+ require.False(t, engine.HasPendingInput(context.Background(), "sess", "msg"), "no sink, nothing pending")++ sink := &fakeSteerSink{queued: []map[string]interface{}{steerEntry("s1", " ")}}+ engine.SetSteerSink(sink)+ require.False(t, engine.HasPendingInput(context.Background(), "sess", "msg"), "blank messages are dropped by drain")++ sink.queued = append(sink.queued, steerEntry("s2", "stop waiting"))+ require.True(t, engine.HasPendingInput(context.Background(), "sess", "msg"))+ require.Empty(t, sink.persisted, "peeking must not consume the message")+}++func TestBackgroundNoticesContentWrapsTrimmedText(t *testing.T) {+ require.Equal(t, "<background_notices>\na\nb\n</background_notices>", backgroundNoticesContent(" a\nb \n"))+ require.False(t, strings.Contains(backgroundNoticesContent("x"), "\n\n"))+}diff --git a/internal/agent/const.go b/internal/agent/const.goindex 64ffff2ad..881ee06f9 100644--- a/internal/agent/const.go+++ b/internal/agent/const.go@@ -12,6 +12,7 @@ import ("time""github.com/Tencent/WeKnora/internal/agent/compaction"+ agenttools "github.com/Tencent/WeKnora/internal/agent/tools""github.com/Tencent/WeKnora/internal/browserskill"webfetch "github.com/Tencent/WeKnora/internal/infrastructure/web_fetch""github.com/Tencent/WeKnora/internal/models/api"@@ -47,6 +48,9 @@ const (// shared page-fetch budget. Same 5s headroom as shell_exec, so web_fetch can// return its own timeout result instead of being cancelled first.webFetchToolSlack = 5 * time.Second+ // workWaitToolSlack is added to work_wait's own wait so the final detailed+ // read still fits after the wait elapses (D3).+ workWaitToolSlack = 30 * time.Second// maxLLMRetries is the maximum number of retries for transient LLM errors.maxLLMRetries = 2@@ -103,6 +107,13 @@ func toolExecutionTimeout(toolName string, arguments ...string) time.Duration {return browserskill.HumanStepTimeout}}+ if toolName == agenttools.ToolWorkWait {+ raw := ""+ if len(arguments) > 0 {+ raw = arguments[0]+ }+ return time.Duration(agenttools.WorkWaitSeconds(raw))*time.Second + workWaitToolSlack+ }switch toolName {case "shell_exec":return shellExecToolTimeoutdiff --git a/internal/agent/const_test.go b/internal/agent/const_test.goindex 51b20e43d..dc7a107c5 100644--- a/internal/agent/const_test.go+++ b/internal/agent/const_test.go@@ -55,3 +55,10 @@ func TestToolExecutionTimeout(t *testing.T) {assert.Equal(t, 35*time.Second, toolExecutionTimeout("web_fetch"))})}++func TestWorkWaitToolTimeoutCoversTheWait(t *testing.T) {+ assert.Equal(t, 90*time.Second, toolExecutionTimeout("work_wait", `{"work_ids":["a"]}`))+ assert.Equal(t, 30*time.Second, toolExecutionTimeout("work_wait", `{"wait_sec":0}`))+ assert.Equal(t, 330*time.Second, toolExecutionTimeout("work_wait", `{"wait_sec":999}`))+ assert.Equal(t, 90*time.Second, toolExecutionTimeout("work_wait"))+}diff --git a/internal/agent/engine.go b/internal/agent/engine.goindex cc433de76..2133fb85f 100644--- a/internal/agent/engine.go+++ b/internal/agent/engine.go@@ -88,7 +88,10 @@ type AgentEngine struct {steerSink types.SteerSinkallowSteerOverrun bool // one extra ReAct round after a loop-end inject past MaxIterationssteerOverruns int // how many times this turn has already used the extra round- workspaceLayout sandbox.WorkspaceLayout+ // backgroundRun, when set, supplies background-work notices and reminders+ // at the same two boundaries as steer. Nil disables them.+ backgroundRun BackgroundRun+ workspaceLayout sandbox.WorkspaceLayout// suspendable lets a user prompt end the run so it can be resumed later;// suspendRequest is saved with the run for that (see EnableSuspension).suspendable bool@@ -103,8 +106,9 @@ type AgentEngine struct {}// maxSteerOverruns caps loop-end injects past MaxIterations. One extra round-// lets a last-moment nudge revise the answer; further injects stay queued for-// a follow-up turn instead of stretching the same run indefinitely.+// lets a last-moment nudge (a steered message or a background reminder) revise+// the answer; further injects stay queued for a follow-up turn instead of+// stretching the same run indefinitely.const maxSteerOverruns = 1// ImageDescriberFunc generates a text description of an image.@@ -890,6 +894,9 @@ func (e *AgentEngine) runReActIteration(if guards.childAnswerRevised {tools = nil}+ // Background-work context rides the same boundary for the same reasons,+ // after the user's own messages.+ e.injectBackground(ctx, state, messagesPtr, false)logger.Infof(ctx, "[Agent][Round-%d/%s] Starting: %d messages, %d tools, est_tokens=%d",round, e.maxIterationsDisplay(), len(*messagesPtr), len(tools), currentTokens)@@ -1040,16 +1047,18 @@ func (e *AgentEngine) runReActIteration(// Create agent stepstep := types.AgentStep{- UserMessagesBefore: state.PendingSteerMessages,- Iteration: state.CurrentRound,- Thought: response.Content,- ReasoningContent: response.ReasoningContent,- ReasoningSignature: response.ReasoningSignature,- ReasoningMetadata: response.ReasoningMetadata,- ToolCalls: make([]types.ToolCall, 0),- Timestamp: time.Now(),+ UserMessagesBefore: state.PendingSteerMessages,+ BackgroundInjections: state.PendingBackground,+ Iteration: state.CurrentRound,+ Thought: response.Content,+ ReasoningContent: response.ReasoningContent,+ ReasoningSignature: response.ReasoningSignature,+ ReasoningMetadata: response.ReasoningMetadata,+ ToolCalls: make([]types.ToolCall, 0),+ Timestamp: time.Now(),}state.PendingSteerMessages = nil+ state.PendingBackground = nil// If the request was cancelled while the LLM was streaming (e.g. the// user pressed "stop"), the stream driver still returns a usable@@ -1064,7 +1073,8 @@ func (e *AgentEngine) runReActIteration(if ctx.Err() != nil {logger.Warnf(ctx, "[Agent][Round-%d] Context cancelled during LLM call; preserving partial step",round)- if step.Thought != "" || len(step.ToolCalls) > 0 || len(step.UserMessagesBefore) > 0 {+ if step.Thought != "" || len(step.ToolCalls) > 0 || len(step.UserMessagesBefore) > 0 ||+ len(step.BackgroundInjections) > 0 {state.RoundSteps = append(state.RoundSteps, step)}return iterOutcomeBreak, nil@@ -1132,6 +1142,7 @@ func (e *AgentEngine) runReActIteration(guards.emptyRetries++if guards.emptyRetries <= maxEmptyResponseRetries {state.PendingSteerMessages = step.UserMessagesBefore+ state.PendingBackground = step.BackgroundInjectionslogger.Warnf(ctx, "[Agent][Round-%d] Empty content with stop - retrying (%d/%d)",round, guards.emptyRetries, maxEmptyResponseRetries)guards.pendingRuntimeNotice = emptyResponseRetryNotice@@ -1176,7 +1187,8 @@ func (e *AgentEngine) runReActIteration(return iterOutcomeBreak, nil}// Loop-end inject: a user message queued while this finishing round- // ran should keep the agent going instead of emitting a final answer.+ // ran, or background work this run started that is still running,+ // keeps the agent going instead of emitting a final answer.// Content-filter stops are terminal and do not take this path.// Past MaxIterations only the first inject gets an extra round;// anything after that stays in the queue for a follow-up turn.@@ -1192,6 +1204,7 @@ func (e *AgentEngine) runReActIteration(ReasoningMetadata: response.ReasoningMetadata,})injected := e.drainSteerMessages(ctx, state, messagesPtr, sessionID, assistantMessageID)+ injected += e.injectBackground(ctx, state, messagesPtr, true)if injected > 0 {*messagesPtr = e.appendCollaborationDeliveryBoundary(*messagesPtr, verdict.finalAnswer)verdict.step.IntermediateAnswer = true@@ -1339,6 +1352,7 @@ func (e *AgentEngine) runRoundTools(e.beginRound(*step, response.ToolCalls)suspended := e.executeToolCalls(ctx, response, step, state.CurrentRound, sessionID, assistantMessageID)e.endRound()+ step.BackgroundSeenIDs = e.takeBackgroundSeen()if suspended == nil {return false}diff --git a/internal/agent/finalize.go b/internal/agent/finalize.goindex 1826ce010..feb963308 100644--- a/internal/agent/finalize.go+++ b/internal/agent/finalize.go@@ -181,11 +181,15 @@ func (e *AgentEngine) emitCompletionEvent(if state.SuspendedStep != nil {steps = append(append([]types.AgentStep(nil), steps...), *state.SuspendedStep)}- if len(state.PendingSteerMessages) > 0 {+ seen := e.takeBackgroundSeen()+ if len(state.PendingSteerMessages) > 0 || len(state.PendingBackground) > 0 || len(seen) > 0 {// A stop or model failure can arrive after delivery but before the next// response exists. Preserve that boundary without inventing an answer.steps = append(append([]types.AgentStep(nil), steps...), types.AgentStep{- Iteration: state.CurrentRound, UserMessagesBefore: state.PendingSteerMessages,+ Iteration: state.CurrentRound,+ UserMessagesBefore: state.PendingSteerMessages,+ BackgroundInjections: state.PendingBackground,+ BackgroundSeenIDs: seen,})}// Convert knowledge refs to interface{} slice for event datadiff --git a/internal/agent/tools/definitions.go b/internal/agent/tools/definitions.goindex d1ecabee8..fbcf4fdb8 100644--- a/internal/agent/tools/definitions.go+++ b/internal/agent/tools/definitions.go@@ -69,6 +69,14 @@ const (// keys it on SkillsEnabled (or install mode), so the shell follows the// skills switch and not a per-agent tool checkbox.ToolShellExec = "shell_exec"+ // ToolShellExecAsync submits a shell command as background work (plan 4).+ // Like shell_exec it is registered per run, not in AvailableToolDefinitions.+ ToolShellExecAsync = "shell_exec_async"+ // ToolWorkWait / ToolWorkCancel wait on and stop background work. They are+ // registered per run, only when the run can submit background work, so+ // they are absent from AvailableToolDefinitions like shell_exec.+ ToolWorkWait = "work_wait"+ ToolWorkCancel = "work_cancel"// Wiki-related tools (only available when wiki KBs are in scope)ToolWikiReadPage = "wiki_read_page"ToolWikiWritePage = "wiki_write_page"@@ -177,6 +185,7 @@ func SwitchManagedTools() []string {ToolWebFetch,ToolSearchMemory,ToolShellExec,+ ToolShellExecAsync,ToolReadFile,ToolListSandboxFiles,ToolWriteSandboxFile,diff --git a/internal/agent/tools/tool_policy_coverage_test.go b/internal/agent/tools/tool_policy_coverage_test.goindex 4e99fadc0..22bec5c13 100644--- a/internal/agent/tools/tool_policy_coverage_test.go+++ b/internal/agent/tools/tool_policy_coverage_test.go@@ -36,6 +36,9 @@ func TestEveryBuiltInToolDeclaresAModelHandlePolicy(t *testing.T) {ToolWriteSkillFile,ToolEditSkillFile,ToolShellExec,+ ToolShellExecAsync,+ ToolWorkWait,+ ToolWorkCancel,ToolWikiReadPage,ToolWikiWritePage,ToolWikiReplaceText,diff --git a/internal/agent/tools/work.go b/internal/agent/tools/work.gonew file mode 100644index 000000000..bfb21c1f8--- /dev/null+++ b/internal/agent/tools/work.go@@ -0,0 +1,511 @@+package tools++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "strings"+ "sync"+ "time"++ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/Tencent/WeKnora/internal/utils"+)++const (+ workWaitDefaultSec = 60+ workWaitMaxSec = 300+ workWaitPoll = time.Second+ // workMaxIDs bounds one call so each ID's share of the output budget stays useful.+ workMaxIDs = 16+ // workOutputBudget caps the stdout and stderr bodies of one work_wait call (D6).+ workOutputBudget = 32 * 1024+ // workWaitOutputLimit is the rune cap ToolRegistry applies to the marshaled+ // work_wait body. JSON escaping can turn one payload byte into \u00XX, and+ // the envelope adds ids, states, and the note. The generic 24000-rune cap+ // would cut a full result into invalid JSON.+ workWaitOutputLimit = workOutputBudget*6 + 8*1024+ // workProbeBudget asks a state-only probe for as little output as possible.+ workProbeBudget = 1+)++const workWaitDescription = `Wait for background work only when you cannot continue without its result.+work_ids come from %s. Do not call it merely because work is still running; you will+be told when it finishes. wait_sec: seconds to wait (default 60, max 300); 0 returns+the current state and new output immediately. mode: all (default) or any. Returns+early if the user sends a message.`++const workCancelDescription = `Stop background work by work_id.`++const (+ workNoteRunning = "Still running; you will be told when it finishes."+ workNoteInterrupted = "The user sent a new message; it is shown to you next. The work keeps running."+ workNoteCancelling = "Cancellation requested; you will be told when it stops."+)++// backgroundSubmissionTools are the tools that start background work. A run+// gets work_wait and work_cancel only when it has at least one of them.+var backgroundSubmissionTools []string++// RegisteredBackgroundSubmitters returns the background submission tools+// registered in r, in declaration order.+func RegisteredBackgroundSubmitters(r *ToolRegistry) []string {+ var out []string+ for _, name := range backgroundSubmissionTools {+ if _, err := r.GetTool(name); err == nil {+ out = append(out, name)+ }+ }+ return out+}++// WorkMarks is the run-scoped tracker the work tools report to.+// *background.RunNotices implements it.+type WorkMarks interface {+ MarkSeen(ids ...string)+}++// WorkWaitInput documents the work_wait arguments for the schema.+type WorkWaitInput struct {+ WorkIDs []string `json:"work_ids" jsonschema:"IDs of the background work to wait for"`+ WaitSec int `json:"wait_sec,omitempty" jsonschema:"Seconds to wait, default 60, max 300; 0 returns at once"`+ Mode string `json:"mode,omitempty" jsonschema:"all (default) waits for every ID; any returns when one ends"`+}++// WorkCancelInput documents the work_cancel arguments for the schema.+type WorkCancelInput struct {+ WorkIDs []string `json:"work_ids" jsonschema:"IDs of the background work to stop"`+}++type workWaitArgs struct {+ WorkIDs []string `json:"work_ids"`+ WaitSec *int `json:"wait_sec"`+ Mode string `json:"mode"`+}++// WorkWaitSeconds returns the wait of a work_wait call: default 60, clamped to+// 0–300. Malformed arguments get the default. The engine derives the tool+// timeout from it (D3).+func WorkWaitSeconds(arguments string) int {+ var args struct {+ WaitSec *int `json:"wait_sec"`+ }+ if json.Unmarshal([]byte(arguments), &args) != nil || args.WaitSec == nil {+ return workWaitDefaultSec+ }+ return min(max(*args.WaitSec, 0), workWaitMaxSec)+}++type workItemView struct {+ WorkID string `json:"work_id"`+ State string `json:"state"`+ Title string `json:"title,omitempty"`+ Result any `json:"result,omitempty"`+}++type workView struct {+ Items []workItemView `json:"items"`+ TimedOut bool `json:"timed_out,omitempty"`+ InterruptedByInput bool `json:"interrupted_by_input,omitempty"`+ Note string `json:"note,omitempty"`+}++// WorkWaitTool waits on background work of the run's session.+type WorkWaitTool struct {+ BaseTool+ handler interfaces.WorkHandler+ session interfaces.SessionRef+ runID string+ marks WorkMarks+ pending func(context.Context) bool+ now func() time.Time+ sleep func(context.Context, time.Duration) bool++ // waitedRunning records IDs that already had a blocking wait and were still+ // running. A later blocking wait in this run does not sleep on them again.+ mu sync.Mutex+ waitedRunning map[string]struct{}+}++// NewWorkWaitTool returns work_wait bound to one run. runID scopes incremental+// output reads; pending reports a waiting user message; submitters are named+// in the description.+func NewWorkWaitTool(+ handler interfaces.WorkHandler, session interfaces.SessionRef, runID string,+ marks WorkMarks, pending func(context.Context) bool, submitters []string,+) *WorkWaitTool {+ description := fmt.Sprintf(workWaitDescription, strings.Join(submitters, ", "))+ return &WorkWaitTool{+ BaseTool: NewBaseTool(ToolWorkWait, description, utils.GenerateSchema[WorkWaitInput]()),+ handler: handler,+ session: session,+ runID: runID,+ marks: marks,+ pending: pending,+ now: time.Now,+ sleep: workSleep,+ }+}++// Execute implements types.Tool.+func (t *WorkWaitTool) Execute(ctx context.Context, raw json.RawMessage) (*types.ToolResult, error) {+ var args workWaitArgs+ if err := json.Unmarshal(raw, &args); err != nil {+ return workError("invalid arguments: " + err.Error()), nil+ }+ ids, err := normalizeWorkIDs(args.WorkIDs)+ if err != nil {+ return workError(err.Error()), nil+ }+ mode := strings.ToLower(strings.TrimSpace(args.Mode))+ if mode == "" {+ mode = "all"+ }+ if mode != "all" && mode != "any" {+ return workError(`mode must be "all" or "any"`), nil+ }+ waitSec := WorkWaitSeconds(string(raw))++ timedOut, interrupted := false, false+ if waitSec > 0 {+ fresh := t.unwaited(ids)+ if len(fresh) > 0 {+ var err error+ timedOut, interrupted, err = t.poll(ctx, fresh, mode, time.Duration(waitSec)*time.Second)+ if err != nil {+ return nil, err+ }+ }+ }+ results, err := t.status(ctx, ids, interfaces.StatusOptions{+ Detailed: true, RunID: t.runID, OutputBudget: workOutputBudget / len(ids),+ })+ if err != nil {+ return workReadError(ids, err)+ }+ if waitSec > 0 {+ t.rememberRunning(results)+ }+ markTerminal(t.marks, results)++ note := ""+ switch {+ case interrupted && anyRunning(results):+ note = workNoteInterrupted+ case anyRunning(results):+ note = workNoteRunning+ }+ rendered := renderWork(results, note, timedOut && anyRunning(results), interrupted)+ rendered.OutputFiles = collectOutputFiles(results)+ return rendered, nil+}++func collectOutputFiles(results []interfaces.WorkResult) []string {+ inspected := false+ var links []string+ for _, result := range results {+ if result.OutputFiles == nil {+ continue+ }+ inspected = true+ links = append(links, result.OutputFiles...)+ }+ if !inspected {+ return nil+ }+ if links == nil {+ return []string{}+ }+ return links+}++// OutputLimitChars keeps a full work_wait body, including JSON framing and+// escaping, inside the registry's truncation ceiling.+func (t *WorkWaitTool) OutputLimitChars(json.RawMessage) int {+ return workWaitOutputLimit+}++// poll waits with cheap reads until mode is satisfied, the user sends a+// message, or wait elapses.+func (t *WorkWaitTool) poll(+ ctx context.Context, ids []string, mode string, wait time.Duration,+) (timedOut, interrupted bool, err error) {+ deadline := t.now().Add(wait)+ probed := make(map[string]bool, len(ids))+ for {+ results, callErr := t.handler.Status(ctx, t.session, ids, interfaces.StatusOptions{RunID: t.runID})+ if workStopped(callErr) {+ return false, false, callErr+ }+ if errors.Is(callErr, interfaces.ErrWorkNotFound) {+ return false, false, nil+ }+ // A failed cheap read is not "everything finished". Keep waiting; the+ // final detailed read is what the model sees.+ if callErr == nil {+ results, callErr = checkWorkResults(ids, results, nil)+ if callErr != nil {+ return false, false, callErr+ }+ var unknown []string+ for _, r := range results {+ if r.State == interfaces.WorkStateUnknown && !probed[r.ID] {+ probed[r.ID] = true+ unknown = append(unknown, r.ID)+ }+ }+ if len(unknown) > 0 {+ // No RunID: a probe must not move the read position the final+ // detailed read hands to the model.+ probe, probeErr := t.handler.Status(ctx, t.session, unknown, interfaces.StatusOptions{+ Detailed: true, OutputBudget: workProbeBudget,+ })+ if workStopped(probeErr) {+ return false, false, probeErr+ }+ if errors.Is(probeErr, interfaces.ErrWorkNotFound) {+ return false, false, nil+ }+ if probeErr == nil {+ probe, probeErr = checkWorkResults(unknown, probe, nil)+ if probeErr != nil {+ return false, false, probeErr+ }+ byID := make(map[string]interfaces.WorkResult, len(probe))+ for _, r := range probe {+ byID[r.ID] = r+ }+ for i, r := range results {+ if p, ok := byID[r.ID]; ok {+ results[i] = p+ }+ }+ }+ }+ if workSatisfied(results, mode) {+ return false, false, nil+ }+ }+ if t.pending != nil && t.pending(ctx) {+ return false, true, nil+ }+ if !t.now().Before(deadline) {+ return true, false, nil+ }+ if !t.sleep(ctx, workWaitPoll) {+ return false, false, ctx.Err()+ }+ }+}++func (t *WorkWaitTool) unwaited(ids []string) []string {+ t.mu.Lock()+ defer t.mu.Unlock()+ fresh := make([]string, 0, len(ids))+ for _, id := range ids {+ if _, waited := t.waitedRunning[id]; !waited {+ fresh = append(fresh, id)+ }+ }+ return fresh+}++func (t *WorkWaitTool) rememberRunning(results []interfaces.WorkResult) {+ t.mu.Lock()+ defer t.mu.Unlock()+ if t.waitedRunning == nil {+ t.waitedRunning = map[string]struct{}{}+ }+ for _, result := range results {+ if !result.Terminal && result.ID != "" {+ t.waitedRunning[result.ID] = struct{}{}+ }+ }+}++func (t *WorkWaitTool) status(+ ctx context.Context, ids []string, opts interfaces.StatusOptions,+) ([]interfaces.WorkResult, error) {+ results, err := t.handler.Status(ctx, t.session, ids, opts)+ return checkWorkResults(ids, results, err)+}++// workStopped reports a cancelled or timed-out call. Those are not a lost+// batch: Execute returns the error, and the IDs are not marked seen.+func workStopped(err error) bool {+ return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded)+}++// WorkCancelTool stops background work of the run's session.+type WorkCancelTool struct {+ BaseTool+ handler interfaces.WorkHandler+ session interfaces.SessionRef+ marks WorkMarks+}++// NewWorkCancelTool returns work_cancel bound to one run.+func NewWorkCancelTool(handler interfaces.WorkHandler, session interfaces.SessionRef, marks WorkMarks) *WorkCancelTool {+ return &WorkCancelTool{+ BaseTool: NewBaseTool(ToolWorkCancel, workCancelDescription, utils.GenerateSchema[WorkCancelInput]()),+ handler: handler,+ session: session,+ marks: marks,+ }+}++// Execute implements types.Tool.+func (t *WorkCancelTool) Execute(ctx context.Context, raw json.RawMessage) (*types.ToolResult, error) {+ var args WorkCancelInput+ if err := json.Unmarshal(raw, &args); err != nil {+ return workError("invalid arguments: " + err.Error()), nil+ }+ ids, err := normalizeWorkIDs(args.WorkIDs)+ if err != nil {+ return workError(err.Error()), nil+ }+ results, err := t.handler.Cancel(ctx, t.session, ids)+ results, err = checkWorkResults(ids, results, err)+ if err != nil {+ return workReadError(ids, err)+ }+ markTerminal(t.marks, results)+ note := ""+ if anyRunning(results) {+ note = workNoteCancelling+ }+ return renderWork(results, note, false, false), nil+}++// checkWorkResults enforces the WorkHandler contract of one result per ID in+// input order. Results are matched by position, so a reordered reply would+// hand one item's state to another; it fails the batch instead (D14).+func checkWorkResults(ids []string, results []interfaces.WorkResult, err error) ([]interfaces.WorkResult, error) {+ if err == nil && len(results) != len(ids) {+ err = fmt.Errorf("handler returned %d results for %d ids", len(results), len(ids))+ }+ for i := 0; err == nil && i < len(ids); i++ {+ if results[i].ID != ids[i] {+ err = fmt.Errorf("handler returned id %q at position %d, want %q", results[i].ID, i, ids[i])+ }+ }+ if err != nil {+ return nil, err+ }+ for i := range results {+ results[i].Terminal = interfaces.IsTerminalWorkState(results[i].State)+ }+ return results, nil+}++// workReadError never acknowledges IDs from a failed batch. Even+// ErrWorkNotFound may refer to only one of several requested IDs; it does not+// confirm that the other jobs ended or that their results reached the model.+func workReadError(ids []string, err error) (*types.ToolResult, error) {+ if !errors.Is(err, interfaces.ErrWorkNotFound) {+ return nil, err+ }+ if len(ids) > 1 {+ return workError("One or more work IDs were not found in this session; check them individually."), nil+ }+ results := make([]interfaces.WorkResult, len(ids))+ for i, id := range ids {+ results[i] = interfaces.WorkResult{+ ID: id, State: interfaces.WorkStateLost, Terminal: true,+ Payload: map[string]any{"error": "work not found in this session"},+ }+ }+ return renderWork(results, "", false, false), nil+}++func normalizeWorkIDs(raw []string) ([]string, error) {+ ids := make([]string, 0, len(raw))+ seen := make(map[string]bool, len(raw))+ for _, id := range raw {+ id = strings.TrimSpace(id)+ if id == "" || seen[id] {+ continue+ }+ seen[id] = true+ ids = append(ids, id)+ }+ if len(ids) == 0 {+ return nil, errors.New("work_ids must name at least one work_id")+ }+ if len(ids) > workMaxIDs {+ return nil, fmt.Errorf("work_ids may name at most %d work_ids", workMaxIDs)+ }+ return ids, nil+}++func workSatisfied(results []interfaces.WorkResult, mode string) bool {+ finished := 0+ for _, r := range results {+ if r.Terminal {+ finished+++ }+ }+ if mode == "any" {+ return finished > 0+ }+ return finished == len(results)+}++func anyRunning(results []interfaces.WorkResult) bool {+ for _, r := range results {+ if !r.Terminal {+ return true+ }+ }+ return false+}++func markTerminal(marks WorkMarks, results []interfaces.WorkResult) {+ if marks == nil {+ return+ }+ var ids []string+ for _, r := range results {+ if r.Terminal {+ ids = append(ids, r.ID)+ }+ }+ if len(ids) > 0 {+ marks.MarkSeen(ids...)+ }+}++func renderWork(results []interfaces.WorkResult, note string, timedOut, interrupted bool) *types.ToolResult {+ view := workView{+ Items: make([]workItemView, 0, len(results)),+ TimedOut: timedOut,+ InterruptedByInput: interrupted,+ Note: note,+ }+ for _, r := range results {+ view.Items = append(view.Items, workItemView{WorkID: r.ID, State: r.State, Title: r.Title, Result: r.Payload})+ }+ raw, err := json.Marshal(view)+ if err != nil {+ return workError("encode work results: " + err.Error())+ }+ return &types.ToolResult{Success: true, Output: string(raw), Data: map[string]interface{}{"work": view}}+}++func workError(msg string) *types.ToolResult {+ return &types.ToolResult{Success: false, Error: msg}+}++// workSleep waits d, returning false if ctx ends first.+func workSleep(ctx context.Context, d time.Duration) bool {+ timer := time.NewTimer(d)+ defer timer.Stop()+ select {+ case <-ctx.Done():+ return false+ case <-timer.C:+ return true+ }+}diff --git a/internal/agent/tools/work_test.go b/internal/agent/tools/work_test.gonew file mode 100644index 000000000..c376b073a--- /dev/null+++ b/internal/agent/tools/work_test.go@@ -0,0 +1,436 @@+package tools++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "sync"+ "testing"+ "time"++ "github.com/Tencent/WeKnora/internal/background"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/stretchr/testify/require"+)++var workSession = interfaces.SessionRef{TenantID: 1, SessionID: "s"}++// fakeWorkHandler serves a scripted sequence of cheap states per ID; the last+// entry repeats. Detailed reads return detailed[id] (default "running").+type fakeWorkHandler struct {+ mu sync.Mutex+ cheap map[string][]string+ detailed map[string]string+ calls []interfaces.StatusOptions+ err error+ detailedErr error+ reverse bool+ omitLast bool+ cancelled [][]string+}++func (h *fakeWorkHandler) Status(+ ctx context.Context, _ interfaces.SessionRef, ids []string, opts interfaces.StatusOptions,+) ([]interfaces.WorkResult, error) {+ h.mu.Lock()+ defer h.mu.Unlock()+ h.calls = append(h.calls, opts)+ if err := ctx.Err(); err != nil {+ return nil, err+ }+ if h.err != nil {+ return nil, h.err+ }+ if opts.Detailed && h.detailedErr != nil {+ return nil, h.detailedErr+ }+ out := make([]interfaces.WorkResult, 0, len(ids))+ for _, id := range ids {+ state := interfaces.WorkStateRunning+ if opts.Detailed {+ if s, ok := h.detailed[id]; ok {+ state = s+ }+ } else if seq := h.cheap[id]; len(seq) > 0 {+ state = seq[0]+ if len(seq) > 1 {+ h.cheap[id] = seq[1:]+ }+ }+ out = append(out, interfaces.WorkResult{+ ID: id, State: state, Title: "job " + id,+ Payload: map[string]any{"stdout": "out-" + id},+ })+ }+ if h.reverse {+ for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 {+ out[i], out[j] = out[j], out[i]+ }+ }+ if h.omitLast && len(out) > 0 {+ out = out[:len(out)-1]+ }+ return out, nil+}++func (h *fakeWorkHandler) Cancel(+ ctx context.Context, _ interfaces.SessionRef, ids []string,+) ([]interfaces.WorkResult, error) {+ if err := ctx.Err(); err != nil {+ return nil, err+ }+ if h.err != nil {+ return nil, h.err+ }+ h.cancelled = append(h.cancelled, ids)+ out := make([]interfaces.WorkResult, 0, len(ids))+ for _, id := range ids {+ state := interfaces.WorkStateRunning+ if id == "sj_done" {+ state = interfaces.WorkStateSucceeded+ }+ out = append(out, interfaces.WorkResult{ID: id, State: state})+ }+ return out, nil+}++type recordingMarks struct {+ seen []string+ started []string+}++func (m *recordingMarks) MarkSeen(ids ...string) { m.seen = append(m.seen, ids...) }++func (m *recordingMarks) MarkStarted(id string) { m.started = append(m.started, id) }++// newTestWorkWait returns a tool on a fake clock: sleeping advances time.+func newTestWorkWait(h *fakeWorkHandler, marks WorkMarks, pending func(context.Context) bool) (*WorkWaitTool, *int) {+ tool := NewWorkWaitTool(h, workSession, "run-1", marks, pending, []string{"shell_exec_async"})+ now := time.Date(2026, 10, 15, 10, 0, 0, 0, time.UTC)+ sleeps := 0+ tool.now = func() time.Time { return now }+ tool.sleep = func(_ context.Context, d time.Duration) bool {+ sleeps+++ now = now.Add(d)+ return true+ }+ return tool, &sleeps+}++func decodeWork(t *testing.T, output string) workView {+ t.Helper()+ var view workView+ require.NoError(t, json.Unmarshal([]byte(output), &view))+ return view+}++func TestWorkWaitZeroReadsOnceInDetail(t *testing.T) {+ h := &fakeWorkHandler{detailed: map[string]string{"sj_1": interfaces.WorkStateSucceeded}}+ marks := &recordingMarks{}+ tool, sleeps := newTestWorkWait(h, marks, nil)++ res, err := tool.Execute(context.Background(), json.RawMessage(`{"work_ids":["sj_1"],"wait_sec":0}`))++ require.NoError(t, err)+ require.True(t, res.Success)+ require.Zero(t, *sleeps)+ require.Equal(t, []interfaces.StatusOptions{{+ Detailed: true, RunID: "run-1", OutputBudget: workOutputBudget,+ }}, h.calls)+ view := decodeWork(t, res.Output)+ require.Equal(t, interfaces.WorkStateSucceeded, view.Items[0].State)+ require.Empty(t, view.Note)+ require.Equal(t, []string{"sj_1"}, marks.seen)+}++func TestWorkWaitAllWaitsForEveryID(t *testing.T) {+ h := &fakeWorkHandler{+ cheap: map[string][]string{+ "sj_1": {interfaces.WorkStateRunning, interfaces.WorkStateSucceeded},+ "sj_2": {interfaces.WorkStateRunning, interfaces.WorkStateRunning, interfaces.WorkStateFailed},+ },+ detailed: map[string]string{"sj_1": interfaces.WorkStateSucceeded, "sj_2": interfaces.WorkStateFailed},+ }+ marks := &recordingMarks{}+ tool, sleeps := newTestWorkWait(h, marks, nil)++ res, err := tool.Execute(context.Background(), json.RawMessage(`{"work_ids":["sj_1","sj_2"],"wait_sec":30}`))++ require.NoError(t, err)+ require.Equal(t, 2, *sleeps)+ view := decodeWork(t, res.Output)+ require.False(t, view.TimedOut)+ require.Equal(t, []string{"sj_1", "sj_2"}, marks.seen)+ final := h.calls[len(h.calls)-1]+ require.Equal(t, interfaces.StatusOptions{+ Detailed: true, RunID: "run-1", OutputBudget: workOutputBudget / 2,+ }, final)+}++func TestWorkWaitAnyReturnsOnTheFirstFinish(t *testing.T) {+ h := &fakeWorkHandler{+ cheap: map[string][]string{+ "sj_1": {interfaces.WorkStateRunning},+ "sj_2": {interfaces.WorkStateRunning, interfaces.WorkStateSucceeded},+ },+ detailed: map[string]string{"sj_2": interfaces.WorkStateSucceeded},+ }+ tool, sleeps := newTestWorkWait(h, &recordingMarks{}, nil)++ res, err := tool.Execute(context.Background(),+ json.RawMessage(`{"work_ids":["sj_1","sj_2"],"wait_sec":30,"mode":"any"}`))++ require.NoError(t, err)+ require.Equal(t, 1, *sleeps)+ view := decodeWork(t, res.Output)+ require.Equal(t, "Still running; you will be told when it finishes.", view.Note)+}++func TestWorkWaitDoesNotBlockAgainForTheSameRunningID(t *testing.T) {+ h := &fakeWorkHandler{}+ tool, sleeps := newTestWorkWait(h, &recordingMarks{}, nil)++ first, err := tool.Execute(context.Background(), json.RawMessage(`{"work_ids":["sj_1"],"wait_sec":3}`))+ require.NoError(t, err)+ require.Equal(t, 3, *sleeps)+ require.Equal(t, workNoteRunning, decodeWork(t, first.Output).Note)++ second, err := tool.Execute(context.Background(), json.RawMessage(`{"work_ids":["sj_1"],"wait_sec":60}`))+ require.NoError(t, err)+ require.Equal(t, 3, *sleeps, "a second blocking wait must not sleep")+ view := decodeWork(t, second.Output)+ require.False(t, view.TimedOut)+ require.Equal(t, workNoteRunning, view.Note)+ require.Equal(t, interfaces.WorkStateRunning, view.Items[0].State)+}++func TestWorkWaitTimesOut(t *testing.T) {+ h := &fakeWorkHandler{}+ marks := &recordingMarks{}+ tool, sleeps := newTestWorkWait(h, marks, nil)++ res, err := tool.Execute(context.Background(), json.RawMessage(`{"work_ids":["sj_1"],"wait_sec":3}`))++ require.NoError(t, err)+ require.Equal(t, 3, *sleeps)+ view := decodeWork(t, res.Output)+ require.True(t, view.TimedOut)+ require.Equal(t, "Still running; you will be told when it finishes.", view.Note)+ require.Empty(t, marks.seen)+}++func TestWorkWaitReturnsEarlyWhenTheUserTypes(t *testing.T) {+ h := &fakeWorkHandler{}+ tool, sleeps := newTestWorkWait(h, &recordingMarks{}, func(context.Context) bool { return true })++ res, err := tool.Execute(context.Background(), json.RawMessage(`{"work_ids":["sj_1"],"wait_sec":300}`))++ require.NoError(t, err)+ require.Zero(t, *sleeps)+ view := decodeWork(t, res.Output)+ require.True(t, view.InterruptedByInput)+ require.Equal(t, "The user sent a new message; it is shown to you next. The work keeps running.", view.Note)+}++func TestWorkWaitProbesUnknownOnceWithoutMovingTheReadPosition(t *testing.T) {+ h := &fakeWorkHandler{+ cheap: map[string][]string{"sj_1": {interfaces.WorkStateUnknown}},+ detailed: map[string]string{"sj_1": interfaces.WorkStateSucceeded},+ }+ tool, _ := newTestWorkWait(h, &recordingMarks{}, nil)++ _, err := tool.Execute(context.Background(), json.RawMessage(`{"work_ids":["sj_1"],"wait_sec":30}`))++ require.NoError(t, err)+ require.Equal(t, []interfaces.StatusOptions{+ {RunID: "run-1"},+ {Detailed: true, OutputBudget: workProbeBudget},+ {Detailed: true, RunID: "run-1", OutputBudget: workOutputBudget},+ }, h.calls)+}++func TestWorkWaitFailedReadsDoNotAcknowledgeNotices(t *testing.T) {+ for name, h := range map[string]*fakeWorkHandler{+ "error": {err: errors.New("sandbox unreachable")},+ "not found": {err: fmt.Errorf("wrap: %w", interfaces.ErrWorkNotFound)},+ "reordered": {reverse: true},+ "incomplete": {omitLast: true},+ } {+ t.Run(name, func(t *testing.T) {+ ctx := context.Background()+ inbox := background.NewMemoryNoticeInbox(nil)+ marks := background.NewRunNotices(workSession, background.NewMemoryActivityRegistry(nil), inbox)+ tool, _ := newTestWorkWait(h, marks, nil)++ res, err := tool.Execute(ctx, json.RawMessage(`{"work_ids":["sj_1","sj_2"],"wait_sec":0}`))++ if errors.Is(h.err, interfaces.ErrWorkNotFound) {+ require.NoError(t, err)+ require.False(t, res.Success)+ require.Empty(t, res.Output, "a missing ID must not turn unrelated IDs into lost jobs")+ } else {+ require.Error(t, err)+ require.Nil(t, res)+ }+ require.Empty(t, marks.TakeSeen(), "a failed batch must not acknowledge any requested ID")+ _, err = inbox.Push(ctx, workSession, interfaces.BackgroundNotice{+ ID: "sj_1", Text: "build succeeded", FinishedAt: time.Now(),+ })+ require.NoError(t, err)+ delivery, err := marks.RoundStart(ctx)+ require.NoError(t, err)+ require.Equal(t, "build succeeded", delivery.Text, "the later real completion must still reach the model")+ })+ }+}++func TestWorkWaitMissingWorkReturnsWithoutSleeping(t *testing.T) {+ for _, mode := range []string{"all", "any"} {+ for _, source := range []string{"cheap", "probe"} {+ t.Run(mode+"/"+source, func(t *testing.T) {+ missing := fmt.Errorf("status: %w", interfaces.ErrWorkNotFound)+ h := &fakeWorkHandler{err: missing}+ wantCalls := 2+ if source == "probe" {+ h = &fakeWorkHandler{+ cheap: map[string][]string{"missing": {interfaces.WorkStateUnknown}}, detailedErr: missing,+ }+ wantCalls = 3+ }+ marks := &recordingMarks{}+ tool, sleeps := newTestWorkWait(h, marks, nil)+ res, err := tool.Execute(context.Background(), json.RawMessage(+ fmt.Sprintf(`{"work_ids":["missing"],"wait_sec":300,"mode":%q}`, mode)))+ require.NoError(t, err)+ require.Zero(t, *sleeps)+ require.Len(t, h.calls, wantCalls)+ require.True(t, h.calls[wantCalls-1].Detailed)+ require.Equal(t, "run-1", h.calls[wantCalls-1].RunID, "always finish with the run's detailed read")+ require.Equal(t, interfaces.WorkStateLost, decodeWork(t, res.Output).Items[0].State)+ require.Empty(t, marks.seen)+ })+ }+ }+}++func TestWorkCancelFailedReadsDoNotAcknowledgeNotices(t *testing.T) {+ for _, callErr := range []error{errors.New("connection interrupted"), interfaces.ErrWorkNotFound} {+ marks := &recordingMarks{}+ tool := NewWorkCancelTool(&fakeWorkHandler{err: callErr}, workSession, marks)+ res, err := tool.Execute(context.Background(), json.RawMessage(`{"work_ids":["sj_1","sj_2"]}`))+ if errors.Is(callErr, interfaces.ErrWorkNotFound) {+ require.NoError(t, err)+ require.False(t, res.Success)+ require.Empty(t, res.Output)+ } else {+ require.ErrorIs(t, err, callErr)+ }+ require.Empty(t, marks.seen)+ }+}++func TestWorkWaitCancelIsAnErrorAndIsNotSeen(t *testing.T) {+ marks := &recordingMarks{}+ tool, _ := newTestWorkWait(&fakeWorkHandler{}, marks, nil)+ ctx, cancel := context.WithCancel(context.Background())+ cancel()++ res, err := tool.Execute(ctx, json.RawMessage(`{"work_ids":["sj_1"],"wait_sec":30}`))++ require.ErrorIs(t, err, context.Canceled)+ require.Nil(t, res)+ require.Empty(t, marks.seen)+}++func TestWorkCancelContextIsAnErrorAndIsNotSeen(t *testing.T) {+ marks := &recordingMarks{}+ tool := NewWorkCancelTool(&fakeWorkHandler{}, workSession, marks)+ ctx, cancel := context.WithCancel(context.Background())+ cancel()++ res, err := tool.Execute(ctx, json.RawMessage(`{"work_ids":["sj_1"]}`))++ require.ErrorIs(t, err, context.Canceled)+ require.Nil(t, res)+ require.Empty(t, marks.seen)+}++func TestWorkWaitInterruptedNoteOnlyWhileSomethingRuns(t *testing.T) {+ h := &fakeWorkHandler{detailed: map[string]string{"sj_1": interfaces.WorkStateSucceeded}}+ marks := &recordingMarks{}+ tool, _ := newTestWorkWait(h, marks, func(context.Context) bool { return true })++ res, err := tool.Execute(context.Background(), json.RawMessage(`{"work_ids":["sj_1"],"wait_sec":30}`))++ require.NoError(t, err)+ view := decodeWork(t, res.Output)+ require.True(t, view.InterruptedByInput)+ require.Empty(t, view.Note)+ require.Equal(t, []string{"sj_1"}, marks.seen)+}++func TestWorkWaitOutputLimitCoversTheMarshaledBudget(t *testing.T) {+ tool, _ := newTestWorkWait(&fakeWorkHandler{}, nil, nil)+ limit := tool.OutputLimitChars(json.RawMessage(`{"work_ids":["sj_1"]}`))+ require.Greater(t, limit, DefaultMaxToolOutput)+ require.Equal(t, workWaitOutputLimit, limit)+}++func TestWorkWaitRejectsBadArguments(t *testing.T) {+ tool, _ := newTestWorkWait(&fakeWorkHandler{}, &recordingMarks{}, nil)+ tooMany := `{"work_ids":["a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q"]}`+ for _, args := range []string{+ `{"work_ids":[]}`, `{"work_ids":[" "]}`, tooMany, `{"work_ids":["a"],"mode":"some"}`,+ } {+ res, err := tool.Execute(context.Background(), json.RawMessage(args))+ require.NoError(t, err)+ require.False(t, res.Success, args)+ }+}++func TestWorkWaitSecondsClamps(t *testing.T) {+ require.Equal(t, 60, WorkWaitSeconds(`{"work_ids":["a"]}`))+ require.Equal(t, 0, WorkWaitSeconds(`{"wait_sec":-5}`))+ require.Equal(t, 300, WorkWaitSeconds(`{"wait_sec":999}`))+ require.Equal(t, 0, WorkWaitSeconds(`{"wait_sec":0}`))+ require.Equal(t, 60, WorkWaitSeconds(`{`))+}++func TestWorkCancelReportsAndMarksTerminal(t *testing.T) {+ h := &fakeWorkHandler{}+ marks := &recordingMarks{}+ tool := NewWorkCancelTool(h, workSession, marks)++ res, err := tool.Execute(context.Background(), json.RawMessage(`{"work_ids":["sj_run","sj_done","sj_run"]}`))++ require.NoError(t, err)+ require.Equal(t, [][]string{{"sj_run", "sj_done"}}, h.cancelled, "IDs are de-duplicated in order")+ require.Equal(t, []string{"sj_done"}, marks.seen)+ view := decodeWork(t, res.Output)+ require.Equal(t, "Cancellation requested; you will be told when it stops.", view.Note)+}++func TestWorkToolDescriptionsNameTheSubmitters(t *testing.T) {+ tool := NewWorkWaitTool(&fakeWorkHandler{}, workSession, "run-1", nil, nil, []string{"shell_exec_async"})+ require.Contains(t, tool.Description(), "work_ids come from shell_exec_async.")+ require.False(t, CanRunConcurrently(ToolWorkWait))+ require.False(t, CanRunConcurrently(ToolWorkCancel))+}++func TestRegisteredBackgroundSubmitters(t *testing.T) {+ saved := backgroundSubmissionTools+ t.Cleanup(func() { backgroundSubmissionTools = saved })+ backgroundSubmissionTools = []string{"submit_a", "submit_b"}+ registry := NewToolRegistry()+ registry.RegisterTool(&fakeNamedTool{BaseTool: NewBaseTool("submit_b", "x", json.RawMessage(`{"type":"object"}`))})++ require.Equal(t, []string{"submit_b"}, RegisteredBackgroundSubmitters(registry))+}++type fakeNamedTool struct{ BaseTool }++func (*fakeNamedTool) Execute(context.Context, json.RawMessage) (*types.ToolResult, error) {+ return nil, nil+}diff --git a/internal/application/service/agent_history.go b/internal/application/service/agent_history.goindex 13af64c7f..b97e10ad3 100644--- a/internal/application/service/agent_history.go+++ b/internal/application/service/agent_history.go@@ -452,6 +452,7 @@ func buildTurnBodyMessages(assistant *types.Message, midRunUsers []*types.MessagappendUser(midRunUsers[next])next++}+ out = append(out, backgroundInjectionMessages(step)...)out = append(out, buildAgentStepMessages(step)...)}for _, user := range midRunUsers {@@ -508,6 +509,7 @@ func buildUserHistoryMessage(m *types.Message) chat.Message {func buildAssistantHistoryMessages(m *types.Message) []chat.Message {msgs := make([]chat.Message, 0, len(m.AgentSteps)*2+1)for _, step := range m.AgentSteps {+ msgs = append(msgs, backgroundInjectionMessages(step)...)msgs = append(msgs, buildAgentStepMessages(step)...)}if final := finalAnswerHistoryMessage(m); final != nil {@@ -516,6 +518,19 @@ func buildAssistantHistoryMessages(m *types.Message) []chat.Message {return msgs}+// backgroundInjectionMessages replays the background-work messages delivered+// before a step, where the model saw them.+func backgroundInjectionMessages(step types.AgentStep) []chat.Message {+ out := make([]chat.Message, 0, len(step.BackgroundInjections))+ for _, injection := range step.BackgroundInjections {+ if strings.TrimSpace(injection.Content) == "" {+ continue+ }+ out = append(out, chat.Message{Role: "user", Content: injection.Content})+ }+ return out+}+// buildAgentStepMessages expands one persisted step into the OpenAI-shaped// assistant + tool pair. Steps whose only calls were terminal or synthetic// produce nothing, so callers can treat an empty result as "not a real round".diff --git a/internal/application/service/agent_history_background_test.go b/internal/application/service/agent_history_background_test.gonew file mode 100644index 000000000..cada608f9--- /dev/null+++ b/internal/application/service/agent_history_background_test.go@@ -0,0 +1,70 @@+package service++import (+ "testing"+ "time"++ "github.com/Tencent/WeKnora/internal/models/chat"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/stretchr/testify/require"+)++const replayNotice = "<background_notices>\nbuild (sj_1) succeeded.\n</background_notices>"++func backgroundStepAssistant() *types.Message {+ return &types.Message{+ ID: "a-1", SessionID: "s", Role: "assistant", Content: "all green",+ AgentSteps: []types.AgentStep{+ {+ Iteration: 0, Thought: "checking the build", IntermediateAnswer: true,+ Timestamp: time.Date(2026, 10, 15, 10, 0, 1, 0, time.UTC),+ BackgroundInjections: []types.BackgroundInjection{{Content: replayNotice, AckIDs: []string{"sj_1"}}},+ },+ },+ }+}++func rolesAndContents(msgs []chat.Message) [][2]string {+ out := make([][2]string, 0, len(msgs))+ for _, m := range msgs {+ out = append(out, [2]string{m.Role, m.Content})+ }+ return out+}++func TestAssistantHistoryReplaysBackgroundInjectionBeforeItsStep(t *testing.T) {+ got := rolesAndContents(buildAssistantHistoryMessages(backgroundStepAssistant()))++ require.Equal(t, [][2]string{+ {"user", replayNotice},+ {"assistant", "checking the build"},+ {"assistant", "all green"},+ }, got)+}++func TestTurnBodyReplaysSteerThenBackgroundThenStep(t *testing.T) {+ assistant := backgroundStepAssistant()+ assistant.AgentSteps[0].UserMessagesBefore = []string{"u-steer"}+ steer := &types.Message{+ ID: "u-steer", SessionID: "s", Role: "user", Content: "also run lint",+ CreatedAt: time.Date(2026, 10, 15, 10, 0, 0, 0, time.UTC),+ }++ got := rolesAndContents(buildTurnBodyMessages(assistant, []*types.Message{steer}))++ require.Len(t, got, 4)+ require.Equal(t, "user", got[0][0])+ require.Contains(t, got[0][1], "also run lint")+ require.Equal(t, [2]string{"user", replayNotice}, got[1])+ require.Equal(t, [2]string{"assistant", "checking the build"}, got[2])+ require.Equal(t, [2]string{"assistant", "all green"}, got[3])+}++func TestHistoryIgnoresBlankBackgroundInjections(t *testing.T) {+ assistant := backgroundStepAssistant()+ assistant.AgentSteps[0].BackgroundInjections = []types.BackgroundInjection{{Content: " "}}++ got := rolesAndContents(buildAssistantHistoryMessages(assistant))++ require.Equal(t, "assistant", got[0][0])+}diff --git a/internal/application/service/background_activity_helpers_test.go b/internal/application/service/background_activity_helpers_test.gonew file mode 100644index 000000000..abcca6d82--- /dev/null+++ b/internal/application/service/background_activity_helpers_test.go@@ -0,0 +1,35 @@+package service++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/Tencent/WeKnora/internal/background"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/stretchr/testify/require"+)++// liveActivityRegistry returns an in-memory registry holding one live+// background item for each ref.+func liveActivityRegistry(t *testing.T, refs ...interfaces.SessionRef) *background.MemoryActivityRegistry {+ t.Helper()+ reg := background.NewMemoryActivityRegistry(nil)+ for i, ref := range refs {+ require.NoError(t, reg.Add(context.Background(), ref, interfaces.BackgroundActivity{+ ID: fmt.Sprintf("sj_%d", i), Kind: "shell_job", Title: "build", StartedAt: time.Now(),+ }, time.Hour, 0))+ }+ return reg+}++// failingActivityRegistry fails every List call.+type failingActivityRegistry struct {+ interfaces.ActivityRegistry+ err error+}++func (r failingActivityRegistry) List(context.Context, interfaces.SessionRef) ([]interfaces.BackgroundActivity, error) {+ return nil, r.err+}diff --git a/internal/application/service/background_keepalive.go b/internal/application/service/background_keepalive.gonew file mode 100644index 000000000..5efc41ff9--- /dev/null+++ b/internal/application/service/background_keepalive.go@@ -0,0 +1,91 @@+package service++import (+ "context"+ "strings"+ "sync"+ "time"++ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+)++// backgroundKeepAliveTimeout bounds one provider refresh.+const backgroundKeepAliveTimeout = 10 * time.Second++// backgroundKeepAlivePruneAt is how many tracked sessions trigger dropping+// entries whose refresh is already due.+const backgroundKeepAlivePruneAt = 1024++// sessionKeepAliveTarget is the sandbox side BackgroundKeepAliver drives.+// *PinnedSessionSandbox implements it.+type sessionKeepAliveTarget interface {+ KeepAliveSession(ctx context.Context, sessionID string) (time.Duration, error)+}++// BackgroundKeepAliver keeps the sandbox of a session with live background+// work from being reclaimed by its provider. Callers may invoke it on every+// poll; it reaches the provider only once the interval the last successful+// refresh reported has passed.+type BackgroundKeepAliver struct {+ target sessionKeepAliveTarget+ now func() time.Time++ mu sync.Mutex+ next map[interfaces.SessionRef]time.Time+}++var _ interfaces.SessionKeepAliver = (*BackgroundKeepAliver)(nil)++// NewBackgroundKeepAliver returns a keep-aliver over the session's pinned sandbox.+func NewBackgroundKeepAliver(pinned *PinnedSessionSandbox) *BackgroundKeepAliver {+ return newBackgroundKeepAliver(pinned, time.Now)+}++func newBackgroundKeepAliver(target sessionKeepAliveTarget, now func() time.Time) *BackgroundKeepAliver {+ return &BackgroundKeepAliver{+ target: target,+ now: now,+ next: make(map[interfaces.SessionRef]time.Time),+ }+}++// KeepAlive implements interfaces.SessionKeepAliver.+func (k *BackgroundKeepAliver) KeepAlive(ctx context.Context, s interfaces.SessionRef) error {+ if k == nil || k.target == nil || s.TenantID == 0 || strings.TrimSpace(s.SessionID) == "" {+ return nil+ }+ now := k.now()+ k.mu.Lock()+ due := k.next[s]+ k.mu.Unlock()+ if now.Before(due) {+ return nil+ }++ // Pollers have no request tenant. The pin lookup reads the execution+ // tenant; the binding key reads the sandbox tenant. Both are the owner.+ ctx = types.WithSandboxTenantID(types.WithExecutionTenant(ctx, s.TenantID), s.TenantID)+ ctx, cancel := context.WithTimeout(ctx, backgroundKeepAliveTimeout)+ defer cancel()+ interval, err := k.target.KeepAliveSession(ctx, s.SessionID)+ if err != nil {+ return err+ }++ k.mu.Lock()+ defer k.mu.Unlock()+ if interval > 0 {+ k.next[s] = now.Add(interval)+ } else {+ delete(k.next, s)+ }+ if len(k.next) >= backgroundKeepAlivePruneAt {+ for ref, at := range k.next {+ if !now.Before(at) {+ delete(k.next, ref)+ }+ }+ }+ return nil+}diff --git a/internal/application/service/background_keepalive_test.go b/internal/application/service/background_keepalive_test.gonew file mode 100644index 000000000..42fbb7225--- /dev/null+++ b/internal/application/service/background_keepalive_test.go@@ -0,0 +1,76 @@+package service++import (+ "context"+ "errors"+ "testing"+ "time"++ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/stretchr/testify/require"+)++type fakeKeepAliveTarget struct {+ next time.Duration+ err error+ calls []string+ tenants []uint64+}++func (f *fakeKeepAliveTarget) KeepAliveSession(ctx context.Context, sessionID string) (time.Duration, error) {+ f.calls = append(f.calls, sessionID)+ tenant, _ := types.SandboxTenantIDFromContext(ctx)+ f.tenants = append(f.tenants, tenant)+ return f.next, f.err+}++func TestBackgroundKeepAliverThrottlesPerSession(t *testing.T) {+ now := time.Date(2026, 10, 15, 10, 0, 0, 0, time.UTC)+ target := &fakeKeepAliveTarget{next: 10 * time.Minute}+ k := newBackgroundKeepAliver(target, func() time.Time { return now })+ ctx := context.Background()+ s1 := interfaces.SessionRef{TenantID: 7, SessionID: "s1"}+ s2 := interfaces.SessionRef{TenantID: 7, SessionID: "s2"}++ require.NoError(t, k.KeepAlive(ctx, s1))+ require.NoError(t, k.KeepAlive(ctx, s1))+ require.NoError(t, k.KeepAlive(ctx, s2))+ require.Equal(t, []string{"s1", "s2"}, target.calls, "a refresh within the interval is skipped")++ now = now.Add(10 * time.Minute)+ require.NoError(t, k.KeepAlive(ctx, s1))+ require.Equal(t, []string{"s1", "s2", "s1"}, target.calls)+ require.Equal(t, []uint64{7, 7, 7}, target.tenants, "the session owner's tenant keys the binding")+}++func TestBackgroundKeepAliverRetriesAfterFailure(t *testing.T) {+ now := time.Date(2026, 10, 15, 10, 0, 0, 0, time.UTC)+ target := &fakeKeepAliveTarget{next: 10 * time.Minute, err: errors.New("provider down")}+ k := newBackgroundKeepAliver(target, func() time.Time { return now })+ s := interfaces.SessionRef{TenantID: 7, SessionID: "s1"}++ require.Error(t, k.KeepAlive(context.Background(), s))+ require.Error(t, k.KeepAlive(context.Background(), s))+ require.Len(t, target.calls, 2, "a failed refresh must not start the quiet interval")+}++func TestBackgroundKeepAliverWithoutIntervalDoesNotThrottle(t *testing.T) {+ now := time.Date(2026, 10, 15, 10, 0, 0, 0, time.UTC)+ target := &fakeKeepAliveTarget{}+ k := newBackgroundKeepAliver(target, func() time.Time { return now })+ s := interfaces.SessionRef{TenantID: 7, SessionID: "s1"}++ require.NoError(t, k.KeepAlive(context.Background(), s))+ require.NoError(t, k.KeepAlive(context.Background(), s))+ require.Len(t, target.calls, 2, "nothing was bound yet, so the next call must look again")+}++func TestBackgroundKeepAliverIgnoresIncompleteRef(t *testing.T) {+ target := &fakeKeepAliveTarget{next: time.Minute}+ k := newBackgroundKeepAliver(target, time.Now)++ require.NoError(t, k.KeepAlive(context.Background(), interfaces.SessionRef{SessionID: "s1"}))+ require.NoError(t, k.KeepAlive(context.Background(), interfaces.SessionRef{TenantID: 7}))+ require.Empty(t, target.calls)+}diff --git a/internal/application/service/background_work.go b/internal/application/service/background_work.gonew file mode 100644index 000000000..b1d3d616e--- /dev/null+++ b/internal/application/service/background_work.go@@ -0,0 +1,32 @@+package service++import (+ "github.com/Tencent/WeKnora/internal/background"+ "github.com/Tencent/WeKnora/internal/logger"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+)++// BackgroundWork bundles the shared background-work parts the agent, the+// session handlers and features use. Handler is nil until a feature that+// submits background work is wired; until then Enabled is false and nothing+// background-related reaches a run.+type BackgroundWork struct {+ Registry interfaces.ActivityRegistry+ Inbox interfaces.NoticeInbox+ Listeners *background.Listeners+ Handler interfaces.WorkHandler+}++// NewBackgroundWork returns the bundle with an empty listener set.+func NewBackgroundWork(registry interfaces.ActivityRegistry, inbox interfaces.NoticeInbox) *BackgroundWork {+ return &BackgroundWork{+ Registry: registry,+ Inbox: inbox,+ Listeners: background.NewListeners(registry, inbox, logger.Warnf),+ }+}++// Enabled reports whether background work can exist at all.+func (b *BackgroundWork) Enabled() bool {+ return b != nil && b.Handler != nil && b.Registry != nil && b.Inbox != nil+}diff --git a/internal/application/service/pinned_session_sandbox.go b/internal/application/service/pinned_session_sandbox.goindex 9edd2d2c7..af03a4a12 100644--- a/internal/application/service/pinned_session_sandbox.go+++ b/internal/application/service/pinned_session_sandbox.go@@ -74,6 +74,17 @@ func (a *PinnedSessionSandbox) manager(ctx context.Context, sessionID string) sareturn mgr}+// BackendIsDocker reports whether the session's pinned manager is the Docker+// backend. The container id is not a backend name, so callers must not infer+// this from the sandbox id. A missing pin is not Docker.+func (a *PinnedSessionSandbox) BackendIsDocker(ctx context.Context, sessionID string) bool {+ mgr := a.manager(ctx, sessionID)+ if mgr == nil {+ return false+ }+ return mgr.GetType() == sandbox.SandboxTypeDocker+}+// BoundSandboxID reports the sandbox currently bound to sessionID without// provisioning one. A session with no pin or no live binding reports// ok=false.@@ -129,6 +140,52 @@ func (a *PinnedSessionSandbox) HasActiveTurn(ctx context.Context, sessionID strireturn false, nil}+// KeepAliveSession refreshes the provider idle timer of the session's pinned+// sandbox. Managers without the method (host, disabled) have nothing to keep+// alive and report (0, nil).+func (a *PinnedSessionSandbox) KeepAliveSession(ctx context.Context, sessionID string) (time.Duration, error) {+ mgr := a.manager(ctx, sessionID)+ type keepAliver interface {+ KeepAliveSession(context.Context, string) (time.Duration, error)+ }+ if k, ok := mgr.(keepAliver); ok {+ return k.KeepAliveSession(ctx, sessionID)+ }+ return 0, nil+}++// KeepAliveSessionExpected refreshes the provider timer of the bound sandbox+// only when it still is expectedID. Managers without the method have nothing+// to refresh and report (0, nil).+func (a *PinnedSessionSandbox) KeepAliveSessionExpected(+ ctx context.Context, sessionID, expectedID string,+) (time.Duration, error) {+ mgr := a.manager(ctx, sessionID)+ type expectedKeepAliver interface {+ KeepAliveSessionExpected(context.Context, string, string) (time.Duration, error)+ }+ if k, ok := mgr.(expectedKeepAliver); ok {+ return k.KeepAliveSessionExpected(ctx, sessionID, expectedID)+ }+ return 0, nil+}++// ListSessionFiles walks dir in the session sandbox. A manager without the+// method reports an error so callers skip output-file inspection.+func (a *PinnedSessionSandbox) ListSessionFiles(+ ctx context.Context, sessionID, dir string,+) ([]sandbox.RemoteDirEntry, error) {+ mgr := a.manager(ctx, sessionID)+ type lister interface {+ ListSessionFiles(context.Context, string, string) ([]sandbox.RemoteDirEntry, error)+ }+ files, ok := mgr.(lister)+ if !ok || files == nil {+ return nil, errors.New("sandbox: session files cannot be listed")+ }+ return files.ListSessionFiles(ctx, sessionID, dir)+}+// TryLockRewind locks rewind through the session's pinned manager. A manager// that can take the session exclusively returns the context to reset with;// one with only the rewind lock keeps ctx; others succeed as a no-op.diff --git a/internal/application/service/pinned_session_sandbox_test.go b/internal/application/service/pinned_session_sandbox_test.goindex 18972bfa6..920ef0f20 100644--- a/internal/application/service/pinned_session_sandbox_test.go+++ b/internal/application/service/pinned_session_sandbox_test.go@@ -235,3 +235,82 @@ func TestHostManagerForTreatsGlobalDefaultAsUnnamed(t *testing.T) {require.Equal(t, host, r.HostManagerFor(context.Background(), "s1"))}++type stubKeepAlivePinnedManager struct {+ stubPinnedManager+ next time.Duration+ calls int+}++func (m *stubKeepAlivePinnedManager) KeepAliveSession(context.Context, string) (time.Duration, error) {+ m.calls+++ return m.next, nil+}++func TestPinnedSessionSandboxKeepAliveSessionDelegates(t *testing.T) {+ mgr := &stubKeepAlivePinnedManager{next: 4 * time.Minute}+ access := NewPinnedSessionSandbox(stubPinReader{configID: "cfg-1"}, &stubTenantSandboxResolver{mgr: mgr}, nil, nil)+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(9))++ next, err := access.KeepAliveSession(ctx, "sess-1")++ require.NoError(t, err)+ require.Equal(t, 4*time.Minute, next)+ require.Equal(t, 1, mgr.calls)+}++type stubExpectedKeepAliveManager struct {+ stubKeepAlivePinnedManager+ expectedID string+ calls int+}++func (m *stubExpectedKeepAliveManager) KeepAliveSessionExpected(+ _ context.Context, _ string, expectedID string,+) (time.Duration, error) {+ m.calls+++ m.expectedID = expectedID+ return 3 * time.Minute, nil+}++func TestPinnedSessionSandboxKeepAliveSessionExpectedDelegates(t *testing.T) {+ mgr := &stubExpectedKeepAliveManager{}+ access := NewPinnedSessionSandbox(stubPinReader{configID: "cfg-1"}, &stubTenantSandboxResolver{mgr: mgr}, nil, nil)+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(9))++ next, err := access.KeepAliveSessionExpected(ctx, "sess-1", "sbx-expected")++ require.NoError(t, err)+ require.Equal(t, 3*time.Minute, next)+ require.Equal(t, 1, mgr.calls)+ require.Equal(t, "sbx-expected", mgr.expectedID)+}++func TestPinnedSessionSandboxKeepAliveSessionExpectedWhenManagerLacksMethod(t *testing.T) {+ access := NewPinnedSessionSandbox(+ stubPinReader{configID: "cfg-1"},+ &stubTenantSandboxResolver{mgr: &stubPinnedManager{id: "sbx-1", ok: true}},+ nil, nil,+ )+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(9))++ next, err := access.KeepAliveSessionExpected(ctx, "sess-1", "sbx-1")++ require.NoError(t, err)+ require.Zero(t, next)+}++func TestPinnedSessionSandboxKeepAliveSessionWhenManagerLacksMethod(t *testing.T) {+ access := NewPinnedSessionSandbox(+ stubPinReader{configID: "cfg-1"},+ &stubTenantSandboxResolver{mgr: &stubPinnedManager{id: "sbx-1", ok: true}},+ nil,+ nil,+ )+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(9))++ next, err := access.KeepAliveSession(ctx, "sess-1")++ require.NoError(t, err)+ require.Zero(t, next)+}diff --git a/internal/application/service/session.go b/internal/application/service/session.goindex 3dc384436..7e892c5f7 100644--- a/internal/application/service/session.go+++ b/internal/application/service/session.go@@ -182,6 +182,7 @@ type sessionService struct {// remote resolver) keeps the historical delete path and writes no// allocation rows.workspaceCleaner *sandboxalloc.WorkspaceCleaner+ background *BackgroundWork}// NewSessionService creates a new session service instance with all required dependencies@@ -208,6 +209,7 @@ func NewSessionService(cfg *config.Config,sandboxConfigRepo repository.TenantSandboxConfigRepository,tenantSkillRepo repository.TenantSkillRepository,busyGate *SessionBusyGate,+ backgroundWork *BackgroundWork,) interfaces.SessionService {return &sessionService{cfg: cfg,@@ -235,6 +237,7 @@ func NewSessionService(cfg *config.Config,sandboxConfigRepo: sandboxConfigRepo,tenantSkillRepo: tenantSkillRepo,busyGate: busyGate,+ background: backgroundWork,}}@@ -590,6 +593,7 @@ func (s *sessionService) DeleteSession(ctx context.Context, id string) error {}}+ s.notifyBackgroundDelete(ctx, session)if !handled[id] {s.destroyBoundSandbox(ctx, id)}@@ -692,6 +696,9 @@ func (s *sessionService) BatchDeleteSessions(ctx context.Context, ids []string)// Tear down sandboxes while session rows (and pins) are still readable.// A session on a shared machine keeps that machine; only its directory// is removed after the rows are gone.+ for _, session := range visible {+ s.notifyBackgroundDelete(ctx, session)+ }for _, id := range visibleIDs {if handled[id] {continue@@ -773,7 +780,11 @@ func (s *sessionService) DeleteAllSessions(ctx context.Context) error {// Tear down sandboxes while session rows (and pins) are still readable.for _, session := range sessions {- if session == nil || session.ID == "" || handled[session.ID] {+ if session == nil || session.ID == "" {+ continue+ }+ s.notifyBackgroundDelete(ctx, session)+ if handled[session.ID] {continue}s.destroyBoundSandbox(ctx, session.ID)@@ -876,6 +887,18 @@ func (s *sessionService) runUserWorkspaceCleanups(ctx context.Context, handled mgo s.workspaceCleaner.RunCleanups(bg, ids)}+// notifyBackgroundDelete lets background features stop their work for a+// session about to be deleted, then clears its activity items and notices.+// Call it before destroyBoundSandbox, while features can still reach the+// sandbox, and while the session row is still readable.+func (s *sessionService) notifyBackgroundDelete(ctx context.Context, session *types.Session) {+ if s.background == nil || s.background.Listeners == nil || session == nil {+ return+ }+ s.background.Listeners.NotifyDelete(context.WithoutCancel(ctx),+ interfaces.SessionRef{TenantID: session.TenantID, SessionID: session.ID})+}+// destroyBoundSandbox tears down the sandbox MicroVM bound to sessionID, if// the configured sandbox backend supports session-scoped instances.//diff --git a/internal/application/service/session_delete_background_test.go b/internal/application/service/session_delete_background_test.gonew file mode 100644index 000000000..8a018c415--- /dev/null+++ b/internal/application/service/session_delete_background_test.go@@ -0,0 +1,79 @@+package service++import (+ "context"+ "testing"+ "time"++ "github.com/Tencent/WeKnora/internal/background"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/stretchr/testify/require"+ "gorm.io/gorm"+)++type recordingDeleteListener struct {+ deleted []interfaces.SessionRef+}++func (l *recordingDeleteListener) OnStop(context.Context, interfaces.SessionRef) error { return nil }++func (l *recordingDeleteListener) OnDelete(_ context.Context, s interfaces.SessionRef) error {+ l.deleted = append(l.deleted, s)+ return nil+}++func newDeleteBackgroundFixture(t *testing.T, ids ...string) (*sessionService, *recordingDeleteListener) {+ t.Helper()+ svc, db, _ := newSessionServiceForForkDeleteTest(t)+ work := NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))+ listener := &recordingDeleteListener{}+ work.Listeners.Register(listener)+ svc.background = work+ for _, id := range ids {+ require.NoError(t, db.Session(&gorm.Session{SkipHooks: true}).Create(&types.Session{+ ID: id, TenantID: 1, UserID: "u1", Title: id,+ }).Error)+ ref := interfaces.SessionRef{TenantID: 1, SessionID: id}+ require.NoError(t, work.Registry.Add(context.Background(), ref, interfaces.BackgroundActivity{+ ID: "sj_" + id, Kind: "shell_job", StartedAt: time.Now(),+ }, time.Hour, 0))+ }+ return svc, listener+}++func requireNoBackgroundLeft(t *testing.T, svc *sessionService, id string) {+ t.Helper()+ live, err := svc.background.Registry.List(context.Background(), interfaces.SessionRef{TenantID: 1, SessionID: id})+ require.NoError(t, err)+ require.Empty(t, live)+}++func TestDeleteSessionNotifiesBackgroundListeners(t *testing.T) {+ svc, listener := newDeleteBackgroundFixture(t, "s1")++ require.NoError(t, svc.DeleteSession(testSessionScopeContext(1, "u1"), "s1"))++ require.Equal(t, []interfaces.SessionRef{{TenantID: 1, SessionID: "s1"}}, listener.deleted)+ requireNoBackgroundLeft(t, svc, "s1")+}++func TestBatchDeleteSessionsNotifiesBackgroundListeners(t *testing.T) {+ svc, listener := newDeleteBackgroundFixture(t, "s1", "s2")++ require.NoError(t, svc.BatchDeleteSessions(testSessionScopeContext(1, "u1"), []string{"s1", "s2"}))++ require.ElementsMatch(t, []interfaces.SessionRef{+ {TenantID: 1, SessionID: "s1"}, {TenantID: 1, SessionID: "s2"},+ }, listener.deleted)+ requireNoBackgroundLeft(t, svc, "s2")+}++func TestDeleteAllSessionsNotifiesBackgroundListeners(t *testing.T) {+ svc, listener := newDeleteBackgroundFixture(t, "s1", "s2")++ require.NoError(t, svc.DeleteAllSessions(testSessionScopeContext(1, "u1")))++ require.Len(t, listener.deleted, 2)+ requireNoBackgroundLeft(t, svc, "s1")+}diff --git a/internal/application/service/session_fork.go b/internal/application/service/session_fork.goindex 678287a41..a2c385213 100644--- a/internal/application/service/session_fork.go+++ b/internal/application/service/session_fork.go@@ -169,6 +169,7 @@ type SessionForkService struct {messages forkMessageStoresandbox SessionForkSandboxPortallocations forkAllocations+ activity interfaces.ActivityRegistry}// NewSessionForkService wires the service. A nil sandbox port makes every fork@@ -187,8 +188,11 @@ func NewSessionForkServiceFromRepos(sessions interfaces.SessionRepository,messages interfaces.MessageRepository,sandboxPort SessionForkSandboxPort,+ activity interfaces.ActivityRegistry,) *SessionForkService {- return NewSessionForkService(sessions, messages, sandboxPort)+ s := NewSessionForkService(sessions, messages, sandboxPort)+ s.activity = activity+ return s}// WithAllocations enables user-mode forks.@@ -259,6 +263,9 @@ func (s *SessionForkService) Fork(return nil, ErrForkSourceBusy}}+ if err := s.rejectIfBackgroundActive(ctx, source); err != nil {+ return nil, err+ }workCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), forkSnapshotTimeout)defer cancel()@@ -469,6 +476,22 @@ func forkLeaseLost(ctx context.Context, err error) error {return nil}+// rejectIfBackgroundActive refuses while background work is live in the+// session. Snapshotting pauses the source sandbox and would freeze the work.+func (s *SessionForkService) rejectIfBackgroundActive(ctx context.Context, source *types.Session) error {+ if s.activity == nil {+ return nil+ }+ live, err := s.activity.List(ctx, interfaces.SessionRef{TenantID: source.TenantID, SessionID: source.ID})+ if err != nil {+ return fmt.Errorf("session fork: check background work: %w", err)+ }+ if len(live) > 0 {+ return ErrForkSourceBusy+ }+ return nil+}+// prepareBootstrap decides whether the fork can carry the source sandbox's// workspace state. When every condition holds, it takes a snapshot.//@@ -632,27 +655,33 @@ func inheritUnconsumedForkSnapshot(source *types.Session, checkpoint *types.Sand}// latestCheckpoint returns the checkpoint of the last assistant message in-// history, or nil when that message has none.+// history, or nil when that message has none or took it while background work+// was live.func latestCheckpoint(history []*types.Message) *types.SandboxCheckpoint {for i := len(history) - 1; i >= 0; i-- {if history[i].Role != "assistant" {continue}- return history[i].SandboxCheckpoint+ cp := history[i].SandboxCheckpoint+ if cp != nil && cp.BackgroundActive {+ return nil+ }+ return cp}return nil}// latestReachableCheckpoint walks kept assistant messages until it finds one-// with a non-empty commit SHA. Rewind uses this so a failed last-turn-// checkpoint does not skip an earlier SHA that can still reset the workspace.+// with a non-empty commit SHA that was not taken during live background work.+// Rewind uses this so a failed or flagged last-turn checkpoint does not skip an+// earlier SHA that can still reset the workspace.func latestReachableCheckpoint(history []*types.Message) *types.SandboxCheckpoint {for i := len(history) - 1; i >= 0; i-- {if history[i] == nil || history[i].Role != "assistant" {continue}cp := history[i].SandboxCheckpoint- if cp == nil || strings.TrimSpace(cp.CommitSHA) == "" {+ if cp == nil || strings.TrimSpace(cp.CommitSHA) == "" || cp.BackgroundActive {continue}return cpdiff --git a/internal/application/service/session_fork_test.go b/internal/application/service/session_fork_test.goindex d23cad5c7..698415a80 100644--- a/internal/application/service/session_fork_test.go+++ b/internal/application/service/session_fork_test.go@@ -12,6 +12,7 @@ import ("github.com/Tencent/WeKnora/internal/sandbox/admission""github.com/Tencent/WeKnora/internal/sandbox/allocation""github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces""github.com/stretchr/testify/require")@@ -1510,3 +1511,38 @@ func TestForkOfSessionModeSourceStillSnapshots(t *testing.T) {require.NotNil(t, sessions.created.ForkBootstrap)require.Equal(t, "snap-1", sessions.created.ForkBootstrap.SnapshotID)}++func TestForkLiveBackgroundWorkIsBusy(t *testing.T) {+ turn := checkpointedTurn("u-msg-1", "a-msg-1", "sbx-1", "sha1", 0)+ forkPoint := &types.Message{+ ID: "u-msg-2", SessionID: "src", Role: "user", CreatedAt: forkBase.Add(10 * time.Second),+ }+ port := &fakeForkSandboxPort{boundID: "sbx-1", bound: true, snapshotID: "snap-1"}+ svc, sessions, _ := newForkFixture(t, port, append(turn, forkPoint))+ svc.activity = liveActivityRegistry(t, interfaces.SessionRef{TenantID: 1, SessionID: "src"})++ got, err := svc.Fork(context.Background(), 1, "u1", "src", "u-msg-2", "")++ require.ErrorIs(t, err, ErrForkSourceBusy)+ require.Nil(t, got)+ require.Zero(t, port.snapshotCalls)+ require.Nil(t, sessions.created)+}++func TestForkDegradesWhenCheckpointWasTakenWhileBackgroundWorkRan(t *testing.T) {+ turn := checkpointedTurn("u-msg-1", "a-msg-1", "sbx-1", "sha1", 0)+ turn[1].SandboxCheckpoint.BackgroundActive = true+ forkPoint := &types.Message{+ ID: "u-msg-2", SessionID: "src", Role: "user", CreatedAt: forkBase.Add(10 * time.Second),+ }+ port := &fakeForkSandboxPort{boundID: "sbx-1", bound: true, snapshotID: "snap-1"}+ svc, sessions, _ := newForkFixture(t, port, append(turn, forkPoint))++ got, err := svc.Fork(context.Background(), 1, "u1", "src", "u-msg-2", "")++ require.NoError(t, err)+ require.True(t, got.Degraded)+ require.Equal(t, ForkDegradeNoCheckpoint, got.Reason)+ require.Zero(t, port.snapshotCalls)+ require.Nil(t, sessions.created.ForkBootstrap)+}diff --git a/internal/application/service/session_rewind.go b/internal/application/service/session_rewind.goindex a9db1d3c4..76cdcb50f 100644--- a/internal/application/service/session_rewind.go+++ b/internal/application/service/session_rewind.go@@ -175,6 +175,7 @@ type SessionRewindService struct {busyGate *SessionBusyGate// suspendedRuns closes runs suspended on prompts before the cut.suspendedRuns *SuspendedRunService+ activity interfaces.ActivityRegistry}// SetSuspendedRuns lets rewind close the session's suspended runs.@@ -211,6 +212,7 @@ func NewSessionRewindServiceFromRepos(fallback sandbox.Manager,streams interfaces.StreamManager,busyGate *SessionBusyGate,+ activity interfaces.ActivityRegistry,) *SessionRewindService {s := NewSessionRewindService(sessions, messages, sandboxPort, knowledge, suggestions)s.snapshots = NewResolverForkSnapshotDeleter(resolver, fallback)@@ -218,6 +220,7 @@ func NewSessionRewindServiceFromRepos(if busyGate != nil {s.busyGate = busyGate}+ s.activity = activityreturn s}@@ -279,6 +282,9 @@ func (s *SessionRewindService) Rewind(if err := s.rejectIfBusy(ctx, sessionID); err != nil {return nil, err}+ if err := s.rejectIfBackgroundActive(ctx, source); err != nil {+ return nil, err+ }history, err := s.keptCheckpointHistory(ctx, sessionID, rewindPoint)if err != nil {@@ -294,6 +300,11 @@ func (s *SessionRewindService) Rewind(return nil, err}defer unlock()+ // Re-check under the lock. The busy gate now holds off new turns, so no+ // new background work can start before the truncate lands.+ if err := s.rejectIfBackgroundActive(persistCtx, source); err != nil {+ return nil, err+ }var abandonedBootstrap *types.ForkBootstrapworkspaceReset, reason, resetErr := s.resetWorkspaceIfPossible(workCtx, sessionID, history)@@ -476,6 +487,7 @@ func (s *SessionRewindService) lockRewind(ctx context.Context, sessionID string)if errors.Is(err, sandbox.ErrSessionRewindLocked) ||errors.Is(err, sandbox.ErrSessionTurnActive) ||errors.Is(err, sandbox.ErrSessionWorkspaceBusy) ||+ errors.Is(err, sandbox.ErrSessionWorkActive) ||errors.Is(err, ErrRewindSourceBusy) {return nil, nil, ErrRewindSourceBusy}@@ -494,6 +506,24 @@ func (s *SessionRewindService) lockRewind(ctx context.Context, sessionID string)}, nil}+// rejectIfBackgroundActive refuses while background work is live in the+// session. On Web the rewind lock refuses as well (tryLockRewindScript); this+// check is what covers Lite, which has no turn leases, and where rewind would+// truncate the conversation a pending completion notice still refers to.+func (s *SessionRewindService) rejectIfBackgroundActive(ctx context.Context, source *types.Session) error {+ if s.activity == nil {+ return nil+ }+ live, err := s.activity.List(ctx, interfaces.SessionRef{TenantID: source.TenantID, SessionID: source.ID})+ if err != nil {+ return fmt.Errorf("session rewind: check background work: %w", err)+ }+ if len(live) > 0 {+ return ErrRewindSourceBusy+ }+ return nil+}+func (s *SessionRewindService) rejectIfBusy(ctx context.Context, sessionID string) error {if s.liveRuns != nil {liveID, _, err := s.liveRuns.GetLiveRun(ctx, sessionID)diff --git a/internal/application/service/session_rewind_test.go b/internal/application/service/session_rewind_test.goindex 0cae00f38..abd7d0f0c 100644--- a/internal/application/service/session_rewind_test.go+++ b/internal/application/service/session_rewind_test.go@@ -12,6 +12,7 @@ import ("github.com/Tencent/WeKnora/internal/sandbox""github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces""github.com/stretchr/testify/require")@@ -949,3 +950,82 @@ func TestRewindKeepsForkSnapshotWhenDeleteFails(t *testing.T) {require.True(t, sessions.bootstrapCleared,"the column write stays ahead of the delete so the fork cannot boot ahead of the transcript")}++func rewindBusyFixture(t *testing.T) (*SessionRewindService, *fakeMessageStore, *fakeRewindSandboxPort) {+ t.Helper()+ turn := rewindCompletedTurn("u-1", "a-1", "sbx-1", rewindSHA1, 0)+ laterUser := &types.Message{+ ID: "u-2", SessionID: "src", Role: "user", CreatedAt: forkBase.Add(10 * time.Second),+ }+ port := newFakeRewindPort()+ svc, msgs := newRewindFixture(t, port, append(turn, laterUser))+ return svc, msgs, port+}++func TestRewindLiveBackgroundWorkIsBusy(t *testing.T) {+ svc, msgs, port := rewindBusyFixture(t)+ svc.activity = liveActivityRegistry(t, interfaces.SessionRef{TenantID: 1, SessionID: "src"})++ got, err := svc.Rewind(context.Background(), 1, "u1", "src", "u-2")++ require.ErrorIs(t, err, ErrRewindSourceBusy)+ require.Nil(t, got)+ require.Equal(t, 0, msgs.deleteFromCalls)+ require.Empty(t, port.runner.calls)+}++func TestRewindBackgroundWorkElsewhereIsNotBusy(t *testing.T) {+ svc, _, _ := rewindBusyFixture(t)+ svc.activity = liveActivityRegistry(t,+ interfaces.SessionRef{TenantID: 1, SessionID: "other"},+ interfaces.SessionRef{TenantID: 2, SessionID: "src"},+ )++ got, err := svc.Rewind(context.Background(), 1, "u1", "src", "u-2")++ require.NoError(t, err)+ require.Equal(t, 1, got.DeletedMessages)+}++func TestRewindBackgroundLookupFailureIsAnError(t *testing.T) {+ svc, msgs, _ := rewindBusyFixture(t)+ svc.activity = failingActivityRegistry{err: errors.New("redis down")}++ _, err := svc.Rewind(context.Background(), 1, "u1", "src", "u-2")++ require.Error(t, err)+ require.NotErrorIs(t, err, ErrRewindSourceBusy)+ require.Equal(t, 0, msgs.deleteFromCalls)+}++func TestRewindLockRefusedByBackgroundWorkIsBusy(t *testing.T) {+ svc, msgs, port := rewindBusyFixture(t)+ port.lockErr = sandbox.ErrSessionWorkActive++ got, err := svc.Rewind(context.Background(), 1, "u1", "src", "u-2")++ require.ErrorIs(t, err, ErrRewindSourceBusy)+ require.Nil(t, got)+ require.Equal(t, 0, msgs.deleteFromCalls)+}++func TestRewindSkipsCheckpointTakenWhileBackgroundWorkRan(t *testing.T) {+ turn1 := rewindCompletedTurn("u-1", "a-1", "sbx-1", rewindSHA1, 0)+ turn2 := rewindCompletedTurn("u-2", "a-2", "sbx-1", rewindSHA2, 10*time.Second)+ turn2[1].SandboxCheckpoint.BackgroundActive = true+ laterUser := &types.Message{+ ID: "u-3", SessionID: "src", Role: "user", CreatedAt: forkBase.Add(20 * time.Second),+ }+ port := newFakeRewindPort()+ messages := append(append(turn1, turn2...), laterUser)+ svc, msgs := newRewindFixture(t, port, messages)++ got, err := svc.Rewind(context.Background(), 1, "u1", "src", "u-3")++ require.NoError(t, err)+ require.True(t, got.WorkspaceReset)+ require.Len(t, port.runner.calls, 1)+ require.Contains(t, port.runner.calls[0], rewindSHA1)+ require.NotContains(t, port.runner.calls[0], rewindSHA2)+ require.Equal(t, []string{"u-1", "a-1", "u-2", "a-2"}, messageIDs(msgs.messages))+}diff --git a/internal/application/service/workspace_checkpointer.go b/internal/application/service/workspace_checkpointer.goindex 6f0db9e48..7e77285b4 100644--- a/internal/application/service/workspace_checkpointer.go+++ b/internal/application/service/workspace_checkpointer.go@@ -26,6 +26,7 @@ import ("github.com/Tencent/WeKnora/internal/logger""github.com/Tencent/WeKnora/internal/sandbox""github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces")// SandboxShellRunner is the narrow subset of sandbox behaviour the@@ -52,7 +53,8 @@ var gitSHAPattern = regexp.MustCompile(`^[0-9a-f]{40}$`)// WorkspaceCheckpointer commits /workspace after each agent turn.type WorkspaceCheckpointer struct {- runner SandboxShellRunner+ runner SandboxShellRunner+ activity interfaces.ActivityRegistry}// NewWorkspaceCheckpointer wires a checkpointer. A nil runner yields a@@ -62,6 +64,34 @@ func NewWorkspaceCheckpointer(runner SandboxShellRunner) *WorkspaceCheckpointerreturn &WorkspaceCheckpointer{runner: runner}}+// WithActivityRegistry makes Checkpoint flag commits taken while background+// work is live. Without a registry every checkpoint is unflagged.+func (c *WorkspaceCheckpointer) WithActivityRegistry(activity interfaces.ActivityRegistry) *WorkspaceCheckpointer {+ if c != nil {+ c.activity = activity+ }+ return c+}++// backgroundActive reports whether background work is live in the session. A+// failed lookup counts as live: an unusable fork point is recoverable, a rewind+// onto a half-written tree is not.+func (c *WorkspaceCheckpointer) backgroundActive(ctx context.Context, sessionID string) bool {+ if c.activity == nil {+ return false+ }+ tenantID, ok := types.SandboxTenantIDFromContext(ctx)+ if !ok || tenantID == 0 {+ return false+ }+ live, err := c.activity.List(ctx, interfaces.SessionRef{TenantID: tenantID, SessionID: sessionID})+ if err != nil {+ logger.Warnf(ctx, "[WorkspaceCheckpointer] background lookup failed session=%s: %v", sessionID, err)+ return true+ }+ return len(live) > 0+}+// checkpointScript renders the idempotent init-and-commit script.//// Ordering matters in two places:@@ -118,6 +148,9 @@ func (c *WorkspaceCheckpointer) Checkpoint(return nil}command := checkpointScriptFor(layout, messageID)+ // Read before committing: no tool runs while the turn is finishing, so no+ // background work can start between this read and the commit.+ backgroundActive := c.backgroundActive(ctx, sessionID)var result *sandbox.ExecuteResultif runner, ok := c.runner.(sandbox.SessionInstallShellExecutor); ok {// Git operates on the existing workspace. Do not prepare artifact/input@@ -156,9 +189,10 @@ func (c *WorkspaceCheckpointer) Checkpoint(}return &types.SandboxCheckpoint{- SandboxID: sandboxID,- CommitSHA: sha,- CommittedAt: time.Now().UTC(),+ SandboxID: sandboxID,+ CommitSHA: sha,+ CommittedAt: time.Now().UTC(),+ BackgroundActive: backgroundActive,}}diff --git a/internal/application/service/workspace_checkpointer_test.go b/internal/application/service/workspace_checkpointer_test.goindex 068ffaaa4..d5a14d372 100644--- a/internal/application/service/workspace_checkpointer_test.go+++ b/internal/application/service/workspace_checkpointer_test.go@@ -8,6 +8,8 @@ import ("time""github.com/Tencent/WeKnora/internal/sandbox"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces""github.com/stretchr/testify/require")@@ -132,3 +134,48 @@ func TestCheckpointSkippedWhenBackendDoesNotVersionWorkspace(t *testing.T) {require.Nil(t, cp.Checkpoint(context.Background(), "s1", "sbx-1", "msg-1"))require.Empty(t, runner.calls, "must not git commit the user's real project")}++func shaRunner() *fakeShellRunner {+ return &fakeShellRunner{result: &sandbox.ExecuteResult{ExitCode: 0, Stdout: strings.Repeat("a", 40) + "\n"}}+}++func TestCheckpointFlagsCommitTakenWhileBackgroundWorkIsLive(t *testing.T) {+ reg := liveActivityRegistry(t, interfaces.SessionRef{TenantID: 7, SessionID: "s1"})+ cp := NewWorkspaceCheckpointer(shaRunner()).WithActivityRegistry(reg)++ got := cp.Checkpoint(types.WithSandboxTenantID(context.Background(), 7), "s1", "sbx-1", "msg-1")++ require.NotNil(t, got)+ require.True(t, got.BackgroundActive)+}++func TestCheckpointLeavesFlagOffWithoutLiveBackgroundWork(t *testing.T) {+ reg := liveActivityRegistry(t,+ interfaces.SessionRef{TenantID: 7, SessionID: "s2"},+ interfaces.SessionRef{TenantID: 8, SessionID: "s1"},+ )+ cp := NewWorkspaceCheckpointer(shaRunner()).WithActivityRegistry(reg)++ got := cp.Checkpoint(types.WithSandboxTenantID(context.Background(), 7), "s1", "sbx-1", "msg-1")++ require.NotNil(t, got)+ require.False(t, got.BackgroundActive)+}++func TestCheckpointFlagsCommitWhenBackgroundLookupFails(t *testing.T) {+ cp := NewWorkspaceCheckpointer(shaRunner()).+ WithActivityRegistry(failingActivityRegistry{err: errors.New("redis down")})++ got := cp.Checkpoint(types.WithSandboxTenantID(context.Background(), 7), "s1", "sbx-1", "msg-1")++ require.NotNil(t, got)+ require.True(t, got.BackgroundActive)+}++func TestCheckpointWithoutRegistryIsUnflagged(t *testing.T) {+ got := NewWorkspaceCheckpointer(shaRunner()).+ Checkpoint(types.WithSandboxTenantID(context.Background(), 7), "s1", "sbx-1", "msg-1")++ require.NotNil(t, got)+ require.False(t, got.BackgroundActive)+}diff --git a/internal/background/backgroundtest/contract.go b/internal/background/backgroundtest/contract.gonew file mode 100644index 000000000..76d8ff17b--- /dev/null+++ b/internal/background/backgroundtest/contract.go@@ -0,0 +1,287 @@+// Package backgroundtest holds contract tests shared by every implementation of+// the background-work interfaces.+package backgroundtest++import (+ "context"+ "fmt"+ "sort"+ "sync"+ "testing"+ "time"++ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/stretchr/testify/require"+)++// Advance moves the implementation's notion of "now" forward by d.+type Advance func(d time.Duration)++var (+ sessA = interfaces.SessionRef{TenantID: 1, SessionID: "sess-a"}+ sessB = interfaces.SessionRef{TenantID: 1, SessionID: "sess-b"}+)++func activity(id, kind string) interfaces.BackgroundActivity {+ return interfaces.BackgroundActivity{+ ID: id, Kind: kind, Title: "title " + id,+ StartedAt: time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC),+ }+}++func listIDs(t *testing.T, r interfaces.ActivityRegistry, s interfaces.SessionRef) []string {+ t.Helper()+ items, err := r.List(context.Background(), s)+ require.NoError(t, err)+ ids := make([]string, 0, len(items))+ for _, it := range items {+ ids = append(ids, it.ID)+ }+ sort.Strings(ids)+ return ids+}++// RunActivityRegistryContract runs the shared ActivityRegistry cases. newRegistry+// must return an empty registry and a function that advances its clock.+func RunActivityRegistryContract(+ t *testing.T,+ newRegistry func(t *testing.T) (interfaces.ActivityRegistry, Advance),+) {+ ctx := context.Background()++ t.Run("AddThenList", func(t *testing.T) {+ r, _ := newRegistry(t)+ require.NoError(t, r.Add(ctx, sessA, activity("sj_1", "shell_job"), time.Minute, 0))+ items, err := r.List(ctx, sessA)+ require.NoError(t, err)+ require.Len(t, items, 1)+ require.Equal(t, "sj_1", items[0].ID)+ require.Equal(t, "shell_job", items[0].Kind)+ require.Equal(t, "title sj_1", items[0].Title)+ require.True(t, items[0].StartedAt.Equal(activity("sj_1", "").StartedAt))+ })++ t.Run("SessionsAreIsolated", func(t *testing.T) {+ r, _ := newRegistry(t)+ require.NoError(t, r.Add(ctx, sessA, activity("sj_1", "shell_job"), time.Minute, 0))+ require.Empty(t, listIDs(t, r, sessB))+ })++ t.Run("CapIsPerKind", func(t *testing.T) {+ r, _ := newRegistry(t)+ require.NoError(t, r.Add(ctx, sessA, activity("a1", "k1"), time.Minute, 2))+ require.NoError(t, r.Add(ctx, sessA, activity("a2", "k1"), time.Minute, 2))+ err := r.Add(ctx, sessA, activity("a3", "k1"), time.Minute, 2)+ require.ErrorIs(t, err, interfaces.ErrActivityLimit)+ require.NoError(t, r.Add(ctx, sessA, activity("b1", "k2"), time.Minute, 2))+ require.Equal(t, []string{"a1", "a2", "b1"}, listIDs(t, r, sessA))+ })++ t.Run("NonPositiveCapMeansUnlimited", func(t *testing.T) {+ r, _ := newRegistry(t)+ for i := 0; i < 10; i++ {+ require.NoError(t, r.Add(ctx, sessA, activity(fmt.Sprintf("a%d", i), "k"), time.Minute, 0))+ }+ require.NoError(t, r.Add(ctx, sessA, activity("neg", "k"), time.Minute, -1))+ require.Len(t, listIDs(t, r, sessA), 11)+ })++ t.Run("ReAddExtendsWithoutCountingAgainstCap", func(t *testing.T) {+ r, advance := newRegistry(t)+ require.NoError(t, r.Add(ctx, sessA, activity("a1", "k"), 10*time.Second, 1))+ advance(8 * time.Second)+ require.NoError(t, r.Add(ctx, sessA, activity("a1", "k"), 10*time.Second, 1))+ advance(8 * time.Second) // 16s after the first Add, 8s after the second+ require.Equal(t, []string{"a1"}, listIDs(t, r, sessA))+ })++ t.Run("ConcurrentAddRespectsCap", func(t *testing.T) {+ r, _ := newRegistry(t)+ const workers = 20+ var wg sync.WaitGroup+ errs := make([]error, workers)+ for i := 0; i < workers; i++ {+ wg.Add(1)+ go func(i int) {+ defer wg.Done()+ errs[i] = r.Add(ctx, sessA, activity(fmt.Sprintf("c%d", i), "k"), time.Minute, 3)+ }(i)+ }+ wg.Wait()+ ok := 0+ for _, err := range errs {+ if err == nil {+ ok+++ continue+ }+ require.ErrorIs(t, err, interfaces.ErrActivityLimit)+ }+ require.Equal(t, 3, ok)+ require.Len(t, listIDs(t, r, sessA), 3)+ })++ t.Run("ExpiredItemsDisappearAndFreeTheCap", func(t *testing.T) {+ r, advance := newRegistry(t)+ require.NoError(t, r.Add(ctx, sessA, activity("a1", "k"), 10*time.Second, 1))+ advance(11 * time.Second)+ require.Empty(t, listIDs(t, r, sessA))+ require.NoError(t, r.Add(ctx, sessA, activity("a2", "k"), 10*time.Second, 1))+ require.Equal(t, []string{"a2"}, listIDs(t, r, sessA))+ })++ t.Run("RenewExtends", func(t *testing.T) {+ r, advance := newRegistry(t)+ require.NoError(t, r.Add(ctx, sessA, activity("a1", "k"), 10*time.Second, 0))+ advance(8 * time.Second)+ found, err := r.Renew(ctx, sessA, "a1", 10*time.Second)+ require.NoError(t, err)+ require.True(t, found)+ advance(8 * time.Second)+ require.Equal(t, []string{"a1"}, listIDs(t, r, sessA))+ })++ t.Run("RenewDoesNotRecreateRemoved", func(t *testing.T) {+ r, _ := newRegistry(t)+ require.NoError(t, r.Add(ctx, sessA, activity("a1", "k"), time.Minute, 0))+ require.NoError(t, r.Remove(ctx, sessA, "a1"))+ found, err := r.Renew(ctx, sessA, "a1", time.Minute)+ require.NoError(t, err)+ require.False(t, found)+ require.Empty(t, listIDs(t, r, sessA))+ })++ t.Run("RenewDoesNotRecreateExpired", func(t *testing.T) {+ r, advance := newRegistry(t)+ require.NoError(t, r.Add(ctx, sessA, activity("a1", "k"), 10*time.Second, 0))+ advance(11 * time.Second)+ found, err := r.Renew(ctx, sessA, "a1", time.Minute)+ require.NoError(t, err)+ require.False(t, found)+ require.Empty(t, listIDs(t, r, sessA))+ })++ t.Run("RenewUnknownIsNotFound", func(t *testing.T) {+ r, _ := newRegistry(t)+ found, err := r.Renew(ctx, sessA, "missing", time.Minute)+ require.NoError(t, err)+ require.False(t, found)+ })++ t.Run("RemoveMissingIsNotAnError", func(t *testing.T) {+ r, _ := newRegistry(t)+ require.NoError(t, r.Remove(ctx, sessA, "missing"))+ })+}++func notice(id string, finishedAt time.Time) interfaces.BackgroundNotice {+ return interfaces.BackgroundNotice{ID: id, Kind: "shell_job", Text: "done " + id, FinishedAt: finishedAt}+}++func pendingIDs(t *testing.T, in interfaces.NoticeInbox, s interfaces.SessionRef) []string {+ t.Helper()+ items, err := in.Pending(context.Background(), s)+ require.NoError(t, err)+ ids := make([]string, 0, len(items))+ for _, it := range items {+ ids = append(ids, it.ID)+ }+ return ids+}++// RunNoticeInboxContract runs the shared NoticeInbox cases. newInbox must return+// an empty inbox whose clock starts at 2026-10-14T10:00:00Z, and a function that+// advances that clock.+func RunNoticeInboxContract(+ t *testing.T,+ newInbox func(t *testing.T) (interfaces.NoticeInbox, Advance),+) {+ ctx := context.Background()+ start := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC)++ t.Run("FirstPushWins", func(t *testing.T) {+ in, _ := newInbox(t)+ first, err := in.Push(ctx, sessA, notice("n1", start))+ require.NoError(t, err)+ require.True(t, first)+ second := notice("n1", start)+ second.Text = "overwritten"+ first, err = in.Push(ctx, sessA, second)+ require.NoError(t, err)+ require.False(t, first)+ items, err := in.Pending(ctx, sessA)+ require.NoError(t, err)+ require.Len(t, items, 1)+ require.Equal(t, "done n1", items[0].Text)+ })++ t.Run("PendingIsOldestFirst", func(t *testing.T) {+ in, _ := newInbox(t)+ _, err := in.Push(ctx, sessA, notice("late", start.Add(2*time.Minute)))+ require.NoError(t, err)+ _, err = in.Push(ctx, sessA, notice("early", start))+ require.NoError(t, err)+ _, err = in.Push(ctx, sessA, notice("mid", start.Add(time.Minute)))+ require.NoError(t, err)+ require.Equal(t, []string{"early", "mid", "late"}, pendingIDs(t, in, sessA))+ })++ t.Run("AckRemoves", func(t *testing.T) {+ in, _ := newInbox(t)+ _, err := in.Push(ctx, sessA, notice("n1", start))+ require.NoError(t, err)+ _, err = in.Push(ctx, sessA, notice("n2", start))+ require.NoError(t, err)+ require.NoError(t, in.Ack(ctx, sessA, []string{"n1", "unknown"}))+ require.Equal(t, []string{"n2"}, pendingIDs(t, in, sessA))+ })++ t.Run("AckEmptyIsNoop", func(t *testing.T) {+ in, _ := newInbox(t)+ require.NoError(t, in.Ack(ctx, sessA, nil))+ })++ t.Run("AckedIDCanBePushedAgain", func(t *testing.T) {+ in, _ := newInbox(t)+ _, err := in.Push(ctx, sessA, notice("n1", start))+ require.NoError(t, err)+ require.NoError(t, in.Ack(ctx, sessA, []string{"n1"}))+ first, err := in.Push(ctx, sessA, notice("n1", start))+ require.NoError(t, err)+ require.True(t, first)+ })++ t.Run("RetentionHidesOldNotices", func(t *testing.T) {+ in, advance := newInbox(t)+ _, err := in.Push(ctx, sessA, notice("n1", start))+ require.NoError(t, err)+ advance(interfaces.NoticeRetention - time.Minute)+ require.Equal(t, []string{"n1"}, pendingIDs(t, in, sessA))+ advance(2 * time.Minute)+ require.Empty(t, pendingIDs(t, in, sessA))+ })++ t.Run("ClearRemovesSessionOnly", func(t *testing.T) {+ in, _ := newInbox(t)+ _, err := in.Push(ctx, sessA, notice("n1", start))+ require.NoError(t, err)+ _, err = in.Push(ctx, sessB, notice("n2", start))+ require.NoError(t, err)+ require.NoError(t, in.Clear(ctx, sessA))+ require.Empty(t, pendingIDs(t, in, sessA))+ require.Equal(t, []string{"n2"}, pendingIDs(t, in, sessB))+ })++ t.Run("PreservesFields", func(t *testing.T) {+ in, _ := newInbox(t)+ n := notice("n1", start)+ _, err := in.Push(ctx, sessA, n)+ require.NoError(t, err)+ items, err := in.Pending(ctx, sessA)+ require.NoError(t, err)+ require.Len(t, items, 1)+ require.Equal(t, n.ID, items[0].ID)+ require.Equal(t, n.Kind, items[0].Kind)+ require.Equal(t, n.Text, items[0].Text)+ require.True(t, n.FinishedAt.Equal(items[0].FinishedAt))+ })+}diff --git a/internal/background/lifecycle.go b/internal/background/lifecycle.gonew file mode 100644index 000000000..aef115004--- /dev/null+++ b/internal/background/lifecycle.go@@ -0,0 +1,98 @@+package background++import (+ "context"+ "fmt"+ "time"++ "github.com/Tencent/WeKnora/internal/types/interfaces"+)++// Lifecycle combines the shared parts in the only orders that are safe.+// Features call these four methods and never the underlying interfaces.+type Lifecycle struct {+ Registry interfaces.ActivityRegistry+ Inbox interfaces.NoticeInbox+ KeepAlive interfaces.SessionKeepAliver // optional+ Events interfaces.WorkEventEmitter // optional+ Logf func(ctx context.Context, format string, args ...any)+}++func (l *Lifecycle) logf(ctx context.Context, format string, args ...any) {+ if l.Logf != nil {+ l.Logf(ctx, format, args...)+ }+}++func (l *Lifecycle) keepAlive(ctx context.Context, s interfaces.SessionRef) {+ if l.KeepAlive == nil {+ return+ }+ if err := l.KeepAlive.KeepAlive(ctx, s); err != nil {+ l.logf(ctx, "[Background] keep-alive failed session=%s: %v", s.SessionID, err)+ }+}++// Begin registers a, then runs start. Registering first means lifecycle+// protection is in force the moment the work exists. If start fails the+// registration is removed. On success the sandbox is kept alive once at once,+// because nothing renewed it before the work began.+func (l *Lifecycle) Begin(+ ctx context.Context, s interfaces.SessionRef, a interfaces.BackgroundActivity,+ lease time.Duration, kindCap int, start func(context.Context) error,+) error {+ if err := l.Registry.Add(ctx, s, a, lease, kindCap); err != nil {+ return err+ }+ if err := start(ctx); err != nil {+ if rmErr := l.Registry.Remove(context.WithoutCancel(ctx), s, a.ID); rmErr != nil {+ l.logf(ctx, "[Background] remove after failed start session=%s id=%s: %v", s.SessionID, a.ID, rmErr)+ }+ return err+ }+ l.keepAlive(ctx, s)+ return nil+}++// Alive renews the registration and, when it still exists, keeps the sandbox+// alive. found=false means the item was removed or expired; the caller decides+// whether to re-register.+func (l *Lifecycle) Alive(+ ctx context.Context, s interfaces.SessionRef, id string, lease time.Duration,+) (bool, error) {+ found, err := l.Registry.Renew(ctx, s, id, lease)+ if err != nil {+ return false, err+ }+ if found {+ l.keepAlive(ctx, s)+ }+ return found, nil+}++// Finish records the completion notice, emits work_finished only if this call+// wrote the notice first, then removes the registration. Push comes first: in+// the other order there is a window where neither the registry nor the inbox+// knows the item. If Push fails the registration is kept so the item stays+// queryable and a later Finish can retry.+func (l *Lifecycle) Finish(+ ctx context.Context, s interfaces.SessionRef, n interfaces.BackgroundNotice,+ r interfaces.WorkResult, parentMessageID string,+) error {+ first, err := l.Inbox.Push(ctx, s, n)+ if err != nil {+ return fmt.Errorf("push background notice: %w", err)+ }+ if first && l.Events != nil {+ if err := l.Events.EmitWorkFinished(ctx, s, parentMessageID, r); err != nil {+ l.logf(ctx, "[Background] emit work_finished failed session=%s id=%s: %v", s.SessionID, n.ID, err)+ }+ }+ return l.Registry.Remove(ctx, s, n.ID)+}++// Discard removes the registration without a notice or event. Use it only when+// the result was already handed to the model directly.+func (l *Lifecycle) Discard(ctx context.Context, s interfaces.SessionRef, id string) error {+ return l.Registry.Remove(ctx, s, id)+}diff --git a/internal/background/lifecycle_test.go b/internal/background/lifecycle_test.gonew file mode 100644index 000000000..d2ac5e603--- /dev/null+++ b/internal/background/lifecycle_test.go@@ -0,0 +1,182 @@+package background++import (+ "context"+ "errors"+ "testing"+ "time"++ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/stretchr/testify/require"+)++type countingKeepAliver struct{ calls int }++func (k *countingKeepAliver) KeepAlive(context.Context, interfaces.SessionRef) error {+ k.calls+++ return nil+}++type recordingEmitter struct{ finished []interfaces.WorkResult }++func (e *recordingEmitter) EmitWorkFinished(+ _ context.Context, _ interfaces.SessionRef, _ string, r interfaces.WorkResult,+) error {+ e.finished = append(e.finished, r)+ return nil+}++type lifecycleFixture struct {+ l *Lifecycle+ registry *MemoryActivityRegistry+ inbox *MemoryNoticeInbox+ keep *countingKeepAliver+ events *recordingEmitter+}++func newLifecycleFixture() lifecycleFixture {+ f := lifecycleFixture{+ registry: NewMemoryActivityRegistry(nil),+ inbox: NewMemoryNoticeInbox(nil),+ keep: &countingKeepAliver{},+ events: &recordingEmitter{},+ }+ f.l = &Lifecycle{Registry: f.registry, Inbox: f.inbox, KeepAlive: f.keep, Events: f.events}+ return f+}++var lcSess = interfaces.SessionRef{TenantID: 1, SessionID: "s"}++func lcActivity(id string) interfaces.BackgroundActivity {+ return interfaces.BackgroundActivity{ID: id, Kind: "shell_job", Title: "make build", StartedAt: time.Now()}+}++func liveIDs(t *testing.T, r interfaces.ActivityRegistry) []string {+ t.Helper()+ items, err := r.List(context.Background(), lcSess)+ require.NoError(t, err)+ ids := make([]string, 0, len(items))+ for _, it := range items {+ ids = append(ids, it.ID)+ }+ return ids+}++func TestBeginRegistersBeforeStartAndKeepsAlive(t *testing.T) {+ f := newLifecycleFixture()+ var registeredAtStart []string+ err := f.l.Begin(context.Background(), lcSess, lcActivity("sj_1"), time.Minute, 0,+ func(context.Context) error {+ registeredAtStart = liveIDs(t, f.registry)+ return nil+ })+ require.NoError(t, err)+ require.Equal(t, []string{"sj_1"}, registeredAtStart)+ require.Equal(t, []string{"sj_1"}, liveIDs(t, f.registry))+ require.Equal(t, 1, f.keep.calls)+}++func TestBeginRemovesRegistrationWhenStartFails(t *testing.T) {+ f := newLifecycleFixture()+ boom := errors.New("start failed")+ err := f.l.Begin(context.Background(), lcSess, lcActivity("sj_1"), time.Minute, 0,+ func(context.Context) error { return boom })+ require.ErrorIs(t, err, boom)+ require.Empty(t, liveIDs(t, f.registry))+ require.Zero(t, f.keep.calls)+}++func TestBeginReturnsLimitWithoutStarting(t *testing.T) {+ f := newLifecycleFixture()+ require.NoError(t, f.l.Begin(context.Background(), lcSess, lcActivity("sj_1"), time.Minute, 1,+ func(context.Context) error { return nil }))+ started := false+ err := f.l.Begin(context.Background(), lcSess, lcActivity("sj_2"), time.Minute, 1,+ func(context.Context) error { started = true; return nil })+ require.ErrorIs(t, err, interfaces.ErrActivityLimit)+ require.False(t, started)+}++func TestAliveRenewsAndKeepsAliveOnlyWhenFound(t *testing.T) {+ f := newLifecycleFixture()+ require.NoError(t, f.registry.Add(context.Background(), lcSess, lcActivity("sj_1"), time.Minute, 0))++ found, err := f.l.Alive(context.Background(), lcSess, "sj_1", time.Minute)+ require.NoError(t, err)+ require.True(t, found)+ require.Equal(t, 1, f.keep.calls)++ found, err = f.l.Alive(context.Background(), lcSess, "missing", time.Minute)+ require.NoError(t, err)+ require.False(t, found)+ require.Equal(t, 1, f.keep.calls)+}++func TestFinishPushesEmitsOnceAndRemoves(t *testing.T) {+ f := newLifecycleFixture()+ ctx := context.Background()+ require.NoError(t, f.registry.Add(ctx, lcSess, lcActivity("sj_1"), time.Minute, 0))+ n := interfaces.BackgroundNotice{ID: "sj_1", Kind: "shell_job", Text: "done", FinishedAt: time.Now()}+ r := interfaces.WorkResult{ID: "sj_1", State: interfaces.WorkStateSucceeded, Terminal: true}++ require.NoError(t, f.l.Finish(ctx, lcSess, n, r, "msg-1"))+ require.NoError(t, f.l.Finish(ctx, lcSess, n, r, "msg-1")) // second reporter++ require.Len(t, f.events.finished, 1)+ require.Empty(t, liveIDs(t, f.registry))+ pending, err := f.inbox.Pending(ctx, lcSess)+ require.NoError(t, err)+ require.Len(t, pending, 1)+}++func TestFinishKeepsRegistrationWhenPushFails(t *testing.T) {+ f := newLifecycleFixture()+ ctx := context.Background()+ require.NoError(t, f.registry.Add(ctx, lcSess, lcActivity("sj_1"), time.Minute, 0))+ f.l.Inbox = failingInbox{}++ err := f.l.Finish(ctx, lcSess, interfaces.BackgroundNotice{ID: "sj_1"}, interfaces.WorkResult{ID: "sj_1"}, "")+ require.Error(t, err)+ require.Equal(t, []string{"sj_1"}, liveIDs(t, f.registry),+ "without a notice the item must stay visible, or queries would find it nowhere")+ require.Empty(t, f.events.finished)+}++func TestDiscardRemovesWithoutNoticeOrEvent(t *testing.T) {+ f := newLifecycleFixture()+ ctx := context.Background()+ require.NoError(t, f.registry.Add(ctx, lcSess, lcActivity("sj_1"), time.Minute, 0))++ require.NoError(t, f.l.Discard(ctx, lcSess, "sj_1"))++ require.Empty(t, liveIDs(t, f.registry))+ pending, err := f.inbox.Pending(ctx, lcSess)+ require.NoError(t, err)+ require.Empty(t, pending)+ require.Empty(t, f.events.finished)+}++func TestLifecycleToleratesNilOptionalParts(t *testing.T) {+ l := &Lifecycle{Registry: NewMemoryActivityRegistry(nil), Inbox: NewMemoryNoticeInbox(nil)}+ ctx := context.Background()+ require.NoError(t, l.Begin(ctx, lcSess, lcActivity("sj_1"), time.Minute, 0,+ func(context.Context) error { return nil }))+ _, err := l.Alive(ctx, lcSess, "sj_1", time.Minute)+ require.NoError(t, err)+ require.NoError(t, l.Finish(ctx, lcSess, interfaces.BackgroundNotice{ID: "sj_1", FinishedAt: time.Now()},+ interfaces.WorkResult{ID: "sj_1"}, ""))+}++type failingInbox struct{}++func (failingInbox) Push(context.Context, interfaces.SessionRef, interfaces.BackgroundNotice) (bool, error) {+ return false, errors.New("redis down")+}++func (failingInbox) Pending(context.Context, interfaces.SessionRef) ([]interfaces.BackgroundNotice, error) {+ return nil, nil+}++func (failingInbox) Ack(context.Context, interfaces.SessionRef, []string) error { return nil }++func (failingInbox) Clear(context.Context, interfaces.SessionRef) error { return nil }diff --git a/internal/background/listeners.go b/internal/background/listeners.gonew file mode 100644index 000000000..457841535--- /dev/null+++ b/internal/background/listeners.go@@ -0,0 +1,81 @@+package background++import (+ "context"+ "sync"++ "github.com/Tencent/WeKnora/internal/types/interfaces"+)++// Listeners is the ordered set of session lifecycle listeners. Feature+// listeners run in registration order; on delete the shared activity+// registry and inbox are cleared after all of them have run.+type Listeners struct {+ registry interfaces.ActivityRegistry+ inbox interfaces.NoticeInbox+ logf func(ctx context.Context, format string, args ...any)++ mu sync.RWMutex+ listeners []interfaces.SessionLifecycleListener+}++// NewListeners returns an empty listener set. A nil logf discards warnings.+func NewListeners(+ registry interfaces.ActivityRegistry,+ inbox interfaces.NoticeInbox,+ logf func(ctx context.Context, format string, args ...any),+) *Listeners {+ if logf == nil {+ logf = func(context.Context, string, ...any) {}+ }+ return &Listeners{registry: registry, inbox: inbox, logf: logf}+}++// Register appends a listener.+func (l *Listeners) Register(listener interfaces.SessionLifecycleListener) {+ l.mu.Lock()+ defer l.mu.Unlock()+ l.listeners = append(l.listeners, listener)+}++func (l *Listeners) snapshot() []interfaces.SessionLifecycleListener {+ l.mu.RLock()+ defer l.mu.RUnlock()+ return append([]interfaces.SessionLifecycleListener(nil), l.listeners...)+}++// NotifyStop calls OnStop on every listener. Errors are logged, never returned.+func (l *Listeners) NotifyStop(ctx context.Context, s interfaces.SessionRef) {+ for _, listener := range l.snapshot() {+ if err := listener.OnStop(ctx, s); err != nil {+ l.logf(ctx, "[Background] OnStop listener failed session=%s: %v", s.SessionID, err)+ }+ }+}++// NotifyDelete calls OnDelete on every listener, then clears the session's+// activity items and notices. Errors are logged, never returned.+func (l *Listeners) NotifyDelete(ctx context.Context, s interfaces.SessionRef) {+ for _, listener := range l.snapshot() {+ if err := listener.OnDelete(ctx, s); err != nil {+ l.logf(ctx, "[Background] OnDelete listener failed session=%s: %v", s.SessionID, err)+ }+ }+ if l.registry != nil {+ items, err := l.registry.List(ctx, s)+ if err != nil {+ l.logf(ctx, "[Background] list activity on delete failed session=%s: %v", s.SessionID, err)+ }+ for _, it := range items {+ if err := l.registry.Remove(ctx, s, it.ID); err != nil {+ l.logf(ctx, "[Background] remove activity on delete failed session=%s id=%s: %v",+ s.SessionID, it.ID, err)+ }+ }+ }+ if l.inbox != nil {+ if err := l.inbox.Clear(ctx, s); err != nil {+ l.logf(ctx, "[Background] clear inbox on delete failed session=%s: %v", s.SessionID, err)+ }+ }+}diff --git a/internal/background/listeners_test.go b/internal/background/listeners_test.gonew file mode 100644index 000000000..1fd10521f--- /dev/null+++ b/internal/background/listeners_test.go@@ -0,0 +1,95 @@+package background++import (+ "context"+ "errors"+ "fmt"+ "testing"+ "time"++ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/stretchr/testify/require"+)++type recordingListener struct {+ name string+ log *[]string+ err error+}++func (l recordingListener) OnStop(context.Context, interfaces.SessionRef) error {+ *l.log = append(*l.log, l.name+":stop")+ return l.err+}++func (l recordingListener) OnDelete(context.Context, interfaces.SessionRef) error {+ *l.log = append(*l.log, l.name+":delete")+ return l.err+}++func TestListenersRunInOrderAndSurviveErrors(t *testing.T) {+ var log []string+ var warnings []string+ logf := func(_ context.Context, format string, args ...any) {+ warnings = append(warnings, fmt.Sprintf(format, args...))+ }+ l := NewListeners(NewMemoryActivityRegistry(nil), NewMemoryNoticeInbox(nil), logf)+ l.Register(recordingListener{name: "a", log: &log, err: errors.New("a failed")})+ l.Register(recordingListener{name: "b", log: &log})++ s := interfaces.SessionRef{TenantID: 1, SessionID: "s"}+ l.NotifyStop(context.Background(), s)+ l.NotifyDelete(context.Background(), s)++ require.Equal(t, []string{"a:stop", "b:stop", "a:delete", "b:delete"}, log)+ require.Len(t, warnings, 2)+}++func TestNotifyDeleteClearsSharedStateLast(t *testing.T) {+ ctx := context.Background()+ s := interfaces.SessionRef{TenantID: 1, SessionID: "s"}+ registry := NewMemoryActivityRegistry(nil)+ inbox := NewMemoryNoticeInbox(nil)+ require.NoError(t, registry.Add(ctx, s, interfaces.BackgroundActivity{ID: "sj_1", Kind: "k"}, time.Hour, 0))+ _, err := inbox.Push(ctx, s, interfaces.BackgroundNotice{ID: "sj_0", FinishedAt: time.Now()})+ require.NoError(t, err)++ var sawDuringListener int+ l := NewListeners(registry, inbox, nil)+ l.Register(listenerFunc(func(ctx context.Context, s interfaces.SessionRef) {+ items, _ := registry.List(ctx, s)+ sawDuringListener = len(items)+ }))++ l.NotifyDelete(ctx, s)++ require.Equal(t, 1, sawDuringListener, "feature listeners must run before shared state is cleared")+ items, err := registry.List(ctx, s)+ require.NoError(t, err)+ require.Empty(t, items)+ pending, err := inbox.Pending(ctx, s)+ require.NoError(t, err)+ require.Empty(t, pending)+}++func TestNotifyStopKeepsSharedState(t *testing.T) {+ ctx := context.Background()+ s := interfaces.SessionRef{TenantID: 1, SessionID: "s"}+ registry := NewMemoryActivityRegistry(nil)+ require.NoError(t, registry.Add(ctx, s, interfaces.BackgroundActivity{ID: "sj_1", Kind: "k"}, time.Hour, 0))++ NewListeners(registry, NewMemoryNoticeInbox(nil), nil).NotifyStop(ctx, s)++ items, err := registry.List(ctx, s)+ require.NoError(t, err)+ require.Len(t, items, 1)+}++type listenerFunc func(ctx context.Context, s interfaces.SessionRef)++func (f listenerFunc) OnStop(context.Context, interfaces.SessionRef) error { return nil }++func (f listenerFunc) OnDelete(ctx context.Context, s interfaces.SessionRef) error {+ f(ctx, s)+ return nil+}diff --git a/internal/background/memory_inbox.go b/internal/background/memory_inbox.gonew file mode 100644index 000000000..3254cbf37--- /dev/null+++ b/internal/background/memory_inbox.go@@ -0,0 +1,97 @@+package background++import (+ "context"+ "sort"+ "sync"++ "github.com/Tencent/WeKnora/internal/types/interfaces"+)++// MemoryNoticeInbox is the single-process NoticeInbox used by Lite.+type MemoryNoticeInbox struct {+ clock Clock+ mu sync.Mutex+ notices map[interfaces.SessionRef]map[string]interfaces.BackgroundNotice+}++var _ interfaces.NoticeInbox = (*MemoryNoticeInbox)(nil)++// NewMemoryNoticeInbox returns an empty inbox. A nil clock means SystemClock.+func NewMemoryNoticeInbox(clock Clock) *MemoryNoticeInbox {+ if clock == nil {+ clock = SystemClock{}+ }+ return &MemoryNoticeInbox{+ clock: clock,+ notices: make(map[interfaces.SessionRef]map[string]interfaces.BackgroundNotice),+ }+}++// Push implements interfaces.NoticeInbox.+func (b *MemoryNoticeInbox) Push(+ _ context.Context, s interfaces.SessionRef, n interfaces.BackgroundNotice,+) (bool, error) {+ b.mu.Lock()+ defer b.mu.Unlock()+ m := b.notices[s]+ if _, ok := m[n.ID]; ok {+ return false, nil+ }+ if m == nil {+ m = make(map[string]interfaces.BackgroundNotice)+ b.notices[s] = m+ }+ m[n.ID] = n+ return true, nil+}++// Pending implements interfaces.NoticeInbox.+func (b *MemoryNoticeInbox) Pending(+ _ context.Context, s interfaces.SessionRef,+) ([]interfaces.BackgroundNotice, error) {+ b.mu.Lock()+ defer b.mu.Unlock()+ cutoff := b.clock.Now().Add(-interfaces.NoticeRetention)+ m := b.notices[s]+ out := make([]interfaces.BackgroundNotice, 0, len(m))+ for id, n := range m {+ if n.FinishedAt.Before(cutoff) {+ delete(m, id)+ continue+ }+ out = append(out, n)+ }+ if len(m) == 0 {+ delete(b.notices, s)+ }+ sort.Slice(out, func(i, j int) bool {+ if out[i].FinishedAt.Equal(out[j].FinishedAt) {+ return out[i].ID < out[j].ID+ }+ return out[i].FinishedAt.Before(out[j].FinishedAt)+ })+ return out, nil+}++// Ack implements interfaces.NoticeInbox.+func (b *MemoryNoticeInbox) Ack(_ context.Context, s interfaces.SessionRef, ids []string) error {+ b.mu.Lock()+ defer b.mu.Unlock()+ m := b.notices[s]+ for _, id := range ids {+ delete(m, id)+ }+ if len(m) == 0 {+ delete(b.notices, s)+ }+ return nil+}++// Clear implements interfaces.NoticeInbox.+func (b *MemoryNoticeInbox) Clear(_ context.Context, s interfaces.SessionRef) error {+ b.mu.Lock()+ defer b.mu.Unlock()+ delete(b.notices, s)+ return nil+}diff --git a/internal/background/memory_registry.go b/internal/background/memory_registry.gonew file mode 100644index 000000000..18f698944--- /dev/null+++ b/internal/background/memory_registry.go@@ -0,0 +1,139 @@+// Package background holds the shared background-work parts: in-memory+// implementations, the lifecycle-listener registry and the lifecycle helpers.+// It must not branch on activity kind.+package background++import (+ "context"+ "sync"+ "time"++ "github.com/Tencent/WeKnora/internal/types/interfaces"+)++// Clock abstracts time so tests can move it.+type Clock interface {+ Now() time.Time+}++// SystemClock is the real wall clock.+type SystemClock struct{}++// Now returns time.Now().+func (SystemClock) Now() time.Time { return time.Now() }++type memoryActivity struct {+ item interfaces.BackgroundActivity+ deadline time.Time+}++// MemoryActivityRegistry is the single-process ActivityRegistry used by Lite.+type MemoryActivityRegistry struct {+ clock Clock+ mu sync.Mutex+ items map[interfaces.SessionRef]map[string]memoryActivity+}++var _ interfaces.ActivityRegistry = (*MemoryActivityRegistry)(nil)++// NewMemoryActivityRegistry returns an empty registry. A nil clock means SystemClock.+func NewMemoryActivityRegistry(clock Clock) *MemoryActivityRegistry {+ if clock == nil {+ clock = SystemClock{}+ }+ return &MemoryActivityRegistry{+ clock: clock,+ items: make(map[interfaces.SessionRef]map[string]memoryActivity),+ }+}++// liveLocked drops expired items of s and returns what remains. Caller holds mu.+func (r *MemoryActivityRegistry) liveLocked(s interfaces.SessionRef, now time.Time) map[string]memoryActivity {+ m := r.items[s]+ for id, it := range m {+ if !now.Before(it.deadline) {+ delete(m, id)+ }+ }+ if len(m) == 0 {+ delete(r.items, s)+ return nil+ }+ return m+}++// Add implements interfaces.ActivityRegistry.+func (r *MemoryActivityRegistry) Add(+ _ context.Context, s interfaces.SessionRef, a interfaces.BackgroundActivity, lease time.Duration, kindCap int,+) error {+ r.mu.Lock()+ defer r.mu.Unlock()+ now := r.clock.Now()+ m := r.liveLocked(s, now)+ if existing, ok := m[a.ID]; ok {+ existing.deadline = now.Add(lease)+ m[a.ID] = existing+ return nil+ }+ if kindCap > 0 {+ count := 0+ for _, it := range m {+ if it.item.Kind == a.Kind {+ count+++ }+ }+ if count >= kindCap {+ return interfaces.ErrActivityLimit+ }+ }+ if m == nil {+ m = make(map[string]memoryActivity)+ r.items[s] = m+ }+ m[a.ID] = memoryActivity{item: a, deadline: now.Add(lease)}+ return nil+}++// Renew implements interfaces.ActivityRegistry.+func (r *MemoryActivityRegistry) Renew(+ _ context.Context, s interfaces.SessionRef, id string, lease time.Duration,+) (bool, error) {+ r.mu.Lock()+ defer r.mu.Unlock()+ now := r.clock.Now()+ m := r.liveLocked(s, now)+ it, ok := m[id]+ if !ok {+ return false, nil+ }+ it.deadline = now.Add(lease)+ m[id] = it+ return true, nil+}++// Remove implements interfaces.ActivityRegistry.+func (r *MemoryActivityRegistry) Remove(_ context.Context, s interfaces.SessionRef, id string) error {+ r.mu.Lock()+ defer r.mu.Unlock()+ if m := r.items[s]; m != nil {+ delete(m, id)+ if len(m) == 0 {+ delete(r.items, s)+ }+ }+ return nil+}++// List implements interfaces.ActivityRegistry.+func (r *MemoryActivityRegistry) List(+ _ context.Context, s interfaces.SessionRef,+) ([]interfaces.BackgroundActivity, error) {+ r.mu.Lock()+ defer r.mu.Unlock()+ m := r.liveLocked(s, r.clock.Now())+ out := make([]interfaces.BackgroundActivity, 0, len(m))+ for _, it := range m {+ out = append(out, it.item)+ }+ return out, nil+}diff --git a/internal/background/memory_test.go b/internal/background/memory_test.gonew file mode 100644index 000000000..06a5ea433--- /dev/null+++ b/internal/background/memory_test.go@@ -0,0 +1,47 @@+package background++import (+ "sync"+ "testing"+ "time"++ "github.com/Tencent/WeKnora/internal/background/backgroundtest"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+)++type fakeClock struct {+ mu sync.Mutex+ now time.Time+}++func newFakeClock() *fakeClock {+ return &fakeClock{now: time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC)}+}++func (c *fakeClock) Now() time.Time {+ c.mu.Lock()+ defer c.mu.Unlock()+ return c.now+}++func (c *fakeClock) Advance(d time.Duration) {+ c.mu.Lock()+ defer c.mu.Unlock()+ c.now = c.now.Add(d)+}++func TestMemoryActivityRegistryContract(t *testing.T) {+ backgroundtest.RunActivityRegistryContract(t,+ func(*testing.T) (interfaces.ActivityRegistry, backgroundtest.Advance) {+ clock := newFakeClock()+ return NewMemoryActivityRegistry(clock), clock.Advance+ })+}++func TestMemoryNoticeInboxContract(t *testing.T) {+ backgroundtest.RunNoticeInboxContract(t,+ func(*testing.T) (interfaces.NoticeInbox, backgroundtest.Advance) {+ clock := newFakeClock()+ return NewMemoryNoticeInbox(clock), clock.Advance+ })+}diff --git a/internal/background/run_notices.go b/internal/background/run_notices.gonew file mode 100644index 000000000..94637e770--- /dev/null+++ b/internal/background/run_notices.go@@ -0,0 +1,185 @@+package background++import (+ "context"+ "fmt"+ "sort"+ "strings"+ "sync"++ "github.com/Tencent/WeKnora/internal/types/interfaces"+)++// Delivery is background-work context for the model at one round boundary.+type Delivery struct {+ // Text is the model-facing body, one line per entry, without a wrapper.+ Text string+ // AckIDs are the completion notices Text delivers. They are acknowledged+ // once the assistant message carrying them is persisted.+ AckIDs []string+}++// RunNotices decides, for one agent run, what the model is told about+// background work and when. A run executes in one process, so its sets live in+// memory; the shared registry and inbox are only read here.+type RunNotices struct {+ session interfaces.SessionRef+ registry interfaces.ActivityRegistry+ inbox interfaces.NoticeInbox++ mu sync.Mutex+ listed bool+ seen map[string]bool+ started map[string]bool+ reminded map[string]bool+ newSeen []string+}++// NewRunNotices returns a tracker for one run of session s.+func NewRunNotices(+ s interfaces.SessionRef, registry interfaces.ActivityRegistry, inbox interfaces.NoticeInbox,+) *RunNotices {+ return &RunNotices{+ session: s,+ registry: registry,+ inbox: inbox,+ seen: make(map[string]bool),+ started: make(map[string]bool),+ reminded: make(map[string]bool),+ }+}++// MarkStarted records work this run submitted and got back as still running.+// Only such work is reminded about before the final answer.+func (r *RunNotices) MarkStarted(id string) {+ if id == "" {+ return+ }+ r.mu.Lock()+ defer r.mu.Unlock()+ r.started[id] = true+}++// MarkSeen records finished IDs a tool already showed the model, so their+// notices are not injected again and are acknowledged with the turn.+func (r *RunNotices) MarkSeen(ids ...string) {+ r.mu.Lock()+ defer r.mu.Unlock()+ for _, id := range ids {+ if id == "" || r.seen[id] {+ continue+ }+ r.seen[id] = true+ r.newSeen = append(r.newSeen, id)+ }+}++// TakeSeen returns the IDs MarkSeen recorded since the previous call.+func (r *RunNotices) TakeSeen() []string {+ r.mu.Lock()+ defer r.mu.Unlock()+ out := r.newSeen+ r.newSeen = nil+ return out+}++// RoundStart returns what to tell the model before a round's LLM call: on the+// run's first boundary, the work still running; on every boundary, completion+// notices this run has not seen.+func (r *RunNotices) RoundStart(ctx context.Context) (Delivery, error) {+ r.mu.Lock()+ first := !r.listed+ r.mu.Unlock()++ var lines []string+ if first {+ live, err := r.registry.List(ctx, r.session)+ if err != nil {+ return Delivery{}, fmt.Errorf("list background work: %w", err)+ }+ if len(live) > 0 {+ lines = append(lines, "Background work still running in this session: "+describeActivities(live)+".")+ }+ }+ notices, err := r.inbox.Pending(ctx, r.session)+ if err != nil {+ return Delivery{}, fmt.Errorf("read background notices: %w", err)+ }++ r.mu.Lock()+ defer r.mu.Unlock()+ r.listed = true+ return r.deliverLocked(notices, lines), nil+}++// BeforeFinal returns what to tell the model before it may finish: unseen+// completion notices, and one reminder for each item this run started that is+// still running and has not been reminded about.+func (r *RunNotices) BeforeFinal(ctx context.Context) (Delivery, error) {+ notices, err := r.inbox.Pending(ctx, r.session)+ if err != nil {+ return Delivery{}, fmt.Errorf("read background notices: %w", err)+ }+ live, err := r.registry.List(ctx, r.session)+ if err != nil {+ return Delivery{}, fmt.Errorf("list background work: %w", err)+ }++ r.mu.Lock()+ defer r.mu.Unlock()+ var remind []interfaces.BackgroundActivity+ for _, item := range live {+ if r.started[item.ID] && !r.reminded[item.ID] {+ remind = append(remind, item)+ }+ }+ var lines []string+ if len(remind) > 0 {+ for _, item := range remind {+ r.reminded[item.ID] = true+ }+ lines = append(lines, "Background work you started in this turn is still running: "++ describeActivities(remind)++ ". Finish your answer. Its result will arrive when it completes. "++ "Call work_wait only if you cannot answer without that output.")+ }+ return r.deliverLocked(notices, lines), nil+}++// deliverLocked puts unseen notices first, then lines, and marks the notices+// seen. Caller holds mu.+func (r *RunNotices) deliverLocked(notices []interfaces.BackgroundNotice, lines []string) Delivery {+ var d Delivery+ text := make([]string, 0, len(notices)+len(lines))+ for _, n := range notices {+ if r.seen[n.ID] || strings.TrimSpace(n.Text) == "" {+ continue+ }+ r.seen[n.ID] = true+ text = append(text, n.Text)+ d.AckIDs = append(d.AckIDs, n.ID)+ }+ text = append(text, lines...)+ d.Text = strings.Join(text, "\n")+ return d+}++// describeActivities renders items as "title (id)" in start order.+func describeActivities(items []interfaces.BackgroundActivity) string {+ sorted := append([]interfaces.BackgroundActivity(nil), items...)+ sort.Slice(sorted, func(i, j int) bool {+ if sorted[i].StartedAt.Equal(sorted[j].StartedAt) {+ return sorted[i].ID < sorted[j].ID+ }+ return sorted[i].StartedAt.Before(sorted[j].StartedAt)+ })+ parts := make([]string, 0, len(sorted))+ for _, item := range sorted {+ if item.Title == "" {+ parts = append(parts, item.ID)+ continue+ }+ parts = append(parts, fmt.Sprintf("%s (%s)", item.Title, item.ID))+ }+ return strings.Join(parts, ", ")+}diff --git a/internal/background/run_notices_test.go b/internal/background/run_notices_test.gonew file mode 100644index 000000000..9c3753abf--- /dev/null+++ b/internal/background/run_notices_test.go@@ -0,0 +1,133 @@+package background++import (+ "context"+ "errors"+ "testing"+ "time"++ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/stretchr/testify/require"+)++var noticeSession = interfaces.SessionRef{TenantID: 1, SessionID: "s"}++func newNoticeFixture(t *testing.T) (*RunNotices, *MemoryActivityRegistry, *MemoryNoticeInbox) {+ t.Helper()+ reg := NewMemoryActivityRegistry(nil)+ inbox := NewMemoryNoticeInbox(nil)+ return NewRunNotices(noticeSession, reg, inbox), reg, inbox+}++func addRunning(t *testing.T, reg *MemoryActivityRegistry, id, title string, started time.Time) {+ t.Helper()+ require.NoError(t, reg.Add(context.Background(), noticeSession, interfaces.BackgroundActivity{+ ID: id, Kind: "shell_job", Title: title, StartedAt: started,+ }, time.Hour, 0))+}++func pushNotice(t *testing.T, inbox *MemoryNoticeInbox, id, text string) {+ t.Helper()+ _, err := inbox.Push(context.Background(), noticeSession, interfaces.BackgroundNotice{+ ID: id, Kind: "shell_job", Text: text, FinishedAt: time.Now(),+ })+ require.NoError(t, err)+}++func TestRunNoticesListsRunningWorkOnFirstRoundOnly(t *testing.T) {+ r, reg, _ := newNoticeFixture(t)+ base := time.Now()+ addRunning(t, reg, "sj_b", "tests", base.Add(time.Second))+ addRunning(t, reg, "sj_a", "build", base)++ first, err := r.RoundStart(context.Background())+ require.NoError(t, err)+ require.Equal(t, "Background work still running in this session: build (sj_a), tests (sj_b).", first.Text)+ require.Empty(t, first.AckIDs)++ second, err := r.RoundStart(context.Background())+ require.NoError(t, err)+ require.Empty(t, second.Text)+}++func TestRunNoticesDeliversEachNoticeOnce(t *testing.T) {+ r, _, inbox := newNoticeFixture(t)+ pushNotice(t, inbox, "sj_1", "build (sj_1) succeeded.")++ got, err := r.RoundStart(context.Background())+ require.NoError(t, err)+ require.Equal(t, "build (sj_1) succeeded.", got.Text)+ require.Equal(t, []string{"sj_1"}, got.AckIDs)++ pushNotice(t, inbox, "sj_2", "tests (sj_2) failed.")+ got, err = r.BeforeFinal(context.Background())+ require.NoError(t, err)+ require.Equal(t, "tests (sj_2) failed.", got.Text, "sj_1 was already delivered this run")+ require.Equal(t, []string{"sj_2"}, got.AckIDs)+}++func TestRunNoticesSkipsNoticesAToolAlreadyShowed(t *testing.T) {+ r, _, inbox := newNoticeFixture(t)+ pushNotice(t, inbox, "sj_1", "build (sj_1) succeeded.")++ r.MarkSeen("sj_1", "sj_1", "")+ require.Equal(t, []string{"sj_1"}, r.TakeSeen())+ require.Empty(t, r.TakeSeen())++ got, err := r.RoundStart(context.Background())+ require.NoError(t, err)+ require.Empty(t, got.Text)+ require.Empty(t, got.AckIDs)+}++func TestRunNoticesRemindsOnlyWorkThisRunStarted(t *testing.T) {+ r, reg, _ := newNoticeFixture(t)+ base := time.Now()+ addRunning(t, reg, "sj_old", "dev server", base)+ addRunning(t, reg, "sj_new", "build", base.Add(time.Second))+ r.MarkStarted("sj_new")++ got, err := r.BeforeFinal(context.Background())+ require.NoError(t, err)+ require.Equal(t, "Background work you started in this turn is still running: build (sj_new). "++ "Finish your answer. Its result will arrive when it completes. "++ "Call work_wait only if you cannot answer without that output.", got.Text)++ again, err := r.BeforeFinal(context.Background())+ require.NoError(t, err)+ require.Empty(t, again.Text, "each item is reminded once")+}++func TestRunNoticesPutsNoticesBeforeOtherLines(t *testing.T) {+ r, reg, inbox := newNoticeFixture(t)+ addRunning(t, reg, "sj_a", "build", time.Now())+ pushNotice(t, inbox, "sj_0", "lint (sj_0) succeeded.")++ got, err := r.RoundStart(context.Background())+ require.NoError(t, err)+ require.Equal(t, "lint (sj_0) succeeded.\nBackground work still running in this session: build (sj_a).", got.Text)+}++type failingRegistry struct {+ interfaces.ActivityRegistry+ err error+}++func (f failingRegistry) List(context.Context, interfaces.SessionRef) ([]interfaces.BackgroundActivity, error) {+ return nil, f.err+}++func TestRunNoticesRetriesTheListingAfterAnError(t *testing.T) {+ reg := NewMemoryActivityRegistry(nil)+ r := NewRunNotices(noticeSession, failingRegistry{ActivityRegistry: reg, err: errors.New("redis down")},+ NewMemoryNoticeInbox(nil))+ addRunning(t, reg, "sj_a", "build", time.Now())++ _, err := r.RoundStart(context.Background())+ require.Error(t, err)++ r.registry = reg // the store recovers+ got, err := r.RoundStart(context.Background())+ require.NoError(t, err)+ require.Contains(t, got.Text, "build (sj_a)", "a failed first boundary must not use up the listing")+}diff --git a/internal/container/background.go b/internal/container/background.gonew file mode 100644index 000000000..57f4c8b2c--- /dev/null+++ b/internal/container/background.go@@ -0,0 +1,40 @@+// Package container - background-work wiring.+package container++import (+ "github.com/Tencent/WeKnora/internal/application/service/sandboxalloc"+ "github.com/Tencent/WeKnora/internal/background"+ "github.com/Tencent/WeKnora/internal/sandbox"+ "github.com/Tencent/WeKnora/internal/stream"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+)++// newBackgroundActivityRegistry returns the registry of live background work.+// With Redis it lives on the binding store, so turn leases, the rebuild check+// and the rewind lock read it in the same Lua call. The memory store uses+// the same lock for activity registration and lifecycle checks.+func newBackgroundActivityRegistry(+ store sandbox.SessionSandboxBindingStore, allocations *sandboxalloc.Service,+) interfaces.ActivityRegistry {+ switch store := store.(type) {+ case *sandbox.RedisSessionSandboxBindingStore:+ registry := sandbox.NewRedisActivityRegistry(store)+ if allocations != nil {+ registry.WithAllocations(allocations)+ }+ return registry+ case *sandbox.MemorySessionSandboxBindingStore:+ return sandbox.NewMemoryActivityRegistry(store)+ }+ return background.NewMemoryActivityRegistry(nil)+}++// newBackgroundNoticeInbox returns the completion-notice inbox. With Redis it+// lives next to the steer lists; otherwise it is process-local, like the+// in-memory stream manager it follows.+func newBackgroundNoticeInbox(streams interfaces.StreamManager) interfaces.NoticeInbox {+ if redisStreams, ok := streams.(*stream.RedisStreamManager); ok {+ return stream.NewRedisNoticeInbox(redisStreams, nil)+ }+ return background.NewMemoryNoticeInbox(nil)+}diff --git a/internal/container/background_test.go b/internal/container/background_test.gonew file mode 100644index 000000000..6f6cdc593--- /dev/null+++ b/internal/container/background_test.go@@ -0,0 +1,72 @@+package container++import (+ "context"+ "testing"+ "time"++ "github.com/Tencent/WeKnora/internal/application/service/sandboxalloc"+ "github.com/Tencent/WeKnora/internal/sandbox"+ "github.com/Tencent/WeKnora/internal/sandbox/allocation"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/alicebob/miniredis/v2"+ "github.com/redis/go-redis/v9"+ "github.com/stretchr/testify/require"+)++func TestBackgroundActivityRegistrySharesMemoryBindingState(t *testing.T) {+ ctx := t.Context()+ store := sandbox.NewMemorySessionSandboxBindingStore()+ registry := newBackgroundActivityRegistry(store, nil)+ ref := interfaces.SessionRef{TenantID: 42, SessionID: "s1"}+ key := sandbox.SessionSandboxKey{TenantID: ref.TenantID, SessionID: ref.SessionID}+ require.NoError(t, registry.Add(ctx, ref, interfaces.BackgroundActivity{ID: "sj_1"}, time.Minute, 0))+ require.NoError(t, store.BeginTurn(ctx, key))+ active, rebuild, err := store.TurnState(ctx, key)+ require.NoError(t, err)+ require.True(t, active)+ require.False(t, rebuild)+ require.NoError(t, store.EndTurn(ctx, key))+ _, err = store.TryLockRewind(ctx, key)+ require.ErrorIs(t, err, sandbox.ErrSessionWorkActive)+}++type backgroundAllocationRepo struct {+ interfaces.SandboxAllocationRepository+ assignment *types.SessionSandboxAssignment+ machine *types.SandboxUserMachine+}++func (r *backgroundAllocationRepo) GetAssignment(context.Context, string) (*types.SessionSandboxAssignment, error) {+ return r.assignment, nil+}++func (r *backgroundAllocationRepo) GetUserMachine(context.Context, string) (*types.SandboxUserMachine, error) {+ return r.machine, nil+}++func TestBackgroundActivityRegistryUsesRecordedUserAllocation(t *testing.T) {+ ctx := t.Context()+ mini := miniredis.RunT(t)+ client := redis.NewClient(&redis.Options{Addr: mini.Addr()})+ t.Cleanup(func() { require.NoError(t, client.Close()) })+ store, err := sandbox.NewRedisSessionSandboxBindingStore(client, "test")+ require.NoError(t, err)+ a := allocation.Assignment{+ SessionID: "0f8fad5b-d9cb-469f-a165-70867728950e", ConfigTenantID: 42, ConfigID: "cfg-1",+ Machine: allocation.UserMachine{OwnerTenantID: 42, OwnerID: "u1", ConfigTenantID: 42, ConfigID: "cfg-1"},+ }+ row, machine := sandboxalloc.ToEntity(a, a.Machine.Digest("test"))+ allocations := sandboxalloc.New(&backgroundAllocationRepo{assignment: row, machine: machine}, nil, nil, "test")+ registry := newBackgroundActivityRegistry(store, allocations)+ ref := interfaces.SessionRef{TenantID: 42, SessionID: a.SessionID}+ require.NoError(t, registry.Add(ctx, ref, interfaces.BackgroundActivity{ID: "sj_1"}, time.Minute, 0))+ key := sandbox.SessionSandboxKey{TenantID: ref.TenantID, SessionID: ref.SessionID, User: a.Machine}+ active, rebuild, err := store.TurnState(ctx, key)+ require.NoError(t, err)+ require.True(t, active, "the production factory must address the same leases as the shared binding store")+ require.False(t, rebuild)+ _, err = store.TryLockRewind(ctx, key)+ require.ErrorIs(t, err, sandbox.ErrSessionWorkActive)+}diff --git a/internal/container/container.go b/internal/container/container.goindex 50493e5b4..3151e2875 100644--- a/internal/container/container.go+++ b/internal/container/container.go@@ -236,6 +236,7 @@ func BuildContainer(container *dig.Container) *dig.Container {return b}))must(container.Provide(newAdmissionRegistry))+ must(container.Provide(newSessionBindingStore))must(container.Provide(newTenantSandboxResolver))// The resolver admits user-mode sessions, so the allocation service has to// exist before the first Invoke that builds a TenantSandboxResolver@@ -247,6 +248,9 @@ func BuildContainer(container *dig.Container) *dig.Container {) *sandboxalloc.Service {return sandboxalloc.New(repo, configs, sessions, sandboxRedisNamespace())}))+ must(container.Provide(newBackgroundActivityRegistry))+ must(container.Provide(newBackgroundNoticeInbox))+ must(container.Provide(service.NewBackgroundWork))// Business service layerlogger.Debugf(ctx, "[Container] Registering business services...")@@ -436,11 +440,12 @@ func BuildContainer(container *dig.Container) *dig.Container {must(container.Provide(func(mgr sandbox.Manager,pinned *service.PinnedSessionSandbox,+ activity interfaces.ActivityRegistry,) *service.WorkspaceCheckpointer {if runner, ok := mgr.(service.SandboxShellRunner); ok {- return service.NewWorkspaceCheckpointer(runner)+ return service.NewWorkspaceCheckpointer(runner).WithActivityRegistry(activity)}- return service.NewWorkspaceCheckpointer(pinned)+ return service.NewWorkspaceCheckpointer(pinned).WithActivityRegistry(activity)}))must(container.Provide(func(mgr sandbox.Manager,diff --git a/internal/container/sandbox.go b/internal/container/sandbox.goindex b0c2080f9..7b86a9527 100644--- a/internal/container/sandbox.go+++ b/internal/container/sandbox.go@@ -71,6 +71,22 @@ func selectSessionBindingStore(return sandbox.NewMemorySessionSandboxBindingStore(), "memory", nil}+// newSessionBindingStore selects the process-wide session→sandbox binding+// store. The tenant sandbox resolver and the background activity registry+// share it: the registry's Redis hash must use the binding store's client,+// namespace and hash tag so the turn-lease scripts can read it. A nil result+// disables per-tenant sandbox configs, as a failed selection always has.+func newSessionBindingStore(redisClient *redis.Client) sandbox.SessionSandboxBindingStore {+ ctx := context.Background()+ store, storeKind, err := selectSessionBindingStore(redisClient, true)+ if err != nil {+ logger.Warnf(ctx, "Per-tenant sandbox config disabled: %v", err)+ return nil+ }+ logger.Infof(ctx, "Session sandbox binding store: %s", storeKind)+ return store+}+// sessionExistenceLookup is the narrow slice of SessionRepository the// session existence checker actually needs. Declaring it here (rather than// depending on interfaces.SessionRepository) keeps the checker easy to test@@ -133,7 +149,7 @@ func buildGlobalSandboxConfig() *sandbox.Config {func newTenantSandboxResolver(defaultManager sandbox.Manager,loader sandbox.TenantSandboxConfigLoader,- redisClient *redis.Client,+ store sandbox.SessionSandboxBindingStore,sessionRepo interfaces.SessionRepository,bootstrapper sandbox.SessionBootstrapper,alloc *sandboxalloc.Service,@@ -143,10 +159,8 @@ func newTenantSandboxResolver(// Tenants may configure any supported backend regardless of process startup// mode. Remote configs use this binding store for session persistence.- store, storeKind, err := selectSessionBindingStore(redisClient, true)- if err != nil {- logger.Warnf(ctx,- "Per-tenant sandbox config disabled: %v", err)+ if store == nil {+ logger.Warnf(ctx, "Per-tenant sandbox config disabled: no session binding store")return nil}resolver, err := sandbox.NewTenantSandboxResolver(sandbox.TenantSandboxResolverDeps{@@ -167,7 +181,7 @@ func newTenantSandboxResolver("(per-tenant sandbox config disabled)", err)return nil}- logger.Infof(ctx, "Tenant sandbox resolver configured: binding=%s", storeKind)+ logger.Infof(ctx, "Tenant sandbox resolver configured")return resolver}diff --git a/internal/handler/session/background.go b/internal/handler/session/background.gonew file mode 100644index 000000000..dd47516eb--- /dev/null+++ b/internal/handler/session/background.go@@ -0,0 +1,66 @@+package session++import (+ "context"++ "github.com/Tencent/WeKnora/internal/logger"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+)++// ackBackgroundNotices acknowledges the background notices a persisted+// assistant message carries: the ones injected into its turn and the ones its+// work tools returned. Until the message is saved they stay in the inbox, so a+// turn that never reached history delivers them again next run. ctx carries+// the session's tenant, as on every completeAssistantMessage path.+func (h *Handler) ackBackgroundNotices(ctx context.Context, msg *types.Message) {+ if h.background == nil || h.background.Inbox == nil || msg == nil {+ return+ }+ ids := backgroundNoticeIDs(msg.AgentSteps)+ if len(ids) == 0 {+ return+ }+ tenantID, ok := types.TenantIDFromContext(ctx)+ if !ok || tenantID == 0 {+ return+ }+ ref := interfaces.SessionRef{TenantID: tenantID, SessionID: msg.SessionID}+ if err := h.background.Inbox.Ack(ctx, ref, ids); err != nil {+ logger.Warnf(ctx, "ack background notices for message %s failed: %v", msg.ID, err)+ }+}++// backgroundNoticeIDs collects every notice ID the steps delivered or showed,+// once each, in order.+func backgroundNoticeIDs(steps []types.AgentStep) []string {+ var ids []string+ seen := make(map[string]bool)+ add := func(id string) {+ if id != "" && !seen[id] {+ seen[id] = true+ ids = append(ids, id)+ }+ }+ for _, step := range steps {+ for _, injection := range step.BackgroundInjections {+ for _, id := range injection.AckIDs {+ add(id)+ }+ }+ for _, id := range step.BackgroundSeenIDs {+ add(id)+ }+ }+ return ids+}++// notifyBackgroundStop tells background features the user stopped the turn.+// Each feature decides what that means for its own work.+func (h *Handler) notifyBackgroundStop(ctx context.Context, session *types.Session) {+ if h.background == nil || h.background.Listeners == nil || session == nil {+ return+ }+ h.background.Listeners.NotifyStop(context.WithoutCancel(ctx),+ interfaces.SessionRef{TenantID: session.TenantID, SessionID: session.ID})+}diff --git a/internal/handler/session/background_test.go b/internal/handler/session/background_test.gonew file mode 100644index 000000000..9505b43e0--- /dev/null+++ b/internal/handler/session/background_test.go@@ -0,0 +1,96 @@+package session++import (+ "context"+ "errors"+ "testing"+ "time"++ "github.com/Tencent/WeKnora/internal/application/service"+ "github.com/Tencent/WeKnora/internal/background"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/stretchr/testify/require"+)++type recordingSessionListener struct {+ stopped []interfaces.SessionRef+}++func (l *recordingSessionListener) OnStop(_ context.Context, s interfaces.SessionRef) error {+ l.stopped = append(l.stopped, s)+ return nil+}++func (l *recordingSessionListener) OnDelete(context.Context, interfaces.SessionRef) error { return nil }++func newAckFixture(t *testing.T) (*service.BackgroundWork, interfaces.SessionRef) {+ t.Helper()+ work := service.NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))+ ref := interfaces.SessionRef{TenantID: 1, SessionID: "s"}+ for _, id := range []string{"sj_1", "sj_2", "sj_3"} {+ _, err := work.Inbox.Push(context.Background(), ref, interfaces.BackgroundNotice{+ ID: id, Kind: "shell_job", Text: id, FinishedAt: time.Now(),+ })+ require.NoError(t, err)+ }+ return work, ref+}++func messageCarryingNotices() *types.Message {+ return &types.Message{+ ID: "m", SessionID: "s", Role: "assistant", Content: "done",+ AgentSteps: []types.AgentStep{+ {BackgroundInjections: []types.BackgroundInjection{{Content: "x", AckIDs: []string{"sj_1"}}}},+ {BackgroundSeenIDs: []string{"sj_2", "sj_1"}},+ },+ }+}++func pendingIDs(t *testing.T, work *service.BackgroundWork, ref interfaces.SessionRef) []string {+ t.Helper()+ notices, err := work.Inbox.Pending(context.Background(), ref)+ require.NoError(t, err)+ ids := make([]string, 0, len(notices))+ for _, n := range notices {+ ids = append(ids, n.ID)+ }+ return ids+}++func TestPersistedMessageAcknowledgesTheNoticesItCarries(t *testing.T) {+ work, ref := newAckFixture(t)+ h := &Handler{messageService: &imageCompletionMessages{}, background: work}+ ctx := types.WithExecutionTenant(context.Background(), 1)++ require.NoError(t, h.completeAssistantMessage(ctx, messageCarryingNotices(), "", ""))++ require.ElementsMatch(t, []string{"sj_3"}, pendingIDs(t, work, ref))+}++func TestFailedPersistLeavesNoticesForTheNextRun(t *testing.T) {+ work, ref := newAckFixture(t)+ h := &Handler{messageService: &imageCompletionMessages{err: errors.New("db down")}, background: work}+ ctx := types.WithExecutionTenant(context.Background(), 1)++ require.Error(t, h.completeAssistantMessage(ctx, messageCarryingNotices(), "", ""))++ require.ElementsMatch(t, []string{"sj_1", "sj_2", "sj_3"}, pendingIDs(t, work, ref))+}++func TestBackgroundNoticeIDsAreDeduplicatedInOrder(t *testing.T) {+ require.Equal(t, []string{"sj_1", "sj_2"}, backgroundNoticeIDs(messageCarryingNotices().AgentSteps))+ require.Empty(t, backgroundNoticeIDs(nil))+}++func TestStopNotifiesBackgroundListeners(t *testing.T) {+ work := service.NewBackgroundWork(nil, nil)+ listener := &recordingSessionListener{}+ work.Listeners.Register(listener)+ h := &Handler{background: work}++ h.notifyBackgroundStop(context.Background(), &types.Session{ID: "s", TenantID: 3})+ (&Handler{}).notifyBackgroundStop(context.Background(), &types.Session{ID: "s", TenantID: 3})++ require.Equal(t, []interfaces.SessionRef{{TenantID: 3, SessionID: "s"}}, listener.stopped)+}diff --git a/internal/handler/session/handler.go b/internal/handler/session/handler.goindex e0c0b0fd1..2ff476758 100644--- a/internal/handler/session/handler.go+++ b/internal/handler/session/handler.go@@ -80,6 +80,7 @@ type Handler struct {suspendedRuns *service.SuspendedRunServicestopCollaboration func(context.Context, string, string) errordelegatedPrompt func(context.Context, *types.AgentPrompt) bool+ background *service.BackgroundWork}// NewHandler creates a new instance of Handler with all necessary dependencies@@ -116,6 +117,7 @@ func NewHandler(forkService *service.SessionForkService,rewindService *service.SessionRewindService,approvedProjectDirs HostProjectDirsLoader,+ backgroundWork *service.BackgroundWork,) *Handler {h := &Handler{browserSkill: browserSkill,@@ -146,6 +148,7 @@ func NewHandler(desktopLast: desktopLast,redis: rdb,approvedProjectDirs: approvedProjectDirs,+ background: backgroundWork,attachmentProcessor: NewAttachmentProcessor(fileService,documentReader,diff --git a/internal/handler/session/qa.go b/internal/handler/session/qa.goindex 23d532f8e..74724bdb6 100644--- a/internal/handler/session/qa.go+++ b/internal/handler/session/qa.go@@ -2168,6 +2168,7 @@ func (h *Handler) completeAssistantMessage(logger.Errorf(ctx, "Failed to persist assistant message %s: %v", assistantMessage.ID, err)return err}+ h.ackBackgroundNotices(ctx, assistantMessage)// Asynchronously index the Q&A pair into the chat history knowledge base for vector search.// Use WithoutCancel so the goroutine survives after the HTTP request context is done.diff --git a/internal/handler/session/stream.go b/internal/handler/session/stream.goindex 4dd2c94d7..9c052cf0b 100644--- a/internal/handler/session/stream.go+++ b/internal/handler/session/stream.go@@ -349,6 +349,7 @@ func (h *Handler) StopSession(c *gin.Context) {}logger.Infof(ctx, "Stop event written successfully for session: %s, message: %s", sessionID, assistantMessageID)+ h.notifyBackgroundStop(ctx, session)c.JSON(200, gin.H{"success": true,"message": "Generation stopped",diff --git a/internal/modelcontext/tool_policy.go b/internal/modelcontext/tool_policy.goindex f4657cb57..688debcb5 100644--- a/internal/modelcontext/tool_policy.go+++ b/internal/modelcontext/tool_policy.go@@ -222,7 +222,10 @@ var toolHandlePolicies = map[string]toolHandlePolicy{"execute_skill_script": {},// Sandbox shell/file tools return whatever a skill script produced, so they// carry the same unknown-shape caveat as skill output above.- "shell_exec": {},+ "shell_exec": {},+ // shell_exec_async results carry opaque work handles (sj_...) that the+ // work_wait / work_cancel tools consume; they are not source identities.+ "shell_exec_async": {},"list_sandbox_files": {},"read_sandbox_file": {},"write_sandbox_file": {},@@ -231,6 +234,10 @@ var toolHandlePolicies = map[string]toolHandlePolicy{// caveat: they echo paths and byte counts, never knowledge identifiers."write_skill_file": {},"edit_skill_file": {},+ // Background-work tools echo work IDs, states and command output; they+ // never carry knowledge identifiers.+ "work_wait": {},+ "work_cancel": {},}// HasToolPolicy reports whether a tool has an explicit model-handle policy.diff --git a/internal/sandbox/activity_registry_memory.go b/internal/sandbox/activity_registry_memory.gonew file mode 100644index 000000000..9a45d7c2a--- /dev/null+++ b/internal/sandbox/activity_registry_memory.go@@ -0,0 +1,85 @@+package sandbox++import (+ "context"+ "time"++ "github.com/Tencent/WeKnora/internal/types/interfaces"+)++// MemoryActivityRegistry shares the binding store's background leases and lock,+// so single-process deployments also protect live work from rebuild and rewind.+type MemoryActivityRegistry struct {+ store *MemorySessionSandboxBindingStore+}++var _ interfaces.ActivityRegistry = (*MemoryActivityRegistry)(nil)++// NewMemoryActivityRegistry returns the activity registry of store.+func NewMemoryActivityRegistry(store *MemorySessionSandboxBindingStore) *MemoryActivityRegistry {+ return &MemoryActivityRegistry{store: store}+}++func memoryActivityKey(ctx context.Context, s interfaces.SessionRef) (SessionSandboxKey, error) {+ key := SessionSandboxKey{TenantID: s.TenantID, SessionID: s.SessionID}+ if err := key.Validate(); err != nil {+ return key, err+ }+ return key, ctx.Err()+}++// Add implements interfaces.ActivityRegistry.+func (r *MemoryActivityRegistry) Add(+ ctx context.Context, s interfaces.SessionRef, a interfaces.BackgroundActivity, lease time.Duration, kindCap int,+) error {+ key, err := memoryActivityKey(ctx, s)+ if err != nil {+ return err+ }+ r.store.mu.Lock()+ defer r.store.mu.Unlock()+ if _, held := r.store.rewinds[key]; held {+ return ErrSessionRewindLocked+ }+ return r.store.activity.Add(ctx, s, a, lease, kindCap)+}++// Renew implements interfaces.ActivityRegistry.+func (r *MemoryActivityRegistry) Renew(+ ctx context.Context, s interfaces.SessionRef, id string, lease time.Duration,+) (bool, error) {+ if _, err := memoryActivityKey(ctx, s); err != nil {+ return false, err+ }+ r.store.mu.Lock()+ defer r.store.mu.Unlock()+ return r.store.activity.Renew(ctx, s, id, lease)+}++// Remove implements interfaces.ActivityRegistry.+func (r *MemoryActivityRegistry) Remove(ctx context.Context, s interfaces.SessionRef, id string) error {+ if _, err := memoryActivityKey(ctx, s); err != nil {+ return err+ }+ r.store.mu.Lock()+ defer r.store.mu.Unlock()+ return r.store.activity.Remove(ctx, s, id)+}++// List implements interfaces.ActivityRegistry.+func (r *MemoryActivityRegistry) List(+ ctx context.Context, s interfaces.SessionRef,+) ([]interfaces.BackgroundActivity, error) {+ if _, err := memoryActivityKey(ctx, s); err != nil {+ return nil, err+ }+ r.store.mu.Lock()+ defer r.store.mu.Unlock()+ return r.store.activity.List(ctx, s)+}++// Caller holds s.mu, serializing the lease check with activity registration.+func (s *MemorySessionSandboxBindingStore) hasSessionActivityLocked(ctx context.Context, key SessionSandboxKey) bool {+ items, err := s.activity.List(ctx, interfaces.SessionRef{TenantID: key.TenantID, SessionID: key.SessionID})+ return err != nil || len(items) > 0+}diff --git a/internal/sandbox/activity_registry_memory_test.go b/internal/sandbox/activity_registry_memory_test.gonew file mode 100644index 000000000..4c128e753--- /dev/null+++ b/internal/sandbox/activity_registry_memory_test.go@@ -0,0 +1,100 @@+package sandbox++import (+ "sync"+ "testing"+ "time"++ "github.com/Tencent/WeKnora/internal/background"+ "github.com/Tencent/WeKnora/internal/background/backgroundtest"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/stretchr/testify/require"+)++type activityTestClock struct {+ mu sync.Mutex+ now time.Time+}++func (c *activityTestClock) Now() time.Time {+ c.mu.Lock()+ defer c.mu.Unlock()+ return c.now+}++func (c *activityTestClock) advance(d time.Duration) {+ c.mu.Lock()+ defer c.mu.Unlock()+ c.now = c.now.Add(d)+}++func TestBindingMemoryActivityRegistryContract(t *testing.T) {+ backgroundtest.RunActivityRegistryContract(t,+ func(_ *testing.T) (interfaces.ActivityRegistry, backgroundtest.Advance) {+ store := NewMemorySessionSandboxBindingStore()+ clock := &activityTestClock{now: time.Now()}+ store.activity = background.NewMemoryActivityRegistry(clock)+ return NewMemoryActivityRegistry(store), clock.advance+ })+}++func TestMemoryActivityProtectsStaleSandboxAcrossTurns(t *testing.T) {+ for _, release := range []string{"remove", "expire"} {+ t.Run(release, func(t *testing.T) {+ ctx := t.Context()+ fx := newLifecycleFixture(t)+ clock := &activityTestClock{now: time.Now()}+ fx.bindings.activity = background.NewMemoryActivityRegistry(clock)+ registry := NewMemoryActivityRegistry(fx.bindings)+ ref := interfaces.SessionRef{TenantID: fx.key.TenantID, SessionID: fx.key.SessionID}+ first, err := fx.lifecycle.Resolve(ctx, fx.key)+ require.NoError(t, err)+ require.NoError(t, registry.Add(ctx, ref, interfaces.BackgroundActivity{ID: "sj_1"}, time.Minute, 0))+ _, err = fx.bindings.InvalidateByConfig(ctx, fx.key.TenantID, "cfg-1")+ require.NoError(t, err)++ active, rebuild, err := fx.bindings.TurnState(ctx, fx.key)+ require.NoError(t, err)+ require.True(t, active, "background work stays active after the submitting turn")+ require.False(t, rebuild)+ require.NoError(t, fx.bindings.BeginTurn(ctx, fx.key))+ kept, err := fx.lifecycle.Resolve(ctx, fx.key)+ require.NoError(t, err)+ require.Equal(t, first.ID(), kept.ID())+ require.Empty(t, fx.client.deleteIDs)+ require.NoError(t, fx.bindings.EndTurn(ctx, fx.key))++ if release == "remove" {+ require.NoError(t, registry.Remove(ctx, ref, "sj_1"))+ } else {+ clock.advance(time.Minute)+ }+ require.NoError(t, fx.bindings.BeginTurn(ctx, fx.key))+ replaced, err := fx.lifecycle.Resolve(ctx, fx.key)+ require.NoError(t, err)+ require.NotEqual(t, first.ID(), replaced.ID(), "completed or expired work must release the rebuild guard")+ require.Equal(t, []string{first.ID()}, fx.client.deleteIDs)+ require.NoError(t, fx.bindings.EndTurn(ctx, fx.key))+ })+ }+}++func TestMemoryActivityAndRewindAreExclusive(t *testing.T) {+ ctx := t.Context()+ store := NewMemorySessionSandboxBindingStore()+ registry := NewMemoryActivityRegistry(store)+ key := SessionSandboxKey{TenantID: 42, SessionID: "session-a"}+ ref := interfaces.SessionRef{TenantID: key.TenantID, SessionID: key.SessionID}+ item := interfaces.BackgroundActivity{ID: "sj_1"}+ require.NoError(t, registry.Add(ctx, ref, item, time.Minute, 0))+ _, err := store.TryLockRewind(ctx, key)+ require.ErrorIs(t, err, ErrSessionWorkActive)+ require.NoError(t, registry.Remove(ctx, ref, item.ID))+ unlock, err := store.TryLockRewind(ctx, key)+ require.NoError(t, err)+ defer unlock()+ require.ErrorIs(t, registry.Add(ctx, ref, item, time.Minute, 0), ErrSessionRewindLocked)+ items, err := registry.List(ctx, ref)+ require.NoError(t, err)+ require.Empty(t, items)+}diff --git a/internal/sandbox/activity_registry_redis.go b/internal/sandbox/activity_registry_redis.gonew file mode 100644index 000000000..033496d92--- /dev/null+++ b/internal/sandbox/activity_registry_redis.go@@ -0,0 +1,297 @@+package sandbox++import (+ "context"+ "encoding/json"+ "fmt"+ "strings"+ "time"++ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/redis/go-redis/v9"+)++// activityKeyGrace keeps the activity hash around briefly after its last+// deadline so a read racing the expiry still sees a well-formed key.+const activityKeyGrace = 60 * time.Second++// activityLiveCountLua defines now_ms() and live_count(key, now_ms): the+// latter deletes expired fields of the activity hash and returns how many+// remain. The turn-lease scripts in session_binding_redis.go prepend it so+// they can refuse a rebuild or a rewind while items are live.+const activityLiveCountLua = `+local function now_ms()+ local t = redis.call('TIME')+ return tonumber(t[1]) * 1000 + math.floor(tonumber(t[2]) / 1000)+end+local function live_count(key, now)+ local raw = redis.call('HGETALL', key)+ local live = 0+ for i = 1, #raw, 2 do+ local v = cjson.decode(raw[i + 1])+ if tonumber(v['deadline_ms']) <= now then+ redis.call('HDEL', key, raw[i])+ else+ live = live + 1+ end+ end+ return live+end+`++// Every write refreshes the hash until its latest member expires. A shared+// machine mirrors each session's leases in a machine hash, in the same script+// and Redis slot, so sibling sessions cannot rebuild underneath them.+const activityWriteLua = `+local function save_activity(key, id, value, now, grace)+ redis.call('HSET', key, id, cjson.encode(value))+ local raw = redis.call('HGETALL', key)+ local maxd = now+ for i = 1, #raw, 2 do+ local d = tonumber(cjson.decode(raw[i + 1])['deadline_ms'])+ if d > maxd then maxd = d end+ end+ redis.call('PEXPIRE', key, maxd - now + grace)+end+`++// KEYS: session activity, machine activity, rewind, machine close, session close.+// ARGV[1] id, ARGV[2] kind, ARGV[3] title,+// ARGV[4] started_at_ms, ARGV[5] lease_ms, ARGV[6] kind cap (<=0 unlimited),+// ARGV[7] key grace ms, ARGV[8] machine member. Returns 1 on success, 0 at the cap.+var addActivityScript = redis.NewScript(activityLiveCountLua + activityWriteLua + `+local now = now_ms()+live_count(KEYS[1], now)+local deadline = now + tonumber(ARGV[5])+local existing = redis.call('HGET', KEYS[1], ARGV[1])+local v+if existing then+ v = cjson.decode(existing)+ v['deadline_ms'] = deadline+else+ if redis.call('EXISTS', KEYS[3]) == 1 then+ return redis.error_reply('ERR_SESSION_REWIND_LOCKED')+ end+ if redis.call('EXISTS', KEYS[4]) == 1 or redis.call('EXISTS', KEYS[5]) == 1 then+ return redis.error_reply('ERR_SCOPE_CLOSED')+ end+ local cap = tonumber(ARGV[6])+ if cap > 0 then+ local raw = redis.call('HGETALL', KEYS[1])+ local same = 0+ for i = 1, #raw, 2 do+ if cjson.decode(raw[i + 1])['kind'] == ARGV[2] then+ same = same + 1+ end+ end+ if same >= cap then+ return 0+ end+ end+ v = {+ kind = ARGV[2], title = ARGV[3],+ started_at_ms = tonumber(ARGV[4]), deadline_ms = deadline,+ }+end+save_activity(KEYS[1], ARGV[1], v, now, tonumber(ARGV[7]))+if KEYS[2] ~= KEYS[1] then+ live_count(KEYS[2], now)+ save_activity(KEYS[2], ARGV[8], v, now, tonumber(ARGV[7]))+end+return 1+`)++// KEYS: session activity, machine activity.+// ARGV: id, lease_ms, key grace ms, machine member.+// Returns 1 when renewed, 0 when the item is missing or expired.+var renewActivityScript = redis.NewScript(activityLiveCountLua + activityWriteLua + `+local now = now_ms()+live_count(KEYS[1], now)+local existing = redis.call('HGET', KEYS[1], ARGV[1])+if not existing then+ if KEYS[2] ~= KEYS[1] then redis.call('HDEL', KEYS[2], ARGV[4]) end+ return 0+end+local v = cjson.decode(existing)+v['deadline_ms'] = now + tonumber(ARGV[2])+save_activity(KEYS[1], ARGV[1], v, now, tonumber(ARGV[3]))+if KEYS[2] ~= KEYS[1] then+ live_count(KEYS[2], now)+ save_activity(KEYS[2], ARGV[4], v, now, tonumber(ARGV[3]))+end+return 1+`)++var removeActivityScript = redis.NewScript(`+redis.call('HDEL', KEYS[1], ARGV[1])+if KEYS[2] ~= KEYS[1] then redis.call('HDEL', KEYS[2], ARGV[2]) end+return 1+`)++var countActivityScript = redis.NewScript(activityLiveCountLua + `+return live_count(KEYS[1], now_ms())+`)++// KEYS[1] activity hash. Returns the live fields as a flat [id, json, ...] list.+var listActivityScript = redis.NewScript(activityLiveCountLua + `+live_count(KEYS[1], now_ms())+return redis.call('HGETALL', KEYS[1])+`)++type redisActivityValue struct {+ Kind string `json:"kind"`+ Title string `json:"title"`+ StartedAtMS int64 `json:"started_at_ms"`+ DeadlineMS int64 `json:"deadline_ms"`+}++// RedisActivityRegistry is the multi-replica ActivityRegistry. It shares the+// binding store's client and hash tag so the turn-lease scripts can read it.+type RedisActivityRegistry struct {+ store *RedisSessionSandboxBindingStore+ allocations AllocationDirectory+}++var _ interfaces.ActivityRegistry = (*RedisActivityRegistry)(nil)++// NewRedisActivityRegistry returns a registry backed by store's Redis client.+func NewRedisActivityRegistry(store *RedisSessionSandboxBindingStore) *RedisActivityRegistry {+ return &RedisActivityRegistry{store: store}+}++// WithAllocations uses each session's recorded placement to address shared+// machine leases. Configure it before using the registry.+func (r *RedisActivityRegistry) WithAllocations(allocations AllocationDirectory) *RedisActivityRegistry {+ r.allocations = allocations+ return r+}++func (s *RedisSessionSandboxBindingStore) activityKey(key SessionSandboxKey) string {+ if key.IsUserScoped() && key.SessionID != "" {+ return s.keyBase(key) + ":session:" + key.SessionID + ":activity"+ }+ return s.keyBase(key) + ":activity"+}++func (r *RedisActivityRegistry) key(ctx context.Context, s interfaces.SessionRef) (SessionSandboxKey, error) {+ k := SessionSandboxKey{TenantID: s.TenantID, SessionID: s.SessionID}+ if err := k.Validate(); err != nil {+ return k, err+ }+ if r.allocations != nil {+ a, found, err := r.allocations.SessionAllocation(ctx, s.SessionID)+ if err != nil {+ return k, fmt.Errorf("read background work allocation: %w", err)+ }+ if found && a.IsUser() {+ k.User = a.Machine+ }+ }+ return k, k.Validate()+}++func (r *RedisActivityRegistry) keys(key SessionSandboxKey) []string {+ return []string{r.store.activityKey(key), r.store.activityKey(key.Machine())}+}++func activityMachineMember(key SessionSandboxKey, id string) string {+ return key.SessionID + ":" + id+}++// hasBackgroundActivity includes every sibling session on a shared machine.+func (s *RedisSessionSandboxBindingStore) hasBackgroundActivity(+ ctx context.Context, key SessionSandboxKey,+) (bool, error) {+ n, err := countActivityScript.Run(ctx, s.client, []string{s.activityKey(key.Machine())}).Int64()+ return n > 0, err+}++// Add implements interfaces.ActivityRegistry.+func (r *RedisActivityRegistry) Add(+ ctx context.Context, s interfaces.SessionRef, a interfaces.BackgroundActivity, lease time.Duration, kindCap int,+) error {+ key, err := r.key(ctx, s)+ if err != nil {+ return err+ }+ keys := append(r.keys(key), r.store.rewindKey(key))+ if key.IsUserScoped() {+ base := r.store.keyBase(key)+ keys = append(keys, base+":machine:closed", base+":session:"+key.SessionID+":closed")+ } else {+ keys = append(keys, r.store.rewindKey(key), r.store.rewindKey(key))+ }+ ok, err := addActivityScript.Run(ctx, r.store.client, keys,+ a.ID, a.Kind, a.Title, a.StartedAt.UnixMilli(), lease.Milliseconds(), kindCap,+ activityKeyGrace.Milliseconds(), activityMachineMember(key, a.ID),+ ).Int64()+ if err != nil {+ if isRewindLockedRedisErr(err) {+ return ErrSessionRewindLocked+ }+ if strings.Contains(err.Error(), "ERR_SCOPE_CLOSED") {+ return ErrSessionWorkspaceBusy+ }+ return fmt.Errorf("add background activity: %w", err)+ }+ if ok == 0 {+ return interfaces.ErrActivityLimit+ }+ return nil+}++// Renew implements interfaces.ActivityRegistry.+func (r *RedisActivityRegistry) Renew(+ ctx context.Context, s interfaces.SessionRef, id string, lease time.Duration,+) (bool, error) {+ key, err := r.key(ctx, s)+ if err != nil {+ return false, err+ }+ ok, err := renewActivityScript.Run(ctx, r.store.client, r.keys(key),+ id, lease.Milliseconds(), activityKeyGrace.Milliseconds(), activityMachineMember(key, id),+ ).Int64()+ if err != nil {+ return false, fmt.Errorf("renew background activity: %w", err)+ }+ return ok == 1, nil+}++// Remove implements interfaces.ActivityRegistry.+func (r *RedisActivityRegistry) Remove(ctx context.Context, s interfaces.SessionRef, id string) error {+ key, err := r.key(ctx, s)+ if err != nil {+ return err+ }+ err = removeActivityScript.Run(ctx, r.store.client, r.keys(key), id, activityMachineMember(key, id)).Err()+ if err != nil {+ return fmt.Errorf("remove background activity: %w", err)+ }+ return nil+}++// List implements interfaces.ActivityRegistry.+func (r *RedisActivityRegistry) List(+ ctx context.Context, s interfaces.SessionRef,+) ([]interfaces.BackgroundActivity, error) {+ key, err := r.key(ctx, s)+ if err != nil {+ return nil, err+ }+ flat, err := listActivityScript.Run(ctx, r.store.client, []string{r.store.activityKey(key)}).StringSlice()+ if err != nil {+ return nil, fmt.Errorf("list background activity: %w", err)+ }+ out := make([]interfaces.BackgroundActivity, 0, len(flat)/2)+ for i := 0; i+1 < len(flat); i += 2 {+ var v redisActivityValue+ if err := json.Unmarshal([]byte(flat[i+1]), &v); err != nil {+ return nil, fmt.Errorf("decode background activity %s: %w", flat[i], err)+ }+ out = append(out, interfaces.BackgroundActivity{+ ID: flat[i], Kind: v.Kind, Title: v.Title,+ StartedAt: time.UnixMilli(v.StartedAtMS).UTC(),+ })+ }+ return out, nil+}diff --git a/internal/sandbox/activity_registry_redis_test.go b/internal/sandbox/activity_registry_redis_test.gonew file mode 100644index 000000000..e89bce7f4--- /dev/null+++ b/internal/sandbox/activity_registry_redis_test.go@@ -0,0 +1,215 @@+package sandbox++import (+ "context"+ "testing"+ "time"++ "github.com/Tencent/WeKnora/internal/background/backgroundtest"+ "github.com/Tencent/WeKnora/internal/sandbox/admission"+ "github.com/Tencent/WeKnora/internal/sandbox/allocation"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/alicebob/miniredis/v2"+ "github.com/stretchr/testify/require"+)++func TestRedisActivityRegistryContract(t *testing.T) {+ backgroundtest.RunActivityRegistryContract(t,+ func(t *testing.T) (interfaces.ActivityRegistry, backgroundtest.Advance) {+ store, _, mini := newRedisBindingTestStore(t)+ now := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC)+ mini.SetTime(now)+ advance := func(d time.Duration) {+ now = now.Add(d)+ mini.SetTime(now)+ mini.FastForward(d) // also expires key TTLs+ }+ return NewRedisActivityRegistry(store), advance+ })+}++func TestRedisActivityRegistryUsesBindingHashTag(t *testing.T) {+ store, _, _ := newRedisBindingTestStore(t)+ key := SessionSandboxKey{TenantID: 7, SessionID: "s1"}+ require.Equal(t, "weknora:sandbox:session:{test:7:s1}:activity", store.activityKey(key))+ require.Equal(t, "weknora:sandbox:session:{test:7:s1}:turn", store.turnKey(key))+}++func TestRedisActivityRegistryRejectsInvalidSession(t *testing.T) {+ store, _, _ := newRedisBindingTestStore(t)+ r := NewRedisActivityRegistry(store)+ err := r.Add(t.Context(), interfaces.SessionRef{}, interfaces.BackgroundActivity{ID: "x"}, time.Minute, 0)+ require.Error(t, err)+}++// These managers use separate stores on the same Redis, as two replicas do.+func newSharedActivityManagers(+ t *testing.T,+) (*userHarness, *userHarness, *RedisActivityRegistry, *miniredis.Miniredis) {+ t.Helper()+ store, client, mini := newRedisBindingTestStore(t)+ other, err := NewRedisSessionSandboxBindingStore(client, store.namespace)+ require.NoError(t, err)+ allocs := newFakeAllocations(userAssignmentFor(userSessionA), userAssignmentFor(userSessionB))+ remote := newFakeRemoteClient(SandboxTypeCube)+ h1 := newUserManagerHarnessWithDirectory(t, remote, store,+ admission.NewRedis(client, "weknora:sandbox:user:"), allocs, store.namespace)+ h2 := newUserManagerHarnessWithDirectory(t, remote, other,+ admission.NewRedis(client, "weknora:sandbox:user:"), allocs, store.namespace)+ return h1, h2, NewRedisActivityRegistry(store).WithAllocations(allocs), mini+}++func useActivityTestSandbox(t *testing.T, mgr *SessionBoundManager, sessionID string) string {+ t.Helper()+ ctx, end, err := mgr.BeginSessionTurn(ownerCtx(), sessionID)+ require.NoError(t, err)+ defer end()+ _, err = mgr.ExecShellCommand(ctx, sessionID, "true", "", time.Second, nil)+ require.NoError(t, err)+ id, bound := mgr.BoundSandboxID(ctx, sessionID)+ require.True(t, bound)+ return id+}++func TestRedisActivityProtectsSharedMachineFromSiblingRebuild(t *testing.T) {+ for _, release := range []string{"remove", "expire"} {+ t.Run(release, func(t *testing.T) {+ h1, h2, registry, mini := newSharedActivityManagers(t)+ now := time.Now()+ mini.SetTime(now)+ ctx := ownerCtx()+ first := useActivityTestSandbox(t, h1.mgr, userSessionA)+ ref := interfaces.SessionRef{TenantID: 42, SessionID: userSessionA}+ require.NoError(t, registry.Add(ctx, ref, interfaces.BackgroundActivity{ID: "sj_1"}, time.Minute, 0))+ n, err := h1.mgr.InvalidateConfigSandboxesForMachines(ctx, "cfg-1",+ []allocation.UserMachine{userAssignmentFor(userSessionA).Machine})+ require.NoError(t, err)+ require.Equal(t, 1, n)++ require.Equal(t, first, useActivityTestSandbox(t, h2.mgr, userSessionB),+ "another replica's sibling turn must not destroy the background job's machine")+ require.Equal(t, first, useActivityTestSandbox(t, h2.mgr, userSessionA),+ "the submitting session's next turn must also keep its machine")+ require.Empty(t, h1.client.deleteIDs)++ if release == "remove" {+ require.NoError(t, registry.Remove(ctx, ref, "sj_1"))+ } else {+ mini.SetTime(now.Add(time.Minute))+ mini.FastForward(time.Minute)+ }+ require.NotEqual(t, first, useActivityTestSandbox(t, h2.mgr, userSessionB))+ require.Equal(t, []string{first}, h1.client.deleteIDs)+ })+ }+}++type activityBeforeCloseRegistry struct {+ admission.Registry+ beforeClose func()+}++func (r *activityBeforeCloseRegistry) Close(+ ctx context.Context, scope admission.Scope, ttl time.Duration,+) (admission.Barrier, error) {+ if r.beforeClose != nil {+ f := r.beforeClose+ r.beforeClose = nil+ f()+ }+ return r.Registry.Close(ctx, scope, ttl)+}++func TestSharedRebuildRechecksActivityAfterClosingAdmission(t *testing.T) {+ h1, h2, registry, _ := newSharedActivityManagers(t)+ ctx := ownerCtx()+ first := useActivityTestSandbox(t, h1.mgr, userSessionA)+ _, err := h1.mgr.InvalidateConfigSandboxesForMachines(ctx, "cfg-1",+ []allocation.UserMachine{userAssignmentFor(userSessionA).Machine})+ require.NoError(t, err)+ ref := interfaces.SessionRef{TenantID: 42, SessionID: userSessionA}+ raced := false+ h2.mgr.lifecycle.admission = &activityBeforeCloseRegistry{+ Registry: h2.mgr.admission,+ beforeClose: func() {+ raced = true+ require.NoError(t, registry.Add(ctx, ref, interfaces.BackgroundActivity{ID: "sj_1"}, time.Minute, 0))+ },+ }+ require.Equal(t, first, useActivityTestSandbox(t, h2.mgr, userSessionB))+ require.True(t, raced, "register between the optimistic check and the machine barrier")+ require.Empty(t, h1.client.deleteIDs)+}++func TestRedisActivityCannotRegisterThroughClosedAdmission(t *testing.T) {+ for _, scopeKind := range []string{"machine", "session", "rewind"} {+ t.Run(scopeKind, func(t *testing.T) {+ h, _, registry, _ := newSharedActivityManagers(t)+ ctx := ownerCtx()+ ref := interfaces.SessionRef{TenantID: 42, SessionID: userSessionA}+ key, err := registry.key(ctx, ref)+ require.NoError(t, err)+ want := ErrSessionWorkspaceBusy+ if scopeKind == "rewind" {+ unlock, err := registry.store.TryLockRewind(ctx, key)+ require.NoError(t, err)+ defer unlock()+ want = ErrSessionRewindLocked+ } else {+ digest := key.User.Digest(registry.store.namespace)+ scope := admission.MachineScope(digest)+ if scopeKind == "session" {+ scope = admission.SessionScope(digest, ref.SessionID)+ }+ barrier, err := h.mgr.admission.Close(ctx, scope, time.Minute)+ require.NoError(t, err)+ defer barrier.Release()+ }+ require.ErrorIs(t, registry.Add(ctx, ref, interfaces.BackgroundActivity{ID: "sj_1"}, time.Minute, 0), want)+ items, err := registry.List(ctx, ref)+ require.NoError(t, err)+ require.Empty(t, items)+ active, err := registry.store.hasBackgroundActivity(ctx, key)+ require.NoError(t, err)+ require.False(t, active)+ })+ }+}++func TestRedisSharedActivityLeasesRemainSessionScoped(t *testing.T) {+ _, _, registry, mini := newSharedActivityManagers(t)+ ctx := ownerCtx()+ now := time.Now()+ mini.SetTime(now)+ a := interfaces.SessionRef{TenantID: 42, SessionID: userSessionA}+ b := interfaces.SessionRef{TenantID: 42, SessionID: userSessionB}+ key, err := registry.key(ctx, a)+ require.NoError(t, err)+ item := interfaces.BackgroundActivity{ID: "sj_1", Kind: "shell_job"}+ require.NoError(t, registry.Add(ctx, a, item, 10*time.Second, 1))+ items, err := registry.List(ctx, b)+ require.NoError(t, err)+ require.Empty(t, items)+ // Caps and IDs are session scoped even on the same machine.+ require.NoError(t, registry.Add(ctx, b, item, 10*time.Second, 1))+ _, err = registry.store.TryLockRewind(ctx, key)+ require.ErrorIs(t, err, ErrSessionWorkActive)+ require.NoError(t, registry.Remove(ctx, a, item.ID))+ unlock, err := registry.store.TryLockRewind(ctx, key)+ require.NoError(t, err, "a sibling's work must not block this session's rewind")+ unlock()+ mini.SetTime(now.Add(8 * time.Second))+ mini.FastForward(8 * time.Second)+ found, err := registry.Renew(ctx, b, item.ID, time.Minute)+ require.NoError(t, err)+ require.True(t, found)+ mini.SetTime(now.Add(20 * time.Second))+ mini.FastForward(12 * time.Second)+ active, err := registry.store.hasBackgroundActivity(ctx, key)+ require.NoError(t, err)+ require.True(t, active, "renew must extend the shared machine guard too")+ require.NoError(t, registry.Remove(ctx, b, item.ID))+ active, err = registry.store.hasBackgroundActivity(ctx, key)+ require.NoError(t, err)+ require.False(t, active)+}diff --git a/internal/sandbox/cube_remote_client.go b/internal/sandbox/cube_remote_client.goindex 158bfc26a..ed82bae48 100644--- a/internal/sandbox/cube_remote_client.go+++ b/internal/sandbox/cube_remote_client.go@@ -945,6 +945,14 @@ func (c *CubeRemoteClient) Exec("cube returned an empty command result", nil,)}+ if request.OnOutput != nil {+ if sdkResult.Stdout != "" {+ request.OnOutput("stdout", []byte(sdkResult.Stdout))+ }+ if sdkResult.Stderr != "" {+ request.OnOutput("stderr", []byte(sdkResult.Stderr))+ }+ }return &RemoteExecResult{Stdout: sdkResult.Stdout,Stderr: sdkResult.Stderr,@@ -1553,6 +1561,20 @@ func (c *CubeRemoteClient) StartDesktopTTLRefresh(ctx context.Context, handle Re})}+// KeepAlive moves the sandbox's idle timeout to ttl from now. Cube already+// counts commands as activity; the explicit call keeps background work from+// depending on which requests the control plane happens to count.+func (c *CubeRemoteClient) KeepAlive(ctx context.Context, handle RemoteSandboxHandle, ttl time.Duration) error {+ sb, err := cubeHandleSandbox("KeepAlive", handle)+ if err != nil {+ return err+ }+ if ttl <= 0 {+ return nil+ }+ return sb.SetTimeout(ctx, ttl)+}+var (_ RemoteSandboxClient = (*CubeRemoteClient)(nil)_ RemoteSnapshotManager = (*CubeRemoteClient)(nil)diff --git a/internal/sandbox/cube_remote_client_test.go b/internal/sandbox/cube_remote_client_test.goindex 03f9ad524..2096f41ee 100644--- a/internal/sandbox/cube_remote_client_test.go+++ b/internal/sandbox/cube_remote_client_test.go@@ -532,6 +532,30 @@ func TestCubeRemoteClientExecArgvAndShell(t *testing.T) {require.True(t, IsRemoteInvalidRequest(err))}+func TestCubeRemoteClientExecForwardsBufferedOutput(t *testing.T) {+ mock := newCubeMockServer(t)+ mock.SetExecutor(func(string, string, []string) (string, string, int) {+ return "out-line\n", "err-line\n", 0+ })+ client := newTestCubeRemoteClient(t, mock)+ ctx := context.Background()+ handle, err := client.Create(ctx, RemoteCreateRequest{TemplateID: "template-a"})+ require.NoError(t, err)++ var got []string+ result, err := client.Exec(ctx, handle, RemoteExecRequest{+ Command: "printf '%s' ok",+ Shell: true,+ OnOutput: func(stream string, chunk []byte) {+ got = append(got, stream+":"+string(chunk))+ },+ })+ require.NoError(t, err)+ require.Equal(t, "out-line\n", result.Stdout)+ require.Equal(t, "err-line\n", result.Stderr)+ require.Equal(t, []string{"stdout:out-line\n", "stderr:err-line\n"}, got)+}+func TestCubeRemoteClientExecTimeoutIsKilled(t *testing.T) {mock := newCubeMockServer(t)// A slow executor lets the outer request timeout fire before the mockdiff --git a/internal/sandbox/docker_remote_client.go b/internal/sandbox/docker_remote_client.goindex 106b752e5..30d303b79 100644--- a/internal/sandbox/docker_remote_client.go+++ b/internal/sandbox/docker_remote_client.go@@ -725,6 +725,19 @@ func (c *DockerRemoteClient) Exec(return result, nil}+// dockerKeepAliveTimeout bounds the no-op exec KeepAlive runs.+const dockerKeepAliveTimeout = 10 * time.Second++// KeepAlive runs a no-op exec. dockerExecCommand touches the activity marker on+// every exec, and the marker's mtime is all the idle sweeper reads, so ttl is+// not needed here.+func (c *DockerRemoteClient) KeepAlive(ctx context.Context, handle RemoteSandboxHandle, _ time.Duration) error {+ _, err := c.Exec(ctx, handle, RemoteExecRequest{+ Command: "true", Timeout: dockerKeepAliveTimeout, User: DefaultSandboxExecUser,+ })+ return err+}+// dockerExecGrace is the slack between the in-container timeout and the// client-side deadline. It covers the round-trip and the wrapper's own// teardown so a script killed at the deadline is still reported as Killeddiff --git a/internal/sandbox/e2b_remote_client.go b/internal/sandbox/e2b_remote_client.goindex 6b0ef8be9..94b7ba5a0 100644--- a/internal/sandbox/e2b_remote_client.go+++ b/internal/sandbox/e2b_remote_client.go@@ -1164,7 +1164,8 @@ func (c *E2BRemoteClient) MakeDir(return e2bInvalidRequest("MakeDir", "path is required", nil)}return makeDirTree(dir, func(component string) error {- return normalizeE2BError("MakeDir", sandbox.Filesystem.MakeDir(ctx, component, e2b.WithFileUser(remoteFileUser(ctx))))+ err := sandbox.Filesystem.MakeDir(ctx, component, e2b.WithFileUser(remoteFileUser(ctx)))+ return normalizeE2BError("MakeDir", err)})}@@ -1598,6 +1599,24 @@ func (c *E2BRemoteClient) StartDesktopTTLRefresh(ctx context.Context, handle Rem})}+// KeepAlive moves the sandbox's timeout to ttl from now. E2B does not count+// commands, file access or URL requests as activity on a running sandbox; only+// SetTimeout resets it.+func (c *E2BRemoteClient) KeepAlive(ctx context.Context, handle RemoteSandboxHandle, ttl time.Duration) error {+ sbx, err := e2bHandleSandbox("KeepAlive", handle)+ if err != nil {+ return err+ }+ seconds, err := e2bTimeoutSeconds(RemoteTimeoutPolicy{Mode: RemoteTimeoutExplicit, Value: ttl}, 0)+ if err != nil {+ return err+ }+ if seconds <= 0 {+ return nil+ }+ return sbx.SetTimeoutWithContext(ctx, seconds)+}+var (_ RemoteSandboxClient = (*E2BRemoteClient)(nil)_ RemoteSnapshotManager = (*E2BRemoteClient)(nil)diff --git a/internal/sandbox/keepalive.go b/internal/sandbox/keepalive.gonew file mode 100644index 000000000..0c9ddb319--- /dev/null+++ b/internal/sandbox/keepalive.go@@ -0,0 +1,102 @@+package sandbox++import (+ "context"+ "fmt"+ "strings"+ "time"+)++// RemoteSandboxKeepAliver refreshes a sandbox's provider idle timer.+// Background work runs outside chat turns, so nothing else tells the provider+// the sandbox is still in use. Cube and E2B move their timeout to ttl from+// now; Docker runs a no-op exec, which refreshes the activity marker its idle+// sweeper reads.+type RemoteSandboxKeepAliver interface {+ KeepAlive(ctx context.Context, handle RemoteSandboxHandle, ttl time.Duration) error+}++// keepAliverFrom narrows client to keep-alive. It looks through the Langfuse+// decorator rather than having the decorator forward: every client+// NewSessionBoundManager holds is wrapped, and a forwarding method would make+// a client without the capability look like it has one.+func keepAliverFrom(client RemoteSandboxClient) (RemoteSandboxKeepAliver, bool) {+ switch c := client.(type) {+ case nil:+ return nil, false+ case *langfuseRemoteClient:+ return keepAliverFrom(c.inner)+ case *langfuseSnapshotClient:+ return keepAliverFrom(c.inner)+ }+ keeper, ok := client.(RemoteSandboxKeepAliver)+ return keeper, ok+}++var (+ _ RemoteSandboxKeepAliver = (*CubeRemoteClient)(nil)+ _ RemoteSandboxKeepAliver = (*E2BRemoteClient)(nil)+ _ RemoteSandboxKeepAliver = (*DockerRemoteClient)(nil)+)++// keepAliveRefreshDivisor sets how often a caller refreshes: every TTL/3+// (shared design §5), so two missed refreshes still land before expiry.+const keepAliveRefreshDivisor = 3++// KeepAliveSession refreshes the provider idle timer of the sandbox bound to+// sessionID and returns how long the caller may wait before the next refresh.+// It is lookup-only: it never provisions, never rebuilds and never connects to+// anything but the bound sandbox, so a session with nothing bound, or a+// provider with nothing to refresh, returns (0, nil).+func (m *SessionBoundManager) KeepAliveSession(ctx context.Context, sessionID string) (time.Duration, error) {+ return m.keepAliveSession(ctx, sessionID, "")+}++// KeepAliveSessionExpected is KeepAliveSession for callers that selected a+// specific sandbox. Lookup is only against expectedSandboxID: a missing+// binding or a different sandbox returns (0, nil) and never allocates.+func (m *SessionBoundManager) KeepAliveSessionExpected(+ ctx context.Context, sessionID, expectedSandboxID string,+) (time.Duration, error) {+ return m.keepAliveSession(ctx, sessionID, expectedSandboxID)+}++func (m *SessionBoundManager) keepAliveSession(+ ctx context.Context, sessionID, expectedSandboxID string,+) (time.Duration, error) {+ if err := m.requireRemoteBackend(); err != nil {+ return 0, err+ }+ keeper, ok := keepAliverFrom(m.client)+ if !ok {+ return 0, nil+ }+ req, err := buildSessionCreateRequest(m.client.Provider(), m.config)+ if err != nil {+ return 0, err+ }+ ttl := req.Timeout.Value+ if ttl <= 0 {+ return 0, nil+ }+ sc, err := m.scope(ctx, sessionID)+ if err != nil {+ return 0, err+ }+ handle, found, err := m.lookupSessionHandleForKey(ctx, sc.machine(), expectedSandboxID)+ if err != nil {+ // A mismatched expected id is "not this sandbox", not a failure to refresh+ // and not a reason to allocate a replacement.+ if expectedSandboxID != "" && strings.Contains(err.Error(), "bound sandbox changed") {+ return 0, nil+ }+ return 0, err+ }+ if !found {+ return 0, nil+ }+ if err := keeper.KeepAlive(ctx, handle, ttl); err != nil {+ return 0, fmt.Errorf("sandbox: keep session sandbox alive: %w", err)+ }+ return ttl / keepAliveRefreshDivisor, nil+}diff --git a/internal/sandbox/keepalive_test.go b/internal/sandbox/keepalive_test.gonew file mode 100644index 000000000..74db567fb--- /dev/null+++ b/internal/sandbox/keepalive_test.go@@ -0,0 +1,244 @@+package sandbox++import (+ "context"+ "errors"+ "sync"+ "testing"+ "time"++ "github.com/stretchr/testify/require"++ "github.com/Tencent/WeKnora/internal/types"+)++type keepAliveCall struct {+ sandboxID string+ ttl time.Duration+}++// keepAliveFakeClient adds the keep-alive capability to the shared fake.+type keepAliveFakeClient struct {+ *fakeRemoteClient+ keepMu sync.Mutex+ keepCalls []keepAliveCall+ keepErr error+}++func (c *keepAliveFakeClient) KeepAlive(_ context.Context, handle RemoteSandboxHandle, ttl time.Duration) error {+ c.keepMu.Lock()+ defer c.keepMu.Unlock()+ id := ""+ if handle != nil {+ id = handle.ID()+ }+ c.keepCalls = append(c.keepCalls, keepAliveCall{sandboxID: id, ttl: ttl})+ return c.keepErr+}++func (c *keepAliveFakeClient) calls() []keepAliveCall {+ c.keepMu.Lock()+ defer c.keepMu.Unlock()+ return append([]keepAliveCall(nil), c.keepCalls...)+}++func pausingCreateRequest() RemoteCreateRequest {+ return RemoteCreateRequest{+ TemplateID: "template-a",+ Timeout: RemoteTimeoutPolicy{+ Mode: RemoteTimeoutExplicit, Value: 15 * time.Minute,+ Action: RemoteOnTimeoutPause, AutoResume: true,+ },+ }+}++func TestCubeKeepAliveRefreshesTimeout(t *testing.T) {+ mock := newCubeMockServer(t)+ client := newTestCubeRemoteClient(t, mock)+ handle, err := client.Create(context.Background(), pausingCreateRequest())+ require.NoError(t, err)++ before := mock.timeoutPOSTs.Load()+ require.NoError(t, client.KeepAlive(context.Background(), handle, 30*time.Minute))+ require.Equal(t, before+1, mock.timeoutPOSTs.Load())+}++func TestE2BKeepAliveRefreshesTimeout(t *testing.T) {+ mock := newE2BMockServer(t)+ client := newTestE2BRemoteClient(t, mock)+ handle, err := client.Create(context.Background(), pausingCreateRequest())+ require.NoError(t, err)++ before := mock.timeoutPOSTs.Load()+ require.NoError(t, client.KeepAlive(context.Background(), handle, 5*time.Minute))+ require.Equal(t, before+1, mock.timeoutPOSTs.Load())+}++func TestDockerKeepAliveRunsANoOpExecThatTouchesTheActivityMarker(t *testing.T) {+ engine := newFakeDockerEngine()+ docker := newTestDockerClient(t, engine)++ require.NoError(t, docker.KeepAlive(context.Background(), testHandle("container-1"), time.Minute))++ require.Len(t, engine.execOptions, 1)+ cmd := engine.execOptions[0].Cmd+ require.Contains(t, cmd[2], dockerActivityMarker)+ require.Equal(t, []string{"weknora-exec", "true"}, cmd[3:])+}++func TestKeepAliverFromSeesThroughLangfuse(t *testing.T) {+ inner := &keepAliveFakeClient{fakeRemoteClient: newFakeRemoteClient(SandboxTypeCube)}+ keeper, ok := keepAliverFrom(wrapLangfuseRemoteClient(inner))+ require.True(t, ok)+ require.NoError(t, keeper.KeepAlive(context.Background(), nil, time.Minute))+ require.Equal(t, []keepAliveCall{{ttl: time.Minute}}, inner.calls())+}++func TestKeepAliverFromRejectsClientWithoutTheCapability(t *testing.T) {+ _, ok := keepAliverFrom(wrapLangfuseRemoteClient(newFakeRemoteClient(SandboxTypeCube)))+ require.False(t, ok, "wrapping must not invent keep-alive support")+ _, ok = keepAliverFrom(nil)+ require.False(t, ok)+}++const keepAliveTestTTL = 12 * time.Minute++func newKeepAliveTestManager(t *testing.T, client RemoteSandboxClient) *SessionBoundManager {+ t.Helper()+ cfg := DefaultConfig()+ cfg.CubeTemplate = "tpl-test"+ cfg.CubeSandboxTTL = keepAliveTestTTL+ mgr, err := NewSessionBoundManager(SessionBoundManagerConfig{+ Config: cfg,+ Client: client,+ Store: NewMemorySessionSandboxBindingStore(),+ Checker: &fakeSessionExistenceChecker{exists: true},+ SkipHealthProbe: true,+ })+ require.NoError(t, err)+ return mgr+}++func keepAliveTestCtx() context.Context {+ return context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7))+}++func bindKeepAliveTestSession(t *testing.T, mgr *SessionBoundManager, sessionID string) string {+ t.Helper()+ ctx := keepAliveTestCtx()+ _, err := mgr.Execute(ctx, &ExecuteConfig{+ SessionID: sessionID, SkipValidation: true, ScriptContent: "print('ok')\n", Script: "hello.py",+ })+ require.NoError(t, err)+ id, ok := mgr.BoundSandboxID(ctx, sessionID)+ require.True(t, ok)+ return id+}++func TestKeepAliveSessionRefreshesTheBoundSandbox(t *testing.T) {+ client := &keepAliveFakeClient{fakeRemoteClient: newFakeRemoteClient(SandboxTypeCube)}+ mgr := newKeepAliveTestManager(t, client)+ sandboxID := bindKeepAliveTestSession(t, mgr, "s1")++ next, err := mgr.KeepAliveSession(keepAliveTestCtx(), "s1")++ require.NoError(t, err)+ require.Equal(t, keepAliveTestTTL/3, next)+ require.Equal(t, []keepAliveCall{{sandboxID: sandboxID, ttl: keepAliveTestTTL}}, client.calls())+}++func TestKeepAliveSessionNeverProvisions(t *testing.T) {+ client := &keepAliveFakeClient{fakeRemoteClient: newFakeRemoteClient(SandboxTypeCube)}+ mgr := newKeepAliveTestManager(t, client)++ next, err := mgr.KeepAliveSession(keepAliveTestCtx(), "unbound")++ require.NoError(t, err)+ require.Zero(t, next)+ require.Empty(t, client.calls())+ creates, _, _, _, _ := client.counts()+ require.Zero(t, creates, "keep-alive must never create a sandbox")+}++func TestKeepAliveSessionWithoutProviderSupportIsNoOp(t *testing.T) {+ client := newFakeRemoteClient(SandboxTypeCube)+ mgr := newKeepAliveTestManager(t, client)+ bindKeepAliveTestSession(t, mgr, "s1")+ _, connectsBefore, _, _, _ := client.counts()++ next, err := mgr.KeepAliveSession(keepAliveTestCtx(), "s1")++ require.NoError(t, err)+ require.Zero(t, next)+ _, connectsAfter, _, _, _ := client.counts()+ require.Equal(t, connectsBefore, connectsAfter, "nothing to refresh, so nothing to connect to")+}++func TestKeepAliveSessionExpectedRefreshesOnlyTheMatchingSandbox(t *testing.T) {+ client := &keepAliveFakeClient{fakeRemoteClient: newFakeRemoteClient(SandboxTypeCube)}+ mgr := newKeepAliveTestManager(t, client)+ sandboxID := bindKeepAliveTestSession(t, mgr, "s1")++ next, err := mgr.KeepAliveSessionExpected(keepAliveTestCtx(), "s1", sandboxID)++ require.NoError(t, err)+ require.Equal(t, keepAliveTestTTL/3, next)+ require.Equal(t, []keepAliveCall{{sandboxID: sandboxID, ttl: keepAliveTestTTL}}, client.calls())+}++func TestKeepAliveSessionExpectedDoesNotRefreshADifferentSandbox(t *testing.T) {+ client := &keepAliveFakeClient{fakeRemoteClient: newFakeRemoteClient(SandboxTypeCube)}+ mgr := newKeepAliveTestManager(t, client)+ bindKeepAliveTestSession(t, mgr, "s1")++ next, err := mgr.KeepAliveSessionExpected(keepAliveTestCtx(), "s1", "sbx-other")++ require.NoError(t, err)+ require.Zero(t, next)+ require.Empty(t, client.calls(), "a different sandbox is not refreshed and not replaced")+}++func TestKeepAliveSessionRefreshesSharedUserMachine(t *testing.T) {+ for _, expected := range []bool{false, true} {+ t.Run(map[bool]string{false: "session", true: "expected"}[expected], func(t *testing.T) {+ h := newUserManagerHarness(t, userAssignmentFor(userSessionA))+ ctx := ownerCtx()+ _, err := h.mgr.ExecShellCommand(ctx, userSessionA, "true", "", time.Second, nil)+ require.NoError(t, err)+ id, found := h.mgr.BoundSandboxID(ctx, userSessionA)+ require.True(t, found)+ keeper := &keepAliveFakeClient{fakeRemoteClient: h.client}+ h.mgr.client = wrapLangfuseRemoteClient(keeper)+ creates, _, _, _, _ := keeper.counts()++ var next time.Duration+ if expected {+ next, err = h.mgr.KeepAliveSessionExpected(ctx, userSessionA, id)+ } else {+ next, err = h.mgr.KeepAliveSession(ctx, userSessionA)+ }+ require.NoError(t, err)+ require.Positive(t, next)+ require.Equal(t, []keepAliveCall{{sandboxID: id, ttl: next * keepAliveRefreshDivisor}}, keeper.calls())+ after, _, _, _, _ := keeper.counts()+ require.Equal(t, creates, after, "keeping a shared machine alive must not create a session sandbox")++ next, err = h.mgr.KeepAliveSessionExpected(ctx, userSessionA, "another-machine")+ require.NoError(t, err)+ require.Zero(t, next)+ require.Len(t, keeper.calls(), 1, "a mismatched shared binding must not be refreshed")+ })+ }+}++func TestKeepAliveSessionReportsProviderFailure(t *testing.T) {+ client := &keepAliveFakeClient{fakeRemoteClient: newFakeRemoteClient(SandboxTypeCube)}+ client.keepErr = errors.New("provider down")+ mgr := newKeepAliveTestManager(t, client)+ bindKeepAliveTestSession(t, mgr, "s1")++ next, err := mgr.KeepAliveSession(keepAliveTestCtx(), "s1")++ require.ErrorContains(t, err, "provider down")+ require.Zero(t, next)+}diff --git a/internal/sandbox/sandbox.go b/internal/sandbox/sandbox.goindex 2b8a7314a..2060ff651 100644--- a/internal/sandbox/sandbox.go+++ b/internal/sandbox/sandbox.go@@ -179,6 +179,9 @@ var (// ErrSessionTurnActive means a chat turn already holds the session lease,// so rewind must not take the exclusive lock or reset the workspace.ErrSessionTurnActive = errors.New("sandbox: session turn is active")+ // ErrSessionWorkActive means background work is still live in the session,+ // so rewind must not take the exclusive lock or reset the workspace.+ ErrSessionWorkActive = errors.New("sandbox: session has live background work")// ErrTerminalUnsupported is returned when the active backend cannot// stream PTYs (Docker, a disabled manager). Distinct from// ErrNoLiveSessionSandbox: the session may well have a live sandbox,diff --git a/internal/sandbox/session_binding.go b/internal/sandbox/session_binding.goindex cb4cddf21..0f5f2b6f8 100644--- a/internal/sandbox/session_binding.go+++ b/internal/sandbox/session_binding.go@@ -9,6 +9,7 @@ import ("time""unicode"+ "github.com/Tencent/WeKnora/internal/background""github.com/Tencent/WeKnora/internal/sandbox/allocation""github.com/Tencent/WeKnora/internal/sandbox/workspace")@@ -369,6 +370,7 @@ type MemorySessionSandboxBindingStore struct {locks map[SessionSandboxKey]*memoryLifecycleLockturns map[SessionSandboxKey]*memoryTurnLeaserewinds map[SessionSandboxKey]struct{}+ activity *background.MemoryActivityRegistry}// NewMemorySessionSandboxBindingStore creates an empty in-memory store.@@ -378,6 +380,7 @@ func NewMemorySessionSandboxBindingStore() *MemorySessionSandboxBindingStore {locks: make(map[SessionSandboxKey]*memoryLifecycleLock),turns: make(map[SessionSandboxKey]*memoryTurnLease),rewinds: make(map[SessionSandboxKey]struct{}),+ activity: background.NewMemoryActivityRegistry(nil),}}@@ -632,7 +635,7 @@ func (s *MemorySessionSandboxBindingStore) BeginTurn(s.turns[key] = lease}if lease.refs == 0 {- lease.rebuildOnce = true+ lease.rebuildOnce = !s.hasSessionActivityLocked(ctx, key)}lease.refs++return nil@@ -679,6 +682,9 @@ func (s *MemorySessionSandboxBindingStore) TryLockRewind(if lease := s.turns[key]; lease != nil && lease.refs > 0 {return nil, ErrSessionTurnActive}+ if s.hasSessionActivityLocked(ctx, key) {+ return nil, ErrSessionWorkActive+ }if _, held := s.rewinds[key]; held {return nil, ErrSessionRewindLocked}@@ -721,6 +727,9 @@ func (s *MemorySessionSandboxBindingStore) TurnState(}s.mu.Lock()defer s.mu.Unlock()+ if s.hasSessionActivityLocked(ctx, key) {+ return true, false, nil+ }lease := s.turns[key]if lease == nil || lease.refs <= 0 {return false, false, nildiff --git a/internal/sandbox/session_binding_redis.go b/internal/sandbox/session_binding_redis.goindex 26e66d209..924fcd1e1 100644--- a/internal/sandbox/session_binding_redis.go+++ b/internal/sandbox/session_binding_redis.go@@ -5,7 +5,6 @@ import ("encoding/json""errors""fmt"- "strconv""strings""sync""time"@@ -78,23 +77,48 @@ return 1// image, which is what we want once no turn is actually using the sandbox.const sessionTurnLeaseTTL = 30 * time.Minute-var beginTurnScript = redis.NewScript(`+// KEYS[1] turn lease, KEYS[2] rewind lock, KEYS[3] background activity.+// ARGV[1] TTL ms. A turn that opens while background work is live does not+// arm its one rebuild: replacing the sandbox would destroy that work.+var beginTurnScript = redis.NewScript(activityLiveCountLua + `if redis.call('EXISTS', KEYS[2]) == 1 thenreturn redis.error_reply('ERR_SESSION_REWIND_LOCKED')endlocal refs = redis.call('HINCRBY', KEYS[1], 'refs', 1)if refs == 1 then- redis.call('HSET', KEYS[1], 'rebuild', '1')+ if live_count(KEYS[3], now_ms()) > 0 then+ redis.call('HSET', KEYS[1], 'rebuild', '0')+ else+ redis.call('HSET', KEYS[1], 'rebuild', '1')+ endendredis.call('PEXPIRE', KEYS[1], ARGV[1])return refs`)-// KEYS[1] turn lease, KEYS[2] rewind lock. ARGV[1] owner token, ARGV[2] TTL ms.-var tryLockRewindScript = redis.NewScript(`+// KEYS[1] turn lease, KEYS[2] background activity. ARGV[1] turn TTL ms.+// Returns {refs, rebuild, live background items}.+var turnStateScript = redis.NewScript(activityLiveCountLua + `+local live = live_count(KEYS[2], now_ms())+local refs = 0+local rebuild = '0'+if redis.call('EXISTS', KEYS[1]) == 1 then+ redis.call('PEXPIRE', KEYS[1], ARGV[1])+ refs = tonumber(redis.call('HGET', KEYS[1], 'refs') or '0') or 0+ rebuild = redis.call('HGET', KEYS[1], 'rebuild') or '0'+end+return {refs, rebuild, live}+`)++// KEYS[1] turn lease, KEYS[2] rewind lock, KEYS[3] background activity.+// ARGV[1] owner token, ARGV[2] TTL ms.+var tryLockRewindScript = redis.NewScript(activityLiveCountLua + `if redis.call('EXISTS', KEYS[1]) == 1 thenreturn redis.error_reply('ERR_SESSION_TURN_ACTIVE')end+if live_count(KEYS[3], now_ms()) > 0 then+ return redis.error_reply('ERR_SESSION_WORK_ACTIVE')+endif redis.call('EXISTS', KEYS[2]) == 1 thenreturn 0end@@ -395,7 +419,8 @@ func (s *RedisSessionSandboxBindingStore) BeginTurn(if ttlMS <= 0 {ttlMS = (30 * time.Minute).Milliseconds()}- if err := beginTurnScript.Run(ctx, s.client, []string{s.turnKey(key), s.rewindKey(key)}, ttlMS).Err(); err != nil {+ keys := []string{s.turnKey(key), s.rewindKey(key), s.activityKey(key)}+ if err := beginTurnScript.Run(ctx, s.client, keys, ttlMS).Err(); err != nil {if isRewindLockedRedisErr(err) {return ErrSessionRewindLocked}@@ -420,7 +445,9 @@ func (s *RedisSessionSandboxBindingStore) EndTurn(}// TurnState reports whether a chat turn is open and whether its first-// resolve may still rebuild a stale sandbox.+// resolve may still rebuild a stale sandbox. Live background work counts as an+// open turn that may not rebuild: it runs outside any turn, and a resolve that+// replaced the sandbox under it would destroy it.func (s *RedisSessionSandboxBindingStore) TurnState(ctx context.Context,key SessionSandboxKey,@@ -428,19 +455,25 @@ func (s *RedisSessionSandboxBindingStore) TurnState(if err := key.Validate(); err != nil {return false, false, err}- values, err := s.client.HGetAll(ctx, s.turnKey(key)).Result()+ values, err := turnStateScript.Run(+ ctx, s.client, []string{s.turnKey(key), s.activityKey(key)}, sessionTurnLeaseTTL.Milliseconds(),+ ).Slice()if err != nil {return false, false, fmt.Errorf("read sandbox turn lease: %w", err)}- if len(values) == 0 {- return false, false, nil+ if len(values) != 3 {+ return false, false, fmt.Errorf("read sandbox turn lease: unexpected reply %v", values)+ }+ refs, _ := values[0].(int64)+ rebuild, _ := values[1].(string)+ live, _ := values[2].(int64)+ if live > 0 {+ return true, false, nil}- _ = s.client.PExpire(ctx, s.turnKey(key), sessionTurnLeaseTTL).Err()- refs, _ := strconv.Atoi(values["refs"])if refs <= 0 {return false, false, nil}- return true, values["rebuild"] == "1", nil+ return true, rebuild == "1", nil}// ConsumeTurnRebuild spends the one rebuild allowed for the current turn.@@ -493,12 +526,15 @@ func (s *RedisSessionSandboxBindingStore) TryLockRewind(ttlMS = sessionRewindLockTTL.Milliseconds()}acquired, err := tryLockRewindScript.Run(- ctx, s.client, []string{s.turnKey(key), s.rewindKey(key)}, token, ttlMS,+ ctx, s.client, []string{s.turnKey(key), s.rewindKey(key), s.activityKey(key)}, token, ttlMS,).Int64()if err != nil {if isTurnActiveRedisErr(err) {return nil, ErrSessionTurnActive}+ if isWorkActiveRedisErr(err) {+ return nil, ErrSessionWorkActive+ }return nil, fmt.Errorf("lock session rewind: %w", err)}if acquired == 0 {@@ -543,6 +579,10 @@ func isTurnActiveRedisErr(err error) bool {return err != nil && strings.Contains(err.Error(), "ERR_SESSION_TURN_ACTIVE")}+func isWorkActiveRedisErr(err error) bool {+ return err != nil && strings.Contains(err.Error(), "ERR_SESSION_WORK_ACTIVE")+}+func (s *RedisSessionSandboxBindingStore) rewindLease() time.Duration {if s != nil && s.rewindLockTTL > 0 {return s.rewindLockTTLdiff --git a/internal/sandbox/session_binding_redis_activity_test.go b/internal/sandbox/session_binding_redis_activity_test.gonew file mode 100644index 000000000..93d411b6f--- /dev/null+++ b/internal/sandbox/session_binding_redis_activity_test.go@@ -0,0 +1,87 @@+package sandbox++import (+ "context"+ "testing"+ "time"++ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/stretchr/testify/require"+)++// addLiveActivity registers one background item for key through the real+// Redis registry, so the turn-lease scripts read exactly what features write.+func addLiveActivity(t *testing.T, store *RedisSessionSandboxBindingStore, key SessionSandboxKey, lease time.Duration) {+ t.Helper()+ reg := NewRedisActivityRegistry(store)+ ref := interfaces.SessionRef{TenantID: key.TenantID, SessionID: key.SessionID}+ require.NoError(t, reg.Add(context.Background(), ref, interfaces.BackgroundActivity{+ ID: "sj_1", Kind: "shell_job", Title: "build", StartedAt: time.Now(),+ }, lease, 0))+}++func TestRedisTurnStateCountsLiveBackgroundWorkAsActiveWithoutRebuild(t *testing.T) {+ store, _, mini := newRedisBindingTestStore(t)+ ctx := context.Background()+ key := SessionSandboxKey{TenantID: 42, SessionID: "session-bg"}+ now := time.Date(2026, 10, 15, 10, 0, 0, 0, time.UTC)+ mini.SetTime(now)++ addLiveActivity(t, store, key, 30*time.Second)++ active, rebuildOnce, err := store.TurnState(ctx, key)+ require.NoError(t, err)+ require.True(t, active, "live background work must keep a turn-less resolve from rebuilding")+ require.False(t, rebuildOnce)++ mini.SetTime(now.Add(31 * time.Second))+ active, rebuildOnce, err = store.TurnState(ctx, key)+ require.NoError(t, err)+ require.False(t, active, "an expired lease no longer protects the sandbox")+ require.False(t, rebuildOnce)+}++func TestRedisBeginTurnDoesNotArmRebuildWhileBackgroundWorkIsLive(t *testing.T) {+ store, _, mini := newRedisBindingTestStore(t)+ ctx := context.Background()+ key := SessionSandboxKey{TenantID: 42, SessionID: "session-bg-turn"}+ now := time.Date(2026, 10, 15, 10, 0, 0, 0, time.UTC)+ mini.SetTime(now)++ addLiveActivity(t, store, key, 30*time.Second)+ require.NoError(t, store.BeginTurn(ctx, key))++ active, rebuildOnce, err := store.TurnState(ctx, key)+ require.NoError(t, err)+ require.True(t, active)+ require.False(t, rebuildOnce)++ // The work ends mid-turn. The turn still must not rebuild: its first+ // resolve already ran against this sandbox.+ mini.SetTime(now.Add(31 * time.Second))+ active, rebuildOnce, err = store.TurnState(ctx, key)+ require.NoError(t, err)+ require.True(t, active)+ require.False(t, rebuildOnce)+ require.NoError(t, store.EndTurn(ctx, key))+}++func TestRedisTryLockRewindFailsWhileBackgroundWorkIsLive(t *testing.T) {+ store, _, mini := newRedisBindingTestStore(t)+ ctx := context.Background()+ key := SessionSandboxKey{TenantID: 42, SessionID: "session-bg-rewind"}+ now := time.Date(2026, 10, 15, 10, 0, 0, 0, time.UTC)+ mini.SetTime(now)++ addLiveActivity(t, store, key, 30*time.Second)+ _, err := store.TryLockRewind(ctx, key)+ require.ErrorIs(t, err, ErrSessionWorkActive)+ held, err := store.HasRewindLock(ctx, key)+ require.NoError(t, err)+ require.False(t, held, "a refused rewind must not leave its lock behind")++ mini.SetTime(now.Add(31 * time.Second))+ unlock, err := store.TryLockRewind(ctx, key)+ require.NoError(t, err)+ unlock()+}diff --git a/internal/sandbox/session_lifecycle.go b/internal/sandbox/session_lifecycle.goindex 09925ace4..53ea83f16 100644--- a/internal/sandbox/session_lifecycle.go+++ b/internal/sandbox/session_lifecycle.go@@ -320,6 +320,12 @@ func (l *remoteSessionLifecycle) rebuildStale(if err != nil || others > 0 {return false, nil}+ // A sibling may have registered background work after the first check+ // and ended its turn before we closed admission. Registration is now+ // blocked, so recheck the machine's leases before destroying it.+ if l.hasLiveMachineActivity(guarded, key) {+ return false, nil+ }if err := barrier.Check(); err != nil {return false, err}@@ -1161,6 +1167,9 @@ func (l *remoteSessionLifecycle) shouldRebuildStaleBinding(if !key.IsUserScoped() {return true}+ if l.hasLiveMachineActivity(ctx, key) {+ return false+ }others, err := l.admission.ActiveExcept(ctx, admission.MachineScope(l.digest(key)), admissionExemptFrom(ctx)...)if err != nil {log.Printf("[sandbox] admission of %s unavailable (%v); keeping the current sandbox", key, err)@@ -1169,6 +1178,21 @@ func (l *remoteSessionLifecycle) shouldRebuildStaleBinding(return others == 0}+func (l *remoteSessionLifecycle) hasLiveMachineActivity(ctx context.Context, key SessionSandboxKey) bool {+ store, ok := l.bindings.(interface {+ hasBackgroundActivity(context.Context, SessionSandboxKey) (bool, error)+ })+ if !ok {+ return false+ }+ active, err := store.hasBackgroundActivity(ctx, key)+ if err != nil {+ log.Printf("[sandbox] background activity of %s unavailable (%v); keeping the current sandbox", key, err)+ return true+ }+ return active+}+func (l *remoteSessionLifecycle) consumeTurnRebuild(ctx context.Context,key SessionSandboxKey,diff --git a/internal/stream/notice_inbox_redis.go b/internal/stream/notice_inbox_redis.gonew file mode 100644index 000000000..dbcd0da9f--- /dev/null+++ b/internal/stream/notice_inbox_redis.go@@ -0,0 +1,156 @@+package stream++import (+ "context"+ "encoding/json"+ "fmt"+ "sort"+ "time"++ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/redis/go-redis/v9"+)++// noticeKeyTTL is a backstop so an abandoned session's inbox disappears.+const noticeKeyTTL = interfaces.NoticeRetention + time.Hour++// pushNoticeScript writes a notice and refreshes the key TTL in one round+// trip. As two commands, a failed EXPIRE after a successful HSETNX would return+// an error for a notice that is already stored; the caller's retry would then+// see first=false and work_finished would never be emitted.+//+// KEYS[1] inbox key; ARGV[1] id, ARGV[2] encoded notice, ARGV[3] TTL in ms.+var pushNoticeScript = redis.NewScript(`+local first = redis.call('HSETNX', KEYS[1], ARGV[1], ARGV[2])+redis.call('PEXPIRE', KEYS[1], ARGV[3])+return first+`)++// dropNoticesScript deletes each field only if it still holds the value the+// caller read. Deleting by ID alone could remove a notice that was acked and+// pushed again between the caller's read and this delete.+//+// KEYS[1] inbox key; ARGV pairs of id, encoded notice as read.+var dropNoticesScript = redis.NewScript(`+local n = 0+for i = 1, #ARGV, 2 do+ if redis.call('HGET', KEYS[1], ARGV[i]) == ARGV[i + 1] then+ n = n + redis.call('HDEL', KEYS[1], ARGV[i])+ end+end+return n+`)++type redisNoticeValue struct {+ Kind string `json:"kind"`+ Text string `json:"text"`+ FinishedAtMS int64 `json:"finished_at_ms"`+}++// RedisNoticeInbox is the multi-replica NoticeInbox. It lives next to the+// steer lists and uses the same key prefix.+type RedisNoticeInbox struct {+ mgr *RedisStreamManager+ clock func() time.Time+}++var _ interfaces.NoticeInbox = (*RedisNoticeInbox)(nil)++// NewRedisNoticeInbox returns an inbox on m's Redis client. A nil clock means time.Now.+func NewRedisNoticeInbox(m *RedisStreamManager, clock func() time.Time) *RedisNoticeInbox {+ if clock == nil {+ clock = time.Now+ }+ return &RedisNoticeInbox{mgr: m, clock: clock}+}++func (b *RedisNoticeInbox) key(s interfaces.SessionRef) string {+ return fmt.Sprintf("%s:%s:bg-inbox", b.mgr.prefix, s.SessionID)+}++// Push implements interfaces.NoticeInbox. An error means the notice was not+// stored. The one exception is a reply lost after the script ran: the retry+// then sees first=false, and the UI's background-work polling catches up.+func (b *RedisNoticeInbox) Push(+ ctx context.Context, s interfaces.SessionRef, n interfaces.BackgroundNotice,+) (bool, error) {+ raw, err := json.Marshal(redisNoticeValue{+ Kind: n.Kind, Text: n.Text, FinishedAtMS: n.FinishedAt.UnixMilli(),+ })+ if err != nil {+ return false, fmt.Errorf("encode background notice: %w", err)+ }+ first, err := pushNoticeScript.Run(ctx, b.mgr.client, []string{b.key(s)},+ n.ID, raw, noticeKeyTTL.Milliseconds()).Int()+ if err != nil {+ return false, fmt.Errorf("push background notice: %w", err)+ }+ return first == 1, nil+}++// Pending implements interfaces.NoticeInbox.+func (b *RedisNoticeInbox) Pending(+ ctx context.Context, s interfaces.SessionRef,+) ([]interfaces.BackgroundNotice, error) {+ key := b.key(s)+ all, err := b.mgr.client.HGetAll(ctx, key).Result()+ if err != nil {+ return nil, fmt.Errorf("read background notices: %w", err)+ }+ cutoff := b.clock().Add(-interfaces.NoticeRetention)+ out := make([]interfaces.BackgroundNotice, 0, len(all))+ expired := make(map[string]string)+ for id, raw := range all {+ var v redisNoticeValue+ if err := json.Unmarshal([]byte(raw), &v); err != nil {+ return nil, fmt.Errorf("decode background notice %s: %w", id, err)+ }+ finished := time.UnixMilli(v.FinishedAtMS).UTC()+ if finished.Before(cutoff) {+ expired[id] = raw+ continue+ }+ out = append(out, interfaces.BackgroundNotice{ID: id, Kind: v.Kind, Text: v.Text, FinishedAt: finished})+ }+ if err := b.dropIfUnchanged(ctx, key, expired); err != nil {+ return nil, fmt.Errorf("drop expired background notices: %w", err)+ }+ sort.Slice(out, func(i, j int) bool {+ if out[i].FinishedAt.Equal(out[j].FinishedAt) {+ return out[i].ID < out[j].ID+ }+ return out[i].FinishedAt.Before(out[j].FinishedAt)+ })+ return out, nil+}++// dropIfUnchanged deletes each id whose stored value still equals read[id].+func (b *RedisNoticeInbox) dropIfUnchanged(ctx context.Context, key string, read map[string]string) error {+ if len(read) == 0 {+ return nil+ }+ args := make([]any, 0, 2*len(read))+ for id, raw := range read {+ args = append(args, id, raw)+ }+ return dropNoticesScript.Run(ctx, b.mgr.client, []string{key}, args...).Err()+}++// Ack implements interfaces.NoticeInbox.+func (b *RedisNoticeInbox) Ack(ctx context.Context, s interfaces.SessionRef, ids []string) error {+ if len(ids) == 0 {+ return nil+ }+ if err := b.mgr.client.HDel(ctx, b.key(s), ids...).Err(); err != nil {+ return fmt.Errorf("ack background notices: %w", err)+ }+ return nil+}++// Clear implements interfaces.NoticeInbox.+func (b *RedisNoticeInbox) Clear(ctx context.Context, s interfaces.SessionRef) error {+ if err := b.mgr.client.Del(ctx, b.key(s)).Err(); err != nil {+ return fmt.Errorf("clear background notices: %w", err)+ }+ return nil+}diff --git a/internal/stream/notice_inbox_redis_test.go b/internal/stream/notice_inbox_redis_test.gonew file mode 100644index 000000000..38a09a78b--- /dev/null+++ b/internal/stream/notice_inbox_redis_test.go@@ -0,0 +1,77 @@+package stream++import (+ "context"+ "testing"+ "time"++ "github.com/Tencent/WeKnora/internal/background/backgroundtest"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/stretchr/testify/require"+)++func TestRedisNoticeInboxContract(t *testing.T) {+ backgroundtest.RunNoticeInboxContract(t,+ func(t *testing.T) (interfaces.NoticeInbox, backgroundtest.Advance) {+ mgr, mini := newTestRedisStreamManager(t, time.Hour)+ now := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC)+ advance := func(d time.Duration) {+ now = now.Add(d)+ mini.FastForward(d)+ }+ return NewRedisNoticeInbox(mgr, func() time.Time { return now }), advance+ })+}++// The write and the TTL refresh happen in one script, so a stored notice+// always carries the backstop TTL, and a repeated push still refreshes it.+func TestRedisNoticeInboxPushSetsKeyTTL(t *testing.T) {+ mgr, mini := newTestRedisStreamManager(t, time.Hour)+ inbox := NewRedisNoticeInbox(mgr, nil)+ ctx := context.Background()+ s := interfaces.SessionRef{TenantID: 1, SessionID: "sess-1"}+ n := interfaces.BackgroundNotice{ID: "sj_1", Kind: "shell", Text: "done", FinishedAt: time.Now()}++ first, err := inbox.Push(ctx, s, n)+ require.NoError(t, err)+ require.True(t, first)+ require.Equal(t, noticeKeyTTL, mini.TTL(inbox.key(s)))++ mini.FastForward(time.Hour)+ first, err = inbox.Push(ctx, s, n)+ require.NoError(t, err)+ require.False(t, first)+ require.Equal(t, noticeKeyTTL, mini.TTL(inbox.key(s)))+}++// Pending reads, then deletes what it saw as expired. A notice acked and+// pushed again under the same ID in between must survive that delete.+func TestRedisNoticeInboxExpiryKeepsNoticeRewrittenAfterRead(t *testing.T) {+ mgr, mini := newTestRedisStreamManager(t, time.Hour)+ inbox := NewRedisNoticeInbox(mgr, nil)+ ctx := context.Background()+ s := interfaces.SessionRef{TenantID: 1, SessionID: "sess-1"}+ key := inbox.key(s)++ old := time.Now().Add(-2 * interfaces.NoticeRetention)+ _, err := inbox.Push(ctx, s, interfaces.BackgroundNotice{ID: "sj_1", Kind: "shell", Text: "old", FinishedAt: old})+ require.NoError(t, err)+ _, err = inbox.Push(ctx, s, interfaces.BackgroundNotice{ID: "sj_2", Kind: "shell", Text: "old", FinishedAt: old})+ require.NoError(t, err)+ read := map[string]string{"sj_1": mini.HGet(key, "sj_1"), "sj_2": mini.HGet(key, "sj_2")}++ require.NoError(t, inbox.Ack(ctx, s, []string{"sj_1"}))+ fresh := interfaces.BackgroundNotice{ID: "sj_1", Kind: "shell", Text: "fresh", FinishedAt: time.Now()}+ first, err := inbox.Push(ctx, s, fresh)+ require.NoError(t, err)+ require.True(t, first)++ require.NoError(t, inbox.dropIfUnchanged(ctx, key, read))++ got, err := inbox.Pending(ctx, s)+ require.NoError(t, err)+ require.Len(t, got, 1)+ require.Equal(t, "sj_1", got[0].ID)+ require.Equal(t, "fresh", got[0].Text)+ require.Empty(t, mini.HGet(key, "sj_2"), "an expired notice that did not change must be dropped")+}diff --git a/internal/types/agent.go b/internal/types/agent.goindex a34086541..42bb3154d 100644--- a/internal/types/agent.go+++ b/internal/types/agent.go@@ -470,6 +470,15 @@ func IsPipelineToolCallID(id string) bool {return strings.HasPrefix(id, PipelineToolCallIDPrefix)}+// BackgroundInjection is one background-work message the engine put in front+// of the model before a step's LLM call. History replays it in place; AckIDs+// are the completion notices it delivered, acknowledged once the assistant+// message carrying the step is persisted.+type BackgroundInjection struct {+ Content string `json:"content"`+ AckIDs []string `json:"ack_ids,omitempty"`+}+// AgentStep represents one iteration of the ReAct looptype AgentStep struct {Iteration int `json:"iteration"` // Iteration number (0-indexed)@@ -480,6 +489,12 @@ type AgentStep struct {// UserMessagesBefore records consumed steer rows in delivery order, before// this model response. Unlike timestamps, this remains unambiguous on replay.UserMessagesBefore []string `json:"user_messages_before,omitempty"`+ // BackgroundInjections are background-work messages delivered before this+ // model response, after the step's steered messages, in delivery order.+ BackgroundInjections []BackgroundInjection `json:"background_injections,omitempty"`+ // BackgroundSeenIDs are finished background items whose results this+ // step's tool calls (work_wait, work_cancel) returned to the model.+ BackgroundSeenIDs []string `json:"background_seen_ids,omitempty"`// IntermediateAnswer preserves a plain answer followed by a loop-end steer.// The canonical final answer is still stored in Message.Content.IntermediateAnswer bool `json:"intermediate_answer,omitempty"`@@ -520,13 +535,14 @@ func (s *AgentStep) GetObservations() []string {// AgentState tracks the execution state of an agent across iterationstype AgentState struct {- PendingSteerMessages []string `json:"-"`- CurrentRound int `json:"current_round"` // Current round number- RoundSteps []AgentStep `json:"round_steps"` // All steps taken so far in the current round- IsComplete bool `json:"is_complete"` // Whether agent has finished- FinalAnswer string `json:"final_answer"` // The final answer to the query- KnowledgeRefs []*SearchResult `json:"knowledge_refs"` // Collected knowledge references- TurnUsage TokenUsage `json:"turn_usage"` // LLM usage accumulated across this turn+ PendingSteerMessages []string `json:"-"`+ PendingBackground []BackgroundInjection `json:"-"`+ CurrentRound int `json:"current_round"` // Current round number+ RoundSteps []AgentStep `json:"round_steps"` // All steps taken so far in the current round+ IsComplete bool `json:"is_complete"` // Whether agent has finished+ FinalAnswer string `json:"final_answer"` // The final answer to the query+ KnowledgeRefs []*SearchResult `json:"knowledge_refs"` // Collected knowledge references+ TurnUsage TokenUsage `json:"turn_usage"` // LLM usage accumulated across this turn// Suspended is set when the run ended waiting on a user prompt.// SuspendedStep is the interrupted round: its thought and the calls// that finished before the blocked one, kept for display.diff --git a/internal/types/chat.go b/internal/types/chat.goindex 803948407..f7ed5d513 100644--- a/internal/types/chat.go+++ b/internal/types/chat.go@@ -248,6 +248,15 @@ const (ResponseTypeInstallOutput ResponseType = "install_output"// ResponseTypeCommandOutput updates the pending tool card without completing it.ResponseTypeCommandOutput ResponseType = "command_output"+ // ResponseTypeWorkStarted is a background work event of shared design §7.+ // Features emit it through service.BackgroundEventEmitter, which picks the stream.+ ResponseTypeWorkStarted ResponseType = "work_started"+ // ResponseTypeWorkEvent is a background work event of shared design §7.+ // Features emit it through service.BackgroundEventEmitter, which picks the stream.+ ResponseTypeWorkEvent ResponseType = "work_event"+ // ResponseTypeWorkFinished is a background work event of shared design §7.+ // Features emit it through service.BackgroundEventEmitter, which picks the stream.+ ResponseTypeWorkFinished ResponseType = "work_finished"// Error response typeResponseTypeError ResponseType = "error"// Reflection response type (for agent reflection)diff --git a/internal/types/interfaces/background.go b/internal/types/interfaces/background.gonew file mode 100644index 000000000..a3ad86be3--- /dev/null+++ b/internal/types/interfaces/background.go@@ -0,0 +1,145 @@+package interfaces++import (+ "context"+ "errors"+ "time"+)++// SessionRef identifies a session in every background-work interface.+type SessionRef struct {+ TenantID uint64+ SessionID string+}++// External work states. The model and the UI learn only these strings; a+// feature maps its own internal states onto them at the boundary.+const (+ WorkStateRunning = "running"+ WorkStateUnknown = "unknown" // cheap lookup could not decide; never returned to the model+ WorkStateSucceeded = "succeeded"+ WorkStateFailed = "failed"+ WorkStateCancelled = "cancelled"+ WorkStateLost = "lost"+)++// IsTerminalWorkState reports whether s is one of the four terminal states.+func IsTerminalWorkState(s string) bool {+ switch s {+ case WorkStateSucceeded, WorkStateFailed, WorkStateCancelled, WorkStateLost:+ return true+ default:+ return false+ }+}++// ErrActivityLimit is returned by ActivityRegistry.Add when the session already+// holds kindCap live items of the same kind.+var ErrActivityLimit = errors.New("background: activity limit reached")++// ErrWorkNotFound is returned by a WorkHandler for an ID that does not exist in+// the given session. Handlers must use it for IDs owned by another session too.+var ErrWorkNotFound = errors.New("background: work not found in this session")++// BackgroundActivity is one live background item registered for a session.+type BackgroundActivity struct {+ ID string // feature-prefixed, e.g. "sj_..."; opaque to shared code+ Kind string // opaque to shared code; used only for per-kind caps and UI badges+ Title string+ StartedAt time.Time+}++// ActivityRegistry records which background items are live in a session. Every+// entry is a lease: it disappears unless renewed before its deadline.+type ActivityRegistry interface {+ // Add registers a under a lease of length lease. If a.ID is already live it+ // only extends the deadline and does not count against kindCap. Otherwise it+ // fails with ErrActivityLimit when the session already holds kindCap live+ // items of a.Kind. kindCap <= 0 means no limit.+ Add(ctx context.Context, s SessionRef, a BackgroundActivity, lease time.Duration, kindCap int) error+ // Renew moves a live item's deadline to now+lease. It returns found=false,+ // and does nothing, when the item was removed or has expired.+ Renew(ctx context.Context, s SessionRef, id string, lease time.Duration) (found bool, err error)+ // Remove deletes the item. Removing a missing item is not an error.+ Remove(ctx context.Context, s SessionRef, id string) error+ // List returns live items only, in no particular order.+ List(ctx context.Context, s SessionRef) ([]BackgroundActivity, error)+}++// NoticeRetention is how long an unacknowledged notice stays after FinishedAt.+const NoticeRetention = 24 * time.Hour++// BackgroundNotice is one completion notice waiting to be shown to the model.+type BackgroundNotice struct {+ ID string // equals the activity ID+ Kind string+ Text string // model-facing text written by the feature; the feature bounds its length+ FinishedAt time.Time+}++// NoticeInbox holds completion notices until a persisted turn acknowledges them.+type NoticeInbox interface {+ // Push stores n unless a notice with the same ID already exists. Only the+ // first push for an ID returns first=true; later ones change nothing.+ Push(ctx context.Context, s SessionRef, n BackgroundNotice) (first bool, err error)+ // Pending returns unacknowledged notices finished within NoticeRetention,+ // oldest first.+ Pending(ctx context.Context, s SessionRef) ([]BackgroundNotice, error)+ // Ack deletes the given IDs. Unknown IDs are ignored.+ Ack(ctx context.Context, s SessionRef, ids []string) error+ // Clear deletes every notice of the session.+ Clear(ctx context.Context, s SessionRef) error+}++// StatusOptions controls one WorkHandler.Status call.+type StatusOptions struct {+ // Detailed=false must be cheap: Redis, memory or a local file only, and+ // never a sandbox exec. It may return WorkStateUnknown.+ Detailed bool+ // RunID scopes incremental output reads to one agent run. Empty means the+ // read is not part of a run: return head and tail, move no read position.+ RunID string+ // OutputBudget caps output bytes for each item; 0 means the handler default.+ OutputBudget int+}++// WorkResult is one item's state as returned by a WorkHandler.+type WorkResult struct {+ ID string+ State string // one of the WorkState* constants+ Terminal bool+ Title string+ Payload any // feature-defined; serialized into tool results as-is+ // OutputFiles is the deliverable list for a finished shell job.+ // Nil means the output directory was not inspected.+ OutputFiles []string+}++// WorkHandler answers status and cancel requests for one feature's work IDs.+type WorkHandler interface {+ // Status returns one result per ID in input order. An ID that does not+ // belong to s must yield ErrWorkNotFound for the whole call or be reported+ // with State=WorkStateLost. A batch error does not confirm the states of+ // the other requested IDs or acknowledge their completion notices.+ Status(ctx context.Context, s SessionRef, ids []string, opts StatusOptions) ([]WorkResult, error)+ // Cancel requests cancellation and returns the state observed at that time.+ Cancel(ctx context.Context, s SessionRef, ids []string) ([]WorkResult, error)+}++// SessionLifecycleListener lets features react to /stop and session deletion.+type SessionLifecycleListener interface {+ OnStop(ctx context.Context, s SessionRef) error+ OnDelete(ctx context.Context, s SessionRef) error+}++// SessionKeepAliver keeps a session's sandbox from being reclaimed by its+// provider. Implementations throttle internally and never provision a sandbox.+type SessionKeepAliver interface {+ KeepAlive(ctx context.Context, s SessionRef) error+}++// WorkEventEmitter publishes the work_finished event. It is called only by the+// writer whose NoticeInbox.Push returned first=true.+type WorkEventEmitter interface {+ EmitWorkFinished(ctx context.Context, s SessionRef, parentMessageID string, r WorkResult) error+}diff --git a/internal/types/interfaces/background_test.go b/internal/types/interfaces/background_test.gonew file mode 100644index 000000000..766645344--- /dev/null+++ b/internal/types/interfaces/background_test.go@@ -0,0 +1,20 @@+package interfaces++import "testing"++func TestIsTerminalWorkState(t *testing.T) {+ terminal := []string{+ WorkStateSucceeded, WorkStateFailed, WorkStateCancelled, WorkStateLost,+ }+ for _, s := range terminal {+ if !IsTerminalWorkState(s) {+ t.Errorf("IsTerminalWorkState(%q) = false, want true", s)+ }+ }+ nonTerminal := []string{WorkStateRunning, WorkStateUnknown, "queued", "timeout", "", "bogus"}+ for _, s := range nonTerminal {+ if IsTerminalWorkState(s) {+ t.Errorf("IsTerminalWorkState(%q) = true, want false", s)+ }+ }+}diff --git a/internal/types/sandbox_checkpoint_test.go b/internal/types/sandbox_checkpoint_test.gonew file mode 100644index 000000000..930a615a6--- /dev/null+++ b/internal/types/sandbox_checkpoint_test.go@@ -0,0 +1,22 @@+package types++import (+ "encoding/json"+ "testing"++ "github.com/stretchr/testify/require"+)++func TestSandboxCheckpointBackgroundActiveJSON(t *testing.T) {+ raw, err := json.Marshal(SandboxCheckpoint{SandboxID: "sbx", CommitSHA: "sha"})+ require.NoError(t, err)+ require.NotContains(t, string(raw), "background_active", "unflagged checkpoints keep their old shape")++ var flagged SandboxCheckpoint+ require.NoError(t, flagged.Scan([]byte(`{"sandbox_id":"sbx","commit_sha":"sha","background_active":true}`)))+ require.True(t, flagged.BackgroundActive)++ var old SandboxCheckpoint+ require.NoError(t, old.Scan([]byte(`{"sandbox_id":"sbx","commit_sha":"sha"}`)))+ require.False(t, old.BackgroundActive, "rows written before the flag existed stay usable")+}diff --git a/internal/types/session_fork.go b/internal/types/session_fork.goindex 10d07dcad..a6b07c814 100644--- a/internal/types/session_fork.go+++ b/internal/types/session_fork.go@@ -20,6 +20,10 @@ type SandboxCheckpoint struct {SandboxID string `json:"sandbox_id"`CommitSHA string `json:"commit_sha"`CommittedAt time.Time `json:"committed_at"`+ // BackgroundActive marks a commit taken while background work was live in+ // the session. The work may have been mid-write, so fork and rewind never+ // roll a workspace back to it.+ BackgroundActive bool `json:"background_active,omitempty"`}// Value implements the driver.Valuer interface for database serialization.