Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

feat(background): shared background-work runtime (#4579)

nullptr-errore28c68cf552c
diff --git a/internal/agent/background.go b/internal/agent/background.go
new file mode 100644
index 000000000..ab9439bb4
--- /dev/null
+++ b/internal/agent/background.go
@@ -0,0 +1,91 @@
+package agent
+
+import (
+ "context"
+ "strings"
+
+ "github.com/Tencent/WeKnora/internal/background"
+ "github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/models/chat"
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+// BackgroundRun is the run-scoped source of background-work context. The
+// engine asks it what to tell the model at the two boundaries steer uses, and
+// which finished items tools showed the model during a step.
+// *background.RunNotices implements it.
+type BackgroundRun interface {
+ RoundStart(ctx context.Context) (background.Delivery, error)
+ BeforeFinal(ctx context.Context) (background.Delivery, error)
+ TakeSeen() []string
+}
+
+// SetBackgroundRun wires background-work context for this run. Nil (the
+// default) disables it. Must be called before Execute.
+func (e *AgentEngine) SetBackgroundRun(run BackgroundRun) {
+ e.backgroundRun = run
+}
+
+// backgroundNoticesContent wraps background-work context so the model can tell
+// it apart from what the user typed.
+func backgroundNoticesContent(text string) string {
+ return "<background_notices>\n" + strings.TrimSpace(text) + "\n</background_notices>"
+}
+
+// injectBackground appends background-work context before the next LLM call
+// and queues it for the next step, which is where history replays it. It
+// returns how many messages it appended (0 or 1). Failures are logged and
+// skipped: the inbox keeps unacknowledged notices for a later boundary.
+func (e *AgentEngine) injectBackground(
+ ctx context.Context, state *types.AgentState, messagesPtr *[]chat.Message, beforeFinal bool,
+) int {
+ if e.backgroundRun == nil {
+ return 0
+ }
+ read := e.backgroundRun.RoundStart
+ if beforeFinal {
+ read = e.backgroundRun.BeforeFinal
+ }
+ delivery, err := read(ctx)
+ if err != nil {
+ logger.Warnf(ctx, "[Agent][Round-%d] Background notices unavailable: %v", state.CurrentRound+1, err)
+ return 0
+ }
+ if strings.TrimSpace(delivery.Text) == "" {
+ return 0
+ }
+ content := backgroundNoticesContent(delivery.Text)
+ *messagesPtr = append(*messagesPtr, chat.Message{Role: "user", Content: content})
+ state.PendingBackground = append(state.PendingBackground, types.BackgroundInjection{
+ Content: content, AckIDs: delivery.AckIDs,
+ })
+ return 1
+}
+
+// takeBackgroundSeen returns the finished IDs tools showed the model since the
+// previous call.
+func (e *AgentEngine) takeBackgroundSeen() []string {
+ if e.backgroundRun == nil {
+ return nil
+ }
+ return e.backgroundRun.TakeSeen()
+}
+
+// HasPendingInput reports whether a steered user message is waiting for the
+// run that owns messageID. Blocking tools poll it so the user is not kept
+// waiting behind them. It reads the queue without consuming it.
+func (e *AgentEngine) HasPendingInput(ctx context.Context, sessionID, messageID string) bool {
+ if e == nil || e.steerSink == nil {
+ return false
+ }
+ events, _, err := e.steerSink.PollSteer(ctx, sessionID, messageID, 0)
+ if err != nil {
+ return false
+ }
+ for _, evt := range events {
+ if sanitizeSteerContent(getString(evt, "content")) != "" {
+ return true
+ }
+ }
+ return false
+}
diff --git a/internal/agent/background_test.go b/internal/agent/background_test.go
new file mode 100644
index 000000000..0bed85851
--- /dev/null
+++ b/internal/agent/background_test.go
@@ -0,0 +1,171 @@
+package agent
+
+import (
+ "context"
+ "strings"
+ "testing"
+ "time"
+
+ agenttools "github.com/Tencent/WeKnora/internal/agent/tools"
+ "github.com/Tencent/WeKnora/internal/background"
+ "github.com/Tencent/WeKnora/internal/event"
+ "github.com/Tencent/WeKnora/internal/models/chat"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/require"
+)
+
+// fakeBackgroundRun hands out queued deliveries one call at a time.
+type fakeBackgroundRun struct {
+ roundStart []background.Delivery
+ beforeFinal []background.Delivery
+ seen [][]string
+}
+
+func (f *fakeBackgroundRun) RoundStart(context.Context) (background.Delivery, error) {
+ if len(f.roundStart) == 0 {
+ return background.Delivery{}, nil
+ }
+ d := f.roundStart[0]
+ f.roundStart = f.roundStart[1:]
+ return d, nil
+}
+
+func (f *fakeBackgroundRun) BeforeFinal(context.Context) (background.Delivery, error) {
+ if len(f.beforeFinal) == 0 {
+ return background.Delivery{}, nil
+ }
+ d := f.beforeFinal[0]
+ f.beforeFinal = f.beforeFinal[1:]
+ return d, nil
+}
+
+func (f *fakeBackgroundRun) TakeSeen() []string {
+ if len(f.seen) == 0 {
+ return nil
+ }
+ s := f.seen[0]
+ f.seen = f.seen[1:]
+ return s
+}
+
+func answer(content string) mockResponse {
+ return mockResponse{chunks: []types.StreamResponse{
+ {ResponseType: types.ResponseTypeAnswer, Content: content, Done: true, FinishReason: "stop"},
+ }}
+}
+
+// userMessageContaining returns the first user message sent to the model that
+// contains needle. Per-call envelopes may follow it, so position is not fixed.
+func userMessageContaining(messages []chat.Message, needle string) string {
+ for _, m := range messages {
+ if m.Role == "user" && strings.Contains(m.Content, needle) {
+ return m.Content
+ }
+ }
+ return ""
+}
+
+func TestBackgroundNoticeIsInjectedBeforeTheRoundAndRecordedOnItsStep(t *testing.T) {
+ model := &mockChat{responses: []mockResponse{answer("done")}}
+ engine := newTestEngine(t, model)
+ engine.SetBackgroundRun(&fakeBackgroundRun{roundStart: []background.Delivery{
+ {Text: "build (sj_1) succeeded.", AckIDs: []string{"sj_1"}},
+ }})
+
+ state := &types.AgentState{}
+ _, err := engine.executeLoop(context.Background(), state, "q", emptyMessages(), nil, "sess", "msg")
+ require.NoError(t, err)
+
+ want := "<background_notices>\nbuild (sj_1) succeeded.\n</background_notices>"
+ require.Contains(t, userMessageContaining(model.calls[0], "<background_notices>"), want)
+ require.Len(t, state.RoundSteps, 1)
+ require.Equal(t, []types.BackgroundInjection{{Content: want, AckIDs: []string{"sj_1"}}},
+ state.RoundSteps[0].BackgroundInjections)
+ require.Empty(t, state.PendingBackground)
+}
+
+func TestBackgroundReminderKeepsTheLoopGoing(t *testing.T) {
+ model := &mockChat{responses: []mockResponse{answer("first"), answer("second")}}
+ engine := newTestEngine(t, model)
+ engine.SetBackgroundRun(&fakeBackgroundRun{beforeFinal: []background.Delivery{
+ {Text: "Background work you started in this turn is still running: build (sj_1)."},
+ }})
+
+ state := &types.AgentState{}
+ _, err := engine.executeLoop(context.Background(), state, "q", emptyMessages(), nil, "sess", "msg")
+ require.NoError(t, err)
+
+ require.Len(t, model.calls, 2)
+ require.NotEmpty(t, userMessageContaining(model.calls[1], "build (sj_1)"))
+ require.Empty(t, userMessageContaining(model.calls[0], "build (sj_1)"), "the reminder comes only before finishing")
+ require.Equal(t, "second", state.FinalAnswer)
+ require.Len(t, state.RoundSteps, 2)
+ require.True(t, state.RoundSteps[0].IntermediateAnswer)
+ require.Len(t, state.RoundSteps[1].BackgroundInjections, 1)
+}
+
+func TestBackgroundSeenIDsRideOnTheToolStep(t *testing.T) {
+ model := &mockChat{responses: []mockResponse{
+ {chunks: []types.StreamResponse{{
+ ResponseType: types.ResponseTypeAnswer,
+ ToolCalls: []types.LLMToolCall{{
+ ID: "call-1", Type: "function",
+ Function: types.FunctionCall{Name: "probe", Arguments: "{}"},
+ }},
+ Done: true, FinishReason: "tool_calls",
+ }}},
+ answer("done"),
+ }}
+ engine := newTestEngine(t, model)
+ engine.toolRegistry = agenttools.NewToolRegistry()
+ engine.toolRegistry.RegisterTool(newCountingTool("probe"))
+ engine.SetBackgroundRun(&fakeBackgroundRun{seen: [][]string{{"sj_9"}}})
+
+ state := &types.AgentState{}
+ _, err := engine.executeLoop(context.Background(), state, "q", emptyMessages(), nil, "sess", "msg")
+ require.NoError(t, err)
+
+ require.Len(t, state.RoundSteps, 2)
+ require.Equal(t, []string{"sj_9"}, state.RoundSteps[0].BackgroundSeenIDs)
+ require.Empty(t, state.RoundSteps[1].BackgroundSeenIDs)
+}
+
+func TestCompletionKeepsUndeliveredBackgroundOnATrailingStep(t *testing.T) {
+ engine := newTestEngine(t, &mockChat{})
+ engine.eventBus = event.NewEventBus()
+ engine.SetBackgroundRun(&fakeBackgroundRun{seen: [][]string{{"sj_2"}}})
+ var steps []types.AgentStep
+ engine.eventBus.On(event.EventAgentComplete, func(_ context.Context, evt event.Event) error {
+ steps, _ = evt.Data.(event.AgentCompleteData).AgentSteps.([]types.AgentStep)
+ return nil
+ })
+ injection := types.BackgroundInjection{
+ Content: "<background_notices>\nx\n</background_notices>",
+ AckIDs: []string{"sj_1"},
+ }
+ state := &types.AgentState{PendingBackground: []types.BackgroundInjection{injection}}
+
+ engine.emitCompletionEvent(context.Background(), state, "sess", "msg", time.Now())
+
+ require.Len(t, steps, 1)
+ require.Equal(t, []types.BackgroundInjection{injection}, steps[0].BackgroundInjections)
+ require.Equal(t, []string{"sj_2"}, steps[0].BackgroundSeenIDs)
+}
+
+func TestHasPendingInputPeeksAtTheSteerQueue(t *testing.T) {
+ engine := newTestEngine(t, &mockChat{})
+ require.False(t, engine.HasPendingInput(context.Background(), "sess", "msg"), "no sink, nothing pending")
+
+ sink := &fakeSteerSink{queued: []map[string]interface{}{steerEntry("s1", " ")}}
+ engine.SetSteerSink(sink)
+ require.False(t, engine.HasPendingInput(context.Background(), "sess", "msg"), "blank messages are dropped by drain")
+
+ sink.queued = append(sink.queued, steerEntry("s2", "stop waiting"))
+ require.True(t, engine.HasPendingInput(context.Background(), "sess", "msg"))
+ require.Empty(t, sink.persisted, "peeking must not consume the message")
+}
+
+func TestBackgroundNoticesContentWrapsTrimmedText(t *testing.T) {
+ require.Equal(t, "<background_notices>\na\nb\n</background_notices>", backgroundNoticesContent(" a\nb \n"))
+ require.False(t, strings.Contains(backgroundNoticesContent("x"), "\n\n"))
+}
diff --git a/internal/agent/const.go b/internal/agent/const.go
index 64ffff2ad..881ee06f9 100644
--- a/internal/agent/const.go
+++ b/internal/agent/const.go
@@ -12,6 +12,7 @@ import (
"time"
"github.com/Tencent/WeKnora/internal/agent/compaction"
+ agenttools "github.com/Tencent/WeKnora/internal/agent/tools"
"github.com/Tencent/WeKnora/internal/browserskill"
webfetch "github.com/Tencent/WeKnora/internal/infrastructure/web_fetch"
"github.com/Tencent/WeKnora/internal/models/api"
@@ -47,6 +48,9 @@ const (
// shared page-fetch budget. Same 5s headroom as shell_exec, so web_fetch can
// return its own timeout result instead of being cancelled first.
webFetchToolSlack = 5 * time.Second
+ // workWaitToolSlack is added to work_wait's own wait so the final detailed
+ // read still fits after the wait elapses (D3).
+ workWaitToolSlack = 30 * time.Second
// maxLLMRetries is the maximum number of retries for transient LLM errors.
maxLLMRetries = 2
@@ -103,6 +107,13 @@ func toolExecutionTimeout(toolName string, arguments ...string) time.Duration {
return browserskill.HumanStepTimeout
}
}
+ if toolName == agenttools.ToolWorkWait {
+ raw := ""
+ if len(arguments) > 0 {
+ raw = arguments[0]
+ }
+ return time.Duration(agenttools.WorkWaitSeconds(raw))*time.Second + workWaitToolSlack
+ }
switch toolName {
case "shell_exec":
return shellExecToolTimeout
diff --git a/internal/agent/const_test.go b/internal/agent/const_test.go
index 51b20e43d..dc7a107c5 100644
--- a/internal/agent/const_test.go
+++ b/internal/agent/const_test.go
@@ -55,3 +55,10 @@ func TestToolExecutionTimeout(t *testing.T) {
assert.Equal(t, 35*time.Second, toolExecutionTimeout("web_fetch"))
})
}
+
+func TestWorkWaitToolTimeoutCoversTheWait(t *testing.T) {
+ assert.Equal(t, 90*time.Second, toolExecutionTimeout("work_wait", `{"work_ids":["a"]}`))
+ assert.Equal(t, 30*time.Second, toolExecutionTimeout("work_wait", `{"wait_sec":0}`))
+ assert.Equal(t, 330*time.Second, toolExecutionTimeout("work_wait", `{"wait_sec":999}`))
+ assert.Equal(t, 90*time.Second, toolExecutionTimeout("work_wait"))
+}
diff --git a/internal/agent/engine.go b/internal/agent/engine.go
index cc433de76..2133fb85f 100644
--- a/internal/agent/engine.go
+++ b/internal/agent/engine.go
@@ -88,7 +88,10 @@ type AgentEngine struct {
steerSink types.SteerSink
allowSteerOverrun bool // one extra ReAct round after a loop-end inject past MaxIterations
steerOverruns int // how many times this turn has already used the extra round
- workspaceLayout sandbox.WorkspaceLayout
+ // backgroundRun, when set, supplies background-work notices and reminders
+ // at the same two boundaries as steer. Nil disables them.
+ backgroundRun BackgroundRun
+ workspaceLayout sandbox.WorkspaceLayout
// suspendable lets a user prompt end the run so it can be resumed later;
// suspendRequest is saved with the run for that (see EnableSuspension).
suspendable bool
@@ -103,8 +106,9 @@ type AgentEngine struct {
}
// maxSteerOverruns caps loop-end injects past MaxIterations. One extra round
-// lets a last-moment nudge revise the answer; further injects stay queued for
-// a follow-up turn instead of stretching the same run indefinitely.
+// lets a last-moment nudge (a steered message or a background reminder) revise
+// the answer; further injects stay queued for a follow-up turn instead of
+// stretching the same run indefinitely.
const maxSteerOverruns = 1
// ImageDescriberFunc generates a text description of an image.
@@ -890,6 +894,9 @@ func (e *AgentEngine) runReActIteration(
if guards.childAnswerRevised {
tools = nil
}
+ // Background-work context rides the same boundary for the same reasons,
+ // after the user's own messages.
+ e.injectBackground(ctx, state, messagesPtr, false)
logger.Infof(ctx, "[Agent][Round-%d/%s] Starting: %d messages, %d tools, est_tokens=%d",
round, e.maxIterationsDisplay(), len(*messagesPtr), len(tools), currentTokens)
@@ -1040,16 +1047,18 @@ func (e *AgentEngine) runReActIteration(
// Create agent step
step := types.AgentStep{
- UserMessagesBefore: state.PendingSteerMessages,
- Iteration: state.CurrentRound,
- Thought: response.Content,
- ReasoningContent: response.ReasoningContent,
- ReasoningSignature: response.ReasoningSignature,
- ReasoningMetadata: response.ReasoningMetadata,
- ToolCalls: make([]types.ToolCall, 0),
- Timestamp: time.Now(),
+ UserMessagesBefore: state.PendingSteerMessages,
+ BackgroundInjections: state.PendingBackground,
+ Iteration: state.CurrentRound,
+ Thought: response.Content,
+ ReasoningContent: response.ReasoningContent,
+ ReasoningSignature: response.ReasoningSignature,
+ ReasoningMetadata: response.ReasoningMetadata,
+ ToolCalls: make([]types.ToolCall, 0),
+ Timestamp: time.Now(),
}
state.PendingSteerMessages = nil
+ state.PendingBackground = nil
// If the request was cancelled while the LLM was streaming (e.g. the
// user pressed "stop"), the stream driver still returns a usable
@@ -1064,7 +1073,8 @@ func (e *AgentEngine) runReActIteration(
if ctx.Err() != nil {
logger.Warnf(ctx, "[Agent][Round-%d] Context cancelled during LLM call; preserving partial step",
round)
- if step.Thought != "" || len(step.ToolCalls) > 0 || len(step.UserMessagesBefore) > 0 {
+ if step.Thought != "" || len(step.ToolCalls) > 0 || len(step.UserMessagesBefore) > 0 ||
+ len(step.BackgroundInjections) > 0 {
state.RoundSteps = append(state.RoundSteps, step)
}
return iterOutcomeBreak, nil
@@ -1132,6 +1142,7 @@ func (e *AgentEngine) runReActIteration(
guards.emptyRetries++
if guards.emptyRetries <= maxEmptyResponseRetries {
state.PendingSteerMessages = step.UserMessagesBefore
+ state.PendingBackground = step.BackgroundInjections
logger.Warnf(ctx, "[Agent][Round-%d] Empty content with stop - retrying (%d/%d)",
round, guards.emptyRetries, maxEmptyResponseRetries)
guards.pendingRuntimeNotice = emptyResponseRetryNotice
@@ -1176,7 +1187,8 @@ func (e *AgentEngine) runReActIteration(
return iterOutcomeBreak, nil
}
// Loop-end inject: a user message queued while this finishing round
- // ran should keep the agent going instead of emitting a final answer.
+ // ran, or background work this run started that is still running,
+ // keeps the agent going instead of emitting a final answer.
// Content-filter stops are terminal and do not take this path.
// Past MaxIterations only the first inject gets an extra round;
// anything after that stays in the queue for a follow-up turn.
@@ -1192,6 +1204,7 @@ func (e *AgentEngine) runReActIteration(
ReasoningMetadata: response.ReasoningMetadata,
})
injected := e.drainSteerMessages(ctx, state, messagesPtr, sessionID, assistantMessageID)
+ injected += e.injectBackground(ctx, state, messagesPtr, true)
if injected > 0 {
*messagesPtr = e.appendCollaborationDeliveryBoundary(*messagesPtr, verdict.finalAnswer)
verdict.step.IntermediateAnswer = true
@@ -1339,6 +1352,7 @@ func (e *AgentEngine) runRoundTools(
e.beginRound(*step, response.ToolCalls)
suspended := e.executeToolCalls(ctx, response, step, state.CurrentRound, sessionID, assistantMessageID)
e.endRound()
+ step.BackgroundSeenIDs = e.takeBackgroundSeen()
if suspended == nil {
return false
}
diff --git a/internal/agent/finalize.go b/internal/agent/finalize.go
index 1826ce010..feb963308 100644
--- a/internal/agent/finalize.go
+++ b/internal/agent/finalize.go
@@ -181,11 +181,15 @@ func (e *AgentEngine) emitCompletionEvent(
if state.SuspendedStep != nil {
steps = append(append([]types.AgentStep(nil), steps...), *state.SuspendedStep)
}
- if len(state.PendingSteerMessages) > 0 {
+ seen := e.takeBackgroundSeen()
+ if len(state.PendingSteerMessages) > 0 || len(state.PendingBackground) > 0 || len(seen) > 0 {
// A stop or model failure can arrive after delivery but before the next
// response exists. Preserve that boundary without inventing an answer.
steps = append(append([]types.AgentStep(nil), steps...), types.AgentStep{
- Iteration: state.CurrentRound, UserMessagesBefore: state.PendingSteerMessages,
+ Iteration: state.CurrentRound,
+ UserMessagesBefore: state.PendingSteerMessages,
+ BackgroundInjections: state.PendingBackground,
+ BackgroundSeenIDs: seen,
})
}
// Convert knowledge refs to interface{} slice for event data
diff --git a/internal/agent/tools/definitions.go b/internal/agent/tools/definitions.go
index d1ecabee8..fbcf4fdb8 100644
--- a/internal/agent/tools/definitions.go
+++ b/internal/agent/tools/definitions.go
@@ -69,6 +69,14 @@ const (
// keys it on SkillsEnabled (or install mode), so the shell follows the
// skills switch and not a per-agent tool checkbox.
ToolShellExec = "shell_exec"
+ // ToolShellExecAsync submits a shell command as background work (plan 4).
+ // Like shell_exec it is registered per run, not in AvailableToolDefinitions.
+ ToolShellExecAsync = "shell_exec_async"
+ // ToolWorkWait / ToolWorkCancel wait on and stop background work. They are
+ // registered per run, only when the run can submit background work, so
+ // they are absent from AvailableToolDefinitions like shell_exec.
+ ToolWorkWait = "work_wait"
+ ToolWorkCancel = "work_cancel"
// Wiki-related tools (only available when wiki KBs are in scope)
ToolWikiReadPage = "wiki_read_page"
ToolWikiWritePage = "wiki_write_page"
@@ -177,6 +185,7 @@ func SwitchManagedTools() []string {
ToolWebFetch,
ToolSearchMemory,
ToolShellExec,
+ ToolShellExecAsync,
ToolReadFile,
ToolListSandboxFiles,
ToolWriteSandboxFile,
diff --git a/internal/agent/tools/tool_policy_coverage_test.go b/internal/agent/tools/tool_policy_coverage_test.go
index 4e99fadc0..22bec5c13 100644
--- a/internal/agent/tools/tool_policy_coverage_test.go
+++ b/internal/agent/tools/tool_policy_coverage_test.go
@@ -36,6 +36,9 @@ func TestEveryBuiltInToolDeclaresAModelHandlePolicy(t *testing.T) {
ToolWriteSkillFile,
ToolEditSkillFile,
ToolShellExec,
+ ToolShellExecAsync,
+ ToolWorkWait,
+ ToolWorkCancel,
ToolWikiReadPage,
ToolWikiWritePage,
ToolWikiReplaceText,
diff --git a/internal/agent/tools/work.go b/internal/agent/tools/work.go
new file mode 100644
index 000000000..bfb21c1f8
--- /dev/null
+++ b/internal/agent/tools/work.go
@@ -0,0 +1,511 @@
+package tools
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/Tencent/WeKnora/internal/utils"
+)
+
+const (
+ workWaitDefaultSec = 60
+ workWaitMaxSec = 300
+ workWaitPoll = time.Second
+ // workMaxIDs bounds one call so each ID's share of the output budget stays useful.
+ workMaxIDs = 16
+ // workOutputBudget caps the stdout and stderr bodies of one work_wait call (D6).
+ workOutputBudget = 32 * 1024
+ // workWaitOutputLimit is the rune cap ToolRegistry applies to the marshaled
+ // work_wait body. JSON escaping can turn one payload byte into \u00XX, and
+ // the envelope adds ids, states, and the note. The generic 24000-rune cap
+ // would cut a full result into invalid JSON.
+ workWaitOutputLimit = workOutputBudget*6 + 8*1024
+ // workProbeBudget asks a state-only probe for as little output as possible.
+ workProbeBudget = 1
+)
+
+const workWaitDescription = `Wait for background work only when you cannot continue without its result.
+work_ids come from %s. Do not call it merely because work is still running; you will
+be told when it finishes. wait_sec: seconds to wait (default 60, max 300); 0 returns
+the current state and new output immediately. mode: all (default) or any. Returns
+early if the user sends a message.`
+
+const workCancelDescription = `Stop background work by work_id.`
+
+const (
+ workNoteRunning = "Still running; you will be told when it finishes."
+ workNoteInterrupted = "The user sent a new message; it is shown to you next. The work keeps running."
+ workNoteCancelling = "Cancellation requested; you will be told when it stops."
+)
+
+// backgroundSubmissionTools are the tools that start background work. A run
+// gets work_wait and work_cancel only when it has at least one of them.
+var backgroundSubmissionTools []string
+
+// RegisteredBackgroundSubmitters returns the background submission tools
+// registered in r, in declaration order.
+func RegisteredBackgroundSubmitters(r *ToolRegistry) []string {
+ var out []string
+ for _, name := range backgroundSubmissionTools {
+ if _, err := r.GetTool(name); err == nil {
+ out = append(out, name)
+ }
+ }
+ return out
+}
+
+// WorkMarks is the run-scoped tracker the work tools report to.
+// *background.RunNotices implements it.
+type WorkMarks interface {
+ MarkSeen(ids ...string)
+}
+
+// WorkWaitInput documents the work_wait arguments for the schema.
+type WorkWaitInput struct {
+ WorkIDs []string `json:"work_ids" jsonschema:"IDs of the background work to wait for"`
+ WaitSec int `json:"wait_sec,omitempty" jsonschema:"Seconds to wait, default 60, max 300; 0 returns at once"`
+ Mode string `json:"mode,omitempty" jsonschema:"all (default) waits for every ID; any returns when one ends"`
+}
+
+// WorkCancelInput documents the work_cancel arguments for the schema.
+type WorkCancelInput struct {
+ WorkIDs []string `json:"work_ids" jsonschema:"IDs of the background work to stop"`
+}
+
+type workWaitArgs struct {
+ WorkIDs []string `json:"work_ids"`
+ WaitSec *int `json:"wait_sec"`
+ Mode string `json:"mode"`
+}
+
+// WorkWaitSeconds returns the wait of a work_wait call: default 60, clamped to
+// 0–300. Malformed arguments get the default. The engine derives the tool
+// timeout from it (D3).
+func WorkWaitSeconds(arguments string) int {
+ var args struct {
+ WaitSec *int `json:"wait_sec"`
+ }
+ if json.Unmarshal([]byte(arguments), &args) != nil || args.WaitSec == nil {
+ return workWaitDefaultSec
+ }
+ return min(max(*args.WaitSec, 0), workWaitMaxSec)
+}
+
+type workItemView struct {
+ WorkID string `json:"work_id"`
+ State string `json:"state"`
+ Title string `json:"title,omitempty"`
+ Result any `json:"result,omitempty"`
+}
+
+type workView struct {
+ Items []workItemView `json:"items"`
+ TimedOut bool `json:"timed_out,omitempty"`
+ InterruptedByInput bool `json:"interrupted_by_input,omitempty"`
+ Note string `json:"note,omitempty"`
+}
+
+// WorkWaitTool waits on background work of the run's session.
+type WorkWaitTool struct {
+ BaseTool
+ handler interfaces.WorkHandler
+ session interfaces.SessionRef
+ runID string
+ marks WorkMarks
+ pending func(context.Context) bool
+ now func() time.Time
+ sleep func(context.Context, time.Duration) bool
+
+ // waitedRunning records IDs that already had a blocking wait and were still
+ // running. A later blocking wait in this run does not sleep on them again.
+ mu sync.Mutex
+ waitedRunning map[string]struct{}
+}
+
+// NewWorkWaitTool returns work_wait bound to one run. runID scopes incremental
+// output reads; pending reports a waiting user message; submitters are named
+// in the description.
+func NewWorkWaitTool(
+ handler interfaces.WorkHandler, session interfaces.SessionRef, runID string,
+ marks WorkMarks, pending func(context.Context) bool, submitters []string,
+) *WorkWaitTool {
+ description := fmt.Sprintf(workWaitDescription, strings.Join(submitters, ", "))
+ return &WorkWaitTool{
+ BaseTool: NewBaseTool(ToolWorkWait, description, utils.GenerateSchema[WorkWaitInput]()),
+ handler: handler,
+ session: session,
+ runID: runID,
+ marks: marks,
+ pending: pending,
+ now: time.Now,
+ sleep: workSleep,
+ }
+}
+
+// Execute implements types.Tool.
+func (t *WorkWaitTool) Execute(ctx context.Context, raw json.RawMessage) (*types.ToolResult, error) {
+ var args workWaitArgs
+ if err := json.Unmarshal(raw, &args); err != nil {
+ return workError("invalid arguments: " + err.Error()), nil
+ }
+ ids, err := normalizeWorkIDs(args.WorkIDs)
+ if err != nil {
+ return workError(err.Error()), nil
+ }
+ mode := strings.ToLower(strings.TrimSpace(args.Mode))
+ if mode == "" {
+ mode = "all"
+ }
+ if mode != "all" && mode != "any" {
+ return workError(`mode must be "all" or "any"`), nil
+ }
+ waitSec := WorkWaitSeconds(string(raw))
+
+ timedOut, interrupted := false, false
+ if waitSec > 0 {
+ fresh := t.unwaited(ids)
+ if len(fresh) > 0 {
+ var err error
+ timedOut, interrupted, err = t.poll(ctx, fresh, mode, time.Duration(waitSec)*time.Second)
+ if err != nil {
+ return nil, err
+ }
+ }
+ }
+ results, err := t.status(ctx, ids, interfaces.StatusOptions{
+ Detailed: true, RunID: t.runID, OutputBudget: workOutputBudget / len(ids),
+ })
+ if err != nil {
+ return workReadError(ids, err)
+ }
+ if waitSec > 0 {
+ t.rememberRunning(results)
+ }
+ markTerminal(t.marks, results)
+
+ note := ""
+ switch {
+ case interrupted && anyRunning(results):
+ note = workNoteInterrupted
+ case anyRunning(results):
+ note = workNoteRunning
+ }
+ rendered := renderWork(results, note, timedOut && anyRunning(results), interrupted)
+ rendered.OutputFiles = collectOutputFiles(results)
+ return rendered, nil
+}
+
+func collectOutputFiles(results []interfaces.WorkResult) []string {
+ inspected := false
+ var links []string
+ for _, result := range results {
+ if result.OutputFiles == nil {
+ continue
+ }
+ inspected = true
+ links = append(links, result.OutputFiles...)
+ }
+ if !inspected {
+ return nil
+ }
+ if links == nil {
+ return []string{}
+ }
+ return links
+}
+
+// OutputLimitChars keeps a full work_wait body, including JSON framing and
+// escaping, inside the registry's truncation ceiling.
+func (t *WorkWaitTool) OutputLimitChars(json.RawMessage) int {
+ return workWaitOutputLimit
+}
+
+// poll waits with cheap reads until mode is satisfied, the user sends a
+// message, or wait elapses.
+func (t *WorkWaitTool) poll(
+ ctx context.Context, ids []string, mode string, wait time.Duration,
+) (timedOut, interrupted bool, err error) {
+ deadline := t.now().Add(wait)
+ probed := make(map[string]bool, len(ids))
+ for {
+ results, callErr := t.handler.Status(ctx, t.session, ids, interfaces.StatusOptions{RunID: t.runID})
+ if workStopped(callErr) {
+ return false, false, callErr
+ }
+ if errors.Is(callErr, interfaces.ErrWorkNotFound) {
+ return false, false, nil
+ }
+ // A failed cheap read is not "everything finished". Keep waiting; the
+ // final detailed read is what the model sees.
+ if callErr == nil {
+ results, callErr = checkWorkResults(ids, results, nil)
+ if callErr != nil {
+ return false, false, callErr
+ }
+ var unknown []string
+ for _, r := range results {
+ if r.State == interfaces.WorkStateUnknown && !probed[r.ID] {
+ probed[r.ID] = true
+ unknown = append(unknown, r.ID)
+ }
+ }
+ if len(unknown) > 0 {
+ // No RunID: a probe must not move the read position the final
+ // detailed read hands to the model.
+ probe, probeErr := t.handler.Status(ctx, t.session, unknown, interfaces.StatusOptions{
+ Detailed: true, OutputBudget: workProbeBudget,
+ })
+ if workStopped(probeErr) {
+ return false, false, probeErr
+ }
+ if errors.Is(probeErr, interfaces.ErrWorkNotFound) {
+ return false, false, nil
+ }
+ if probeErr == nil {
+ probe, probeErr = checkWorkResults(unknown, probe, nil)
+ if probeErr != nil {
+ return false, false, probeErr
+ }
+ byID := make(map[string]interfaces.WorkResult, len(probe))
+ for _, r := range probe {
+ byID[r.ID] = r
+ }
+ for i, r := range results {
+ if p, ok := byID[r.ID]; ok {
+ results[i] = p
+ }
+ }
+ }
+ }
+ if workSatisfied(results, mode) {
+ return false, false, nil
+ }
+ }
+ if t.pending != nil && t.pending(ctx) {
+ return false, true, nil
+ }
+ if !t.now().Before(deadline) {
+ return true, false, nil
+ }
+ if !t.sleep(ctx, workWaitPoll) {
+ return false, false, ctx.Err()
+ }
+ }
+}
+
+func (t *WorkWaitTool) unwaited(ids []string) []string {
+ t.mu.Lock()
+ defer t.mu.Unlock()
+ fresh := make([]string, 0, len(ids))
+ for _, id := range ids {
+ if _, waited := t.waitedRunning[id]; !waited {
+ fresh = append(fresh, id)
+ }
+ }
+ return fresh
+}
+
+func (t *WorkWaitTool) rememberRunning(results []interfaces.WorkResult) {
+ t.mu.Lock()
+ defer t.mu.Unlock()
+ if t.waitedRunning == nil {
+ t.waitedRunning = map[string]struct{}{}
+ }
+ for _, result := range results {
+ if !result.Terminal && result.ID != "" {
+ t.waitedRunning[result.ID] = struct{}{}
+ }
+ }
+}
+
+func (t *WorkWaitTool) status(
+ ctx context.Context, ids []string, opts interfaces.StatusOptions,
+) ([]interfaces.WorkResult, error) {
+ results, err := t.handler.Status(ctx, t.session, ids, opts)
+ return checkWorkResults(ids, results, err)
+}
+
+// workStopped reports a cancelled or timed-out call. Those are not a lost
+// batch: Execute returns the error, and the IDs are not marked seen.
+func workStopped(err error) bool {
+ return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded)
+}
+
+// WorkCancelTool stops background work of the run's session.
+type WorkCancelTool struct {
+ BaseTool
+ handler interfaces.WorkHandler
+ session interfaces.SessionRef
+ marks WorkMarks
+}
+
+// NewWorkCancelTool returns work_cancel bound to one run.
+func NewWorkCancelTool(handler interfaces.WorkHandler, session interfaces.SessionRef, marks WorkMarks) *WorkCancelTool {
+ return &WorkCancelTool{
+ BaseTool: NewBaseTool(ToolWorkCancel, workCancelDescription, utils.GenerateSchema[WorkCancelInput]()),
+ handler: handler,
+ session: session,
+ marks: marks,
+ }
+}
+
+// Execute implements types.Tool.
+func (t *WorkCancelTool) Execute(ctx context.Context, raw json.RawMessage) (*types.ToolResult, error) {
+ var args WorkCancelInput
+ if err := json.Unmarshal(raw, &args); err != nil {
+ return workError("invalid arguments: " + err.Error()), nil
+ }
+ ids, err := normalizeWorkIDs(args.WorkIDs)
+ if err != nil {
+ return workError(err.Error()), nil
+ }
+ results, err := t.handler.Cancel(ctx, t.session, ids)
+ results, err = checkWorkResults(ids, results, err)
+ if err != nil {
+ return workReadError(ids, err)
+ }
+ markTerminal(t.marks, results)
+ note := ""
+ if anyRunning(results) {
+ note = workNoteCancelling
+ }
+ return renderWork(results, note, false, false), nil
+}
+
+// checkWorkResults enforces the WorkHandler contract of one result per ID in
+// input order. Results are matched by position, so a reordered reply would
+// hand one item's state to another; it fails the batch instead (D14).
+func checkWorkResults(ids []string, results []interfaces.WorkResult, err error) ([]interfaces.WorkResult, error) {
+ if err == nil && len(results) != len(ids) {
+ err = fmt.Errorf("handler returned %d results for %d ids", len(results), len(ids))
+ }
+ for i := 0; err == nil && i < len(ids); i++ {
+ if results[i].ID != ids[i] {
+ err = fmt.Errorf("handler returned id %q at position %d, want %q", results[i].ID, i, ids[i])
+ }
+ }
+ if err != nil {
+ return nil, err
+ }
+ for i := range results {
+ results[i].Terminal = interfaces.IsTerminalWorkState(results[i].State)
+ }
+ return results, nil
+}
+
+// workReadError never acknowledges IDs from a failed batch. Even
+// ErrWorkNotFound may refer to only one of several requested IDs; it does not
+// confirm that the other jobs ended or that their results reached the model.
+func workReadError(ids []string, err error) (*types.ToolResult, error) {
+ if !errors.Is(err, interfaces.ErrWorkNotFound) {
+ return nil, err
+ }
+ if len(ids) > 1 {
+ return workError("One or more work IDs were not found in this session; check them individually."), nil
+ }
+ results := make([]interfaces.WorkResult, len(ids))
+ for i, id := range ids {
+ results[i] = interfaces.WorkResult{
+ ID: id, State: interfaces.WorkStateLost, Terminal: true,
+ Payload: map[string]any{"error": "work not found in this session"},
+ }
+ }
+ return renderWork(results, "", false, false), nil
+}
+
+func normalizeWorkIDs(raw []string) ([]string, error) {
+ ids := make([]string, 0, len(raw))
+ seen := make(map[string]bool, len(raw))
+ for _, id := range raw {
+ id = strings.TrimSpace(id)
+ if id == "" || seen[id] {
+ continue
+ }
+ seen[id] = true
+ ids = append(ids, id)
+ }
+ if len(ids) == 0 {
+ return nil, errors.New("work_ids must name at least one work_id")
+ }
+ if len(ids) > workMaxIDs {
+ return nil, fmt.Errorf("work_ids may name at most %d work_ids", workMaxIDs)
+ }
+ return ids, nil
+}
+
+func workSatisfied(results []interfaces.WorkResult, mode string) bool {
+ finished := 0
+ for _, r := range results {
+ if r.Terminal {
+ finished++
+ }
+ }
+ if mode == "any" {
+ return finished > 0
+ }
+ return finished == len(results)
+}
+
+func anyRunning(results []interfaces.WorkResult) bool {
+ for _, r := range results {
+ if !r.Terminal {
+ return true
+ }
+ }
+ return false
+}
+
+func markTerminal(marks WorkMarks, results []interfaces.WorkResult) {
+ if marks == nil {
+ return
+ }
+ var ids []string
+ for _, r := range results {
+ if r.Terminal {
+ ids = append(ids, r.ID)
+ }
+ }
+ if len(ids) > 0 {
+ marks.MarkSeen(ids...)
+ }
+}
+
+func renderWork(results []interfaces.WorkResult, note string, timedOut, interrupted bool) *types.ToolResult {
+ view := workView{
+ Items: make([]workItemView, 0, len(results)),
+ TimedOut: timedOut,
+ InterruptedByInput: interrupted,
+ Note: note,
+ }
+ for _, r := range results {
+ view.Items = append(view.Items, workItemView{WorkID: r.ID, State: r.State, Title: r.Title, Result: r.Payload})
+ }
+ raw, err := json.Marshal(view)
+ if err != nil {
+ return workError("encode work results: " + err.Error())
+ }
+ return &types.ToolResult{Success: true, Output: string(raw), Data: map[string]interface{}{"work": view}}
+}
+
+func workError(msg string) *types.ToolResult {
+ return &types.ToolResult{Success: false, Error: msg}
+}
+
+// workSleep waits d, returning false if ctx ends first.
+func workSleep(ctx context.Context, d time.Duration) bool {
+ timer := time.NewTimer(d)
+ defer timer.Stop()
+ select {
+ case <-ctx.Done():
+ return false
+ case <-timer.C:
+ return true
+ }
+}
diff --git a/internal/agent/tools/work_test.go b/internal/agent/tools/work_test.go
new file mode 100644
index 000000000..c376b073a
--- /dev/null
+++ b/internal/agent/tools/work_test.go
@@ -0,0 +1,436 @@
+package tools
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/background"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+var workSession = interfaces.SessionRef{TenantID: 1, SessionID: "s"}
+
+// fakeWorkHandler serves a scripted sequence of cheap states per ID; the last
+// entry repeats. Detailed reads return detailed[id] (default "running").
+type fakeWorkHandler struct {
+ mu sync.Mutex
+ cheap map[string][]string
+ detailed map[string]string
+ calls []interfaces.StatusOptions
+ err error
+ detailedErr error
+ reverse bool
+ omitLast bool
+ cancelled [][]string
+}
+
+func (h *fakeWorkHandler) Status(
+ ctx context.Context, _ interfaces.SessionRef, ids []string, opts interfaces.StatusOptions,
+) ([]interfaces.WorkResult, error) {
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ h.calls = append(h.calls, opts)
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ if h.err != nil {
+ return nil, h.err
+ }
+ if opts.Detailed && h.detailedErr != nil {
+ return nil, h.detailedErr
+ }
+ out := make([]interfaces.WorkResult, 0, len(ids))
+ for _, id := range ids {
+ state := interfaces.WorkStateRunning
+ if opts.Detailed {
+ if s, ok := h.detailed[id]; ok {
+ state = s
+ }
+ } else if seq := h.cheap[id]; len(seq) > 0 {
+ state = seq[0]
+ if len(seq) > 1 {
+ h.cheap[id] = seq[1:]
+ }
+ }
+ out = append(out, interfaces.WorkResult{
+ ID: id, State: state, Title: "job " + id,
+ Payload: map[string]any{"stdout": "out-" + id},
+ })
+ }
+ if h.reverse {
+ for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 {
+ out[i], out[j] = out[j], out[i]
+ }
+ }
+ if h.omitLast && len(out) > 0 {
+ out = out[:len(out)-1]
+ }
+ return out, nil
+}
+
+func (h *fakeWorkHandler) Cancel(
+ ctx context.Context, _ interfaces.SessionRef, ids []string,
+) ([]interfaces.WorkResult, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ if h.err != nil {
+ return nil, h.err
+ }
+ h.cancelled = append(h.cancelled, ids)
+ out := make([]interfaces.WorkResult, 0, len(ids))
+ for _, id := range ids {
+ state := interfaces.WorkStateRunning
+ if id == "sj_done" {
+ state = interfaces.WorkStateSucceeded
+ }
+ out = append(out, interfaces.WorkResult{ID: id, State: state})
+ }
+ return out, nil
+}
+
+type recordingMarks struct {
+ seen []string
+ started []string
+}
+
+func (m *recordingMarks) MarkSeen(ids ...string) { m.seen = append(m.seen, ids...) }
+
+func (m *recordingMarks) MarkStarted(id string) { m.started = append(m.started, id) }
+
+// newTestWorkWait returns a tool on a fake clock: sleeping advances time.
+func newTestWorkWait(h *fakeWorkHandler, marks WorkMarks, pending func(context.Context) bool) (*WorkWaitTool, *int) {
+ tool := NewWorkWaitTool(h, workSession, "run-1", marks, pending, []string{"shell_exec_async"})
+ now := time.Date(2026, 10, 15, 10, 0, 0, 0, time.UTC)
+ sleeps := 0
+ tool.now = func() time.Time { return now }
+ tool.sleep = func(_ context.Context, d time.Duration) bool {
+ sleeps++
+ now = now.Add(d)
+ return true
+ }
+ return tool, &sleeps
+}
+
+func decodeWork(t *testing.T, output string) workView {
+ t.Helper()
+ var view workView
+ require.NoError(t, json.Unmarshal([]byte(output), &view))
+ return view
+}
+
+func TestWorkWaitZeroReadsOnceInDetail(t *testing.T) {
+ h := &fakeWorkHandler{detailed: map[string]string{"sj_1": interfaces.WorkStateSucceeded}}
+ marks := &recordingMarks{}
+ tool, sleeps := newTestWorkWait(h, marks, nil)
+
+ res, err := tool.Execute(context.Background(), json.RawMessage(`{"work_ids":["sj_1"],"wait_sec":0}`))
+
+ require.NoError(t, err)
+ require.True(t, res.Success)
+ require.Zero(t, *sleeps)
+ require.Equal(t, []interfaces.StatusOptions{{
+ Detailed: true, RunID: "run-1", OutputBudget: workOutputBudget,
+ }}, h.calls)
+ view := decodeWork(t, res.Output)
+ require.Equal(t, interfaces.WorkStateSucceeded, view.Items[0].State)
+ require.Empty(t, view.Note)
+ require.Equal(t, []string{"sj_1"}, marks.seen)
+}
+
+func TestWorkWaitAllWaitsForEveryID(t *testing.T) {
+ h := &fakeWorkHandler{
+ cheap: map[string][]string{
+ "sj_1": {interfaces.WorkStateRunning, interfaces.WorkStateSucceeded},
+ "sj_2": {interfaces.WorkStateRunning, interfaces.WorkStateRunning, interfaces.WorkStateFailed},
+ },
+ detailed: map[string]string{"sj_1": interfaces.WorkStateSucceeded, "sj_2": interfaces.WorkStateFailed},
+ }
+ marks := &recordingMarks{}
+ tool, sleeps := newTestWorkWait(h, marks, nil)
+
+ res, err := tool.Execute(context.Background(), json.RawMessage(`{"work_ids":["sj_1","sj_2"],"wait_sec":30}`))
+
+ require.NoError(t, err)
+ require.Equal(t, 2, *sleeps)
+ view := decodeWork(t, res.Output)
+ require.False(t, view.TimedOut)
+ require.Equal(t, []string{"sj_1", "sj_2"}, marks.seen)
+ final := h.calls[len(h.calls)-1]
+ require.Equal(t, interfaces.StatusOptions{
+ Detailed: true, RunID: "run-1", OutputBudget: workOutputBudget / 2,
+ }, final)
+}
+
+func TestWorkWaitAnyReturnsOnTheFirstFinish(t *testing.T) {
+ h := &fakeWorkHandler{
+ cheap: map[string][]string{
+ "sj_1": {interfaces.WorkStateRunning},
+ "sj_2": {interfaces.WorkStateRunning, interfaces.WorkStateSucceeded},
+ },
+ detailed: map[string]string{"sj_2": interfaces.WorkStateSucceeded},
+ }
+ tool, sleeps := newTestWorkWait(h, &recordingMarks{}, nil)
+
+ res, err := tool.Execute(context.Background(),
+ json.RawMessage(`{"work_ids":["sj_1","sj_2"],"wait_sec":30,"mode":"any"}`))
+
+ require.NoError(t, err)
+ require.Equal(t, 1, *sleeps)
+ view := decodeWork(t, res.Output)
+ require.Equal(t, "Still running; you will be told when it finishes.", view.Note)
+}
+
+func TestWorkWaitDoesNotBlockAgainForTheSameRunningID(t *testing.T) {
+ h := &fakeWorkHandler{}
+ tool, sleeps := newTestWorkWait(h, &recordingMarks{}, nil)
+
+ first, err := tool.Execute(context.Background(), json.RawMessage(`{"work_ids":["sj_1"],"wait_sec":3}`))
+ require.NoError(t, err)
+ require.Equal(t, 3, *sleeps)
+ require.Equal(t, workNoteRunning, decodeWork(t, first.Output).Note)
+
+ second, err := tool.Execute(context.Background(), json.RawMessage(`{"work_ids":["sj_1"],"wait_sec":60}`))
+ require.NoError(t, err)
+ require.Equal(t, 3, *sleeps, "a second blocking wait must not sleep")
+ view := decodeWork(t, second.Output)
+ require.False(t, view.TimedOut)
+ require.Equal(t, workNoteRunning, view.Note)
+ require.Equal(t, interfaces.WorkStateRunning, view.Items[0].State)
+}
+
+func TestWorkWaitTimesOut(t *testing.T) {
+ h := &fakeWorkHandler{}
+ marks := &recordingMarks{}
+ tool, sleeps := newTestWorkWait(h, marks, nil)
+
+ res, err := tool.Execute(context.Background(), json.RawMessage(`{"work_ids":["sj_1"],"wait_sec":3}`))
+
+ require.NoError(t, err)
+ require.Equal(t, 3, *sleeps)
+ view := decodeWork(t, res.Output)
+ require.True(t, view.TimedOut)
+ require.Equal(t, "Still running; you will be told when it finishes.", view.Note)
+ require.Empty(t, marks.seen)
+}
+
+func TestWorkWaitReturnsEarlyWhenTheUserTypes(t *testing.T) {
+ h := &fakeWorkHandler{}
+ tool, sleeps := newTestWorkWait(h, &recordingMarks{}, func(context.Context) bool { return true })
+
+ res, err := tool.Execute(context.Background(), json.RawMessage(`{"work_ids":["sj_1"],"wait_sec":300}`))
+
+ require.NoError(t, err)
+ require.Zero(t, *sleeps)
+ view := decodeWork(t, res.Output)
+ require.True(t, view.InterruptedByInput)
+ require.Equal(t, "The user sent a new message; it is shown to you next. The work keeps running.", view.Note)
+}
+
+func TestWorkWaitProbesUnknownOnceWithoutMovingTheReadPosition(t *testing.T) {
+ h := &fakeWorkHandler{
+ cheap: map[string][]string{"sj_1": {interfaces.WorkStateUnknown}},
+ detailed: map[string]string{"sj_1": interfaces.WorkStateSucceeded},
+ }
+ tool, _ := newTestWorkWait(h, &recordingMarks{}, nil)
+
+ _, err := tool.Execute(context.Background(), json.RawMessage(`{"work_ids":["sj_1"],"wait_sec":30}`))
+
+ require.NoError(t, err)
+ require.Equal(t, []interfaces.StatusOptions{
+ {RunID: "run-1"},
+ {Detailed: true, OutputBudget: workProbeBudget},
+ {Detailed: true, RunID: "run-1", OutputBudget: workOutputBudget},
+ }, h.calls)
+}
+
+func TestWorkWaitFailedReadsDoNotAcknowledgeNotices(t *testing.T) {
+ for name, h := range map[string]*fakeWorkHandler{
+ "error": {err: errors.New("sandbox unreachable")},
+ "not found": {err: fmt.Errorf("wrap: %w", interfaces.ErrWorkNotFound)},
+ "reordered": {reverse: true},
+ "incomplete": {omitLast: true},
+ } {
+ t.Run(name, func(t *testing.T) {
+ ctx := context.Background()
+ inbox := background.NewMemoryNoticeInbox(nil)
+ marks := background.NewRunNotices(workSession, background.NewMemoryActivityRegistry(nil), inbox)
+ tool, _ := newTestWorkWait(h, marks, nil)
+
+ res, err := tool.Execute(ctx, json.RawMessage(`{"work_ids":["sj_1","sj_2"],"wait_sec":0}`))
+
+ if errors.Is(h.err, interfaces.ErrWorkNotFound) {
+ require.NoError(t, err)
+ require.False(t, res.Success)
+ require.Empty(t, res.Output, "a missing ID must not turn unrelated IDs into lost jobs")
+ } else {
+ require.Error(t, err)
+ require.Nil(t, res)
+ }
+ require.Empty(t, marks.TakeSeen(), "a failed batch must not acknowledge any requested ID")
+ _, err = inbox.Push(ctx, workSession, interfaces.BackgroundNotice{
+ ID: "sj_1", Text: "build succeeded", FinishedAt: time.Now(),
+ })
+ require.NoError(t, err)
+ delivery, err := marks.RoundStart(ctx)
+ require.NoError(t, err)
+ require.Equal(t, "build succeeded", delivery.Text, "the later real completion must still reach the model")
+ })
+ }
+}
+
+func TestWorkWaitMissingWorkReturnsWithoutSleeping(t *testing.T) {
+ for _, mode := range []string{"all", "any"} {
+ for _, source := range []string{"cheap", "probe"} {
+ t.Run(mode+"/"+source, func(t *testing.T) {
+ missing := fmt.Errorf("status: %w", interfaces.ErrWorkNotFound)
+ h := &fakeWorkHandler{err: missing}
+ wantCalls := 2
+ if source == "probe" {
+ h = &fakeWorkHandler{
+ cheap: map[string][]string{"missing": {interfaces.WorkStateUnknown}}, detailedErr: missing,
+ }
+ wantCalls = 3
+ }
+ marks := &recordingMarks{}
+ tool, sleeps := newTestWorkWait(h, marks, nil)
+ res, err := tool.Execute(context.Background(), json.RawMessage(
+ fmt.Sprintf(`{"work_ids":["missing"],"wait_sec":300,"mode":%q}`, mode)))
+ require.NoError(t, err)
+ require.Zero(t, *sleeps)
+ require.Len(t, h.calls, wantCalls)
+ require.True(t, h.calls[wantCalls-1].Detailed)
+ require.Equal(t, "run-1", h.calls[wantCalls-1].RunID, "always finish with the run's detailed read")
+ require.Equal(t, interfaces.WorkStateLost, decodeWork(t, res.Output).Items[0].State)
+ require.Empty(t, marks.seen)
+ })
+ }
+ }
+}
+
+func TestWorkCancelFailedReadsDoNotAcknowledgeNotices(t *testing.T) {
+ for _, callErr := range []error{errors.New("connection interrupted"), interfaces.ErrWorkNotFound} {
+ marks := &recordingMarks{}
+ tool := NewWorkCancelTool(&fakeWorkHandler{err: callErr}, workSession, marks)
+ res, err := tool.Execute(context.Background(), json.RawMessage(`{"work_ids":["sj_1","sj_2"]}`))
+ if errors.Is(callErr, interfaces.ErrWorkNotFound) {
+ require.NoError(t, err)
+ require.False(t, res.Success)
+ require.Empty(t, res.Output)
+ } else {
+ require.ErrorIs(t, err, callErr)
+ }
+ require.Empty(t, marks.seen)
+ }
+}
+
+func TestWorkWaitCancelIsAnErrorAndIsNotSeen(t *testing.T) {
+ marks := &recordingMarks{}
+ tool, _ := newTestWorkWait(&fakeWorkHandler{}, marks, nil)
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+
+ res, err := tool.Execute(ctx, json.RawMessage(`{"work_ids":["sj_1"],"wait_sec":30}`))
+
+ require.ErrorIs(t, err, context.Canceled)
+ require.Nil(t, res)
+ require.Empty(t, marks.seen)
+}
+
+func TestWorkCancelContextIsAnErrorAndIsNotSeen(t *testing.T) {
+ marks := &recordingMarks{}
+ tool := NewWorkCancelTool(&fakeWorkHandler{}, workSession, marks)
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+
+ res, err := tool.Execute(ctx, json.RawMessage(`{"work_ids":["sj_1"]}`))
+
+ require.ErrorIs(t, err, context.Canceled)
+ require.Nil(t, res)
+ require.Empty(t, marks.seen)
+}
+
+func TestWorkWaitInterruptedNoteOnlyWhileSomethingRuns(t *testing.T) {
+ h := &fakeWorkHandler{detailed: map[string]string{"sj_1": interfaces.WorkStateSucceeded}}
+ marks := &recordingMarks{}
+ tool, _ := newTestWorkWait(h, marks, func(context.Context) bool { return true })
+
+ res, err := tool.Execute(context.Background(), json.RawMessage(`{"work_ids":["sj_1"],"wait_sec":30}`))
+
+ require.NoError(t, err)
+ view := decodeWork(t, res.Output)
+ require.True(t, view.InterruptedByInput)
+ require.Empty(t, view.Note)
+ require.Equal(t, []string{"sj_1"}, marks.seen)
+}
+
+func TestWorkWaitOutputLimitCoversTheMarshaledBudget(t *testing.T) {
+ tool, _ := newTestWorkWait(&fakeWorkHandler{}, nil, nil)
+ limit := tool.OutputLimitChars(json.RawMessage(`{"work_ids":["sj_1"]}`))
+ require.Greater(t, limit, DefaultMaxToolOutput)
+ require.Equal(t, workWaitOutputLimit, limit)
+}
+
+func TestWorkWaitRejectsBadArguments(t *testing.T) {
+ tool, _ := newTestWorkWait(&fakeWorkHandler{}, &recordingMarks{}, nil)
+ tooMany := `{"work_ids":["a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q"]}`
+ for _, args := range []string{
+ `{"work_ids":[]}`, `{"work_ids":[" "]}`, tooMany, `{"work_ids":["a"],"mode":"some"}`,
+ } {
+ res, err := tool.Execute(context.Background(), json.RawMessage(args))
+ require.NoError(t, err)
+ require.False(t, res.Success, args)
+ }
+}
+
+func TestWorkWaitSecondsClamps(t *testing.T) {
+ require.Equal(t, 60, WorkWaitSeconds(`{"work_ids":["a"]}`))
+ require.Equal(t, 0, WorkWaitSeconds(`{"wait_sec":-5}`))
+ require.Equal(t, 300, WorkWaitSeconds(`{"wait_sec":999}`))
+ require.Equal(t, 0, WorkWaitSeconds(`{"wait_sec":0}`))
+ require.Equal(t, 60, WorkWaitSeconds(`{`))
+}
+
+func TestWorkCancelReportsAndMarksTerminal(t *testing.T) {
+ h := &fakeWorkHandler{}
+ marks := &recordingMarks{}
+ tool := NewWorkCancelTool(h, workSession, marks)
+
+ res, err := tool.Execute(context.Background(), json.RawMessage(`{"work_ids":["sj_run","sj_done","sj_run"]}`))
+
+ require.NoError(t, err)
+ require.Equal(t, [][]string{{"sj_run", "sj_done"}}, h.cancelled, "IDs are de-duplicated in order")
+ require.Equal(t, []string{"sj_done"}, marks.seen)
+ view := decodeWork(t, res.Output)
+ require.Equal(t, "Cancellation requested; you will be told when it stops.", view.Note)
+}
+
+func TestWorkToolDescriptionsNameTheSubmitters(t *testing.T) {
+ tool := NewWorkWaitTool(&fakeWorkHandler{}, workSession, "run-1", nil, nil, []string{"shell_exec_async"})
+ require.Contains(t, tool.Description(), "work_ids come from shell_exec_async.")
+ require.False(t, CanRunConcurrently(ToolWorkWait))
+ require.False(t, CanRunConcurrently(ToolWorkCancel))
+}
+
+func TestRegisteredBackgroundSubmitters(t *testing.T) {
+ saved := backgroundSubmissionTools
+ t.Cleanup(func() { backgroundSubmissionTools = saved })
+ backgroundSubmissionTools = []string{"submit_a", "submit_b"}
+ registry := NewToolRegistry()
+ registry.RegisterTool(&fakeNamedTool{BaseTool: NewBaseTool("submit_b", "x", json.RawMessage(`{"type":"object"}`))})
+
+ require.Equal(t, []string{"submit_b"}, RegisteredBackgroundSubmitters(registry))
+}
+
+type fakeNamedTool struct{ BaseTool }
+
+func (*fakeNamedTool) Execute(context.Context, json.RawMessage) (*types.ToolResult, error) {
+ return nil, nil
+}
diff --git a/internal/application/service/agent_history.go b/internal/application/service/agent_history.go
index 13af64c7f..b97e10ad3 100644
--- a/internal/application/service/agent_history.go
+++ b/internal/application/service/agent_history.go
@@ -452,6 +452,7 @@ func buildTurnBodyMessages(assistant *types.Message, midRunUsers []*types.Messag
appendUser(midRunUsers[next])
next++
}
+ out = append(out, backgroundInjectionMessages(step)...)
out = append(out, buildAgentStepMessages(step)...)
}
for _, user := range midRunUsers {
@@ -508,6 +509,7 @@ func buildUserHistoryMessage(m *types.Message) chat.Message {
func buildAssistantHistoryMessages(m *types.Message) []chat.Message {
msgs := make([]chat.Message, 0, len(m.AgentSteps)*2+1)
for _, step := range m.AgentSteps {
+ msgs = append(msgs, backgroundInjectionMessages(step)...)
msgs = append(msgs, buildAgentStepMessages(step)...)
}
if final := finalAnswerHistoryMessage(m); final != nil {
@@ -516,6 +518,19 @@ func buildAssistantHistoryMessages(m *types.Message) []chat.Message {
return msgs
}
+// backgroundInjectionMessages replays the background-work messages delivered
+// before a step, where the model saw them.
+func backgroundInjectionMessages(step types.AgentStep) []chat.Message {
+ out := make([]chat.Message, 0, len(step.BackgroundInjections))
+ for _, injection := range step.BackgroundInjections {
+ if strings.TrimSpace(injection.Content) == "" {
+ continue
+ }
+ out = append(out, chat.Message{Role: "user", Content: injection.Content})
+ }
+ return out
+}
+
// buildAgentStepMessages expands one persisted step into the OpenAI-shaped
// assistant + tool pair. Steps whose only calls were terminal or synthetic
// produce nothing, so callers can treat an empty result as "not a real round".
diff --git a/internal/application/service/agent_history_background_test.go b/internal/application/service/agent_history_background_test.go
new file mode 100644
index 000000000..cada608f9
--- /dev/null
+++ b/internal/application/service/agent_history_background_test.go
@@ -0,0 +1,70 @@
+package service
+
+import (
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/models/chat"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/require"
+)
+
+const replayNotice = "<background_notices>\nbuild (sj_1) succeeded.\n</background_notices>"
+
+func backgroundStepAssistant() *types.Message {
+ return &types.Message{
+ ID: "a-1", SessionID: "s", Role: "assistant", Content: "all green",
+ AgentSteps: []types.AgentStep{
+ {
+ Iteration: 0, Thought: "checking the build", IntermediateAnswer: true,
+ Timestamp: time.Date(2026, 10, 15, 10, 0, 1, 0, time.UTC),
+ BackgroundInjections: []types.BackgroundInjection{{Content: replayNotice, AckIDs: []string{"sj_1"}}},
+ },
+ },
+ }
+}
+
+func rolesAndContents(msgs []chat.Message) [][2]string {
+ out := make([][2]string, 0, len(msgs))
+ for _, m := range msgs {
+ out = append(out, [2]string{m.Role, m.Content})
+ }
+ return out
+}
+
+func TestAssistantHistoryReplaysBackgroundInjectionBeforeItsStep(t *testing.T) {
+ got := rolesAndContents(buildAssistantHistoryMessages(backgroundStepAssistant()))
+
+ require.Equal(t, [][2]string{
+ {"user", replayNotice},
+ {"assistant", "checking the build"},
+ {"assistant", "all green"},
+ }, got)
+}
+
+func TestTurnBodyReplaysSteerThenBackgroundThenStep(t *testing.T) {
+ assistant := backgroundStepAssistant()
+ assistant.AgentSteps[0].UserMessagesBefore = []string{"u-steer"}
+ steer := &types.Message{
+ ID: "u-steer", SessionID: "s", Role: "user", Content: "also run lint",
+ CreatedAt: time.Date(2026, 10, 15, 10, 0, 0, 0, time.UTC),
+ }
+
+ got := rolesAndContents(buildTurnBodyMessages(assistant, []*types.Message{steer}))
+
+ require.Len(t, got, 4)
+ require.Equal(t, "user", got[0][0])
+ require.Contains(t, got[0][1], "also run lint")
+ require.Equal(t, [2]string{"user", replayNotice}, got[1])
+ require.Equal(t, [2]string{"assistant", "checking the build"}, got[2])
+ require.Equal(t, [2]string{"assistant", "all green"}, got[3])
+}
+
+func TestHistoryIgnoresBlankBackgroundInjections(t *testing.T) {
+ assistant := backgroundStepAssistant()
+ assistant.AgentSteps[0].BackgroundInjections = []types.BackgroundInjection{{Content: " "}}
+
+ got := rolesAndContents(buildAssistantHistoryMessages(assistant))
+
+ require.Equal(t, "assistant", got[0][0])
+}
diff --git a/internal/application/service/background_activity_helpers_test.go b/internal/application/service/background_activity_helpers_test.go
new file mode 100644
index 000000000..abcca6d82
--- /dev/null
+++ b/internal/application/service/background_activity_helpers_test.go
@@ -0,0 +1,35 @@
+package service
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/background"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+// liveActivityRegistry returns an in-memory registry holding one live
+// background item for each ref.
+func liveActivityRegistry(t *testing.T, refs ...interfaces.SessionRef) *background.MemoryActivityRegistry {
+ t.Helper()
+ reg := background.NewMemoryActivityRegistry(nil)
+ for i, ref := range refs {
+ require.NoError(t, reg.Add(context.Background(), ref, interfaces.BackgroundActivity{
+ ID: fmt.Sprintf("sj_%d", i), Kind: "shell_job", Title: "build", StartedAt: time.Now(),
+ }, time.Hour, 0))
+ }
+ return reg
+}
+
+// failingActivityRegistry fails every List call.
+type failingActivityRegistry struct {
+ interfaces.ActivityRegistry
+ err error
+}
+
+func (r failingActivityRegistry) List(context.Context, interfaces.SessionRef) ([]interfaces.BackgroundActivity, error) {
+ return nil, r.err
+}
diff --git a/internal/application/service/background_keepalive.go b/internal/application/service/background_keepalive.go
new file mode 100644
index 000000000..5efc41ff9
--- /dev/null
+++ b/internal/application/service/background_keepalive.go
@@ -0,0 +1,91 @@
+package service
+
+import (
+ "context"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+// backgroundKeepAliveTimeout bounds one provider refresh.
+const backgroundKeepAliveTimeout = 10 * time.Second
+
+// backgroundKeepAlivePruneAt is how many tracked sessions trigger dropping
+// entries whose refresh is already due.
+const backgroundKeepAlivePruneAt = 1024
+
+// sessionKeepAliveTarget is the sandbox side BackgroundKeepAliver drives.
+// *PinnedSessionSandbox implements it.
+type sessionKeepAliveTarget interface {
+ KeepAliveSession(ctx context.Context, sessionID string) (time.Duration, error)
+}
+
+// BackgroundKeepAliver keeps the sandbox of a session with live background
+// work from being reclaimed by its provider. Callers may invoke it on every
+// poll; it reaches the provider only once the interval the last successful
+// refresh reported has passed.
+type BackgroundKeepAliver struct {
+ target sessionKeepAliveTarget
+ now func() time.Time
+
+ mu sync.Mutex
+ next map[interfaces.SessionRef]time.Time
+}
+
+var _ interfaces.SessionKeepAliver = (*BackgroundKeepAliver)(nil)
+
+// NewBackgroundKeepAliver returns a keep-aliver over the session's pinned sandbox.
+func NewBackgroundKeepAliver(pinned *PinnedSessionSandbox) *BackgroundKeepAliver {
+ return newBackgroundKeepAliver(pinned, time.Now)
+}
+
+func newBackgroundKeepAliver(target sessionKeepAliveTarget, now func() time.Time) *BackgroundKeepAliver {
+ return &BackgroundKeepAliver{
+ target: target,
+ now: now,
+ next: make(map[interfaces.SessionRef]time.Time),
+ }
+}
+
+// KeepAlive implements interfaces.SessionKeepAliver.
+func (k *BackgroundKeepAliver) KeepAlive(ctx context.Context, s interfaces.SessionRef) error {
+ if k == nil || k.target == nil || s.TenantID == 0 || strings.TrimSpace(s.SessionID) == "" {
+ return nil
+ }
+ now := k.now()
+ k.mu.Lock()
+ due := k.next[s]
+ k.mu.Unlock()
+ if now.Before(due) {
+ return nil
+ }
+
+ // Pollers have no request tenant. The pin lookup reads the execution
+ // tenant; the binding key reads the sandbox tenant. Both are the owner.
+ ctx = types.WithSandboxTenantID(types.WithExecutionTenant(ctx, s.TenantID), s.TenantID)
+ ctx, cancel := context.WithTimeout(ctx, backgroundKeepAliveTimeout)
+ defer cancel()
+ interval, err := k.target.KeepAliveSession(ctx, s.SessionID)
+ if err != nil {
+ return err
+ }
+
+ k.mu.Lock()
+ defer k.mu.Unlock()
+ if interval > 0 {
+ k.next[s] = now.Add(interval)
+ } else {
+ delete(k.next, s)
+ }
+ if len(k.next) >= backgroundKeepAlivePruneAt {
+ for ref, at := range k.next {
+ if !now.Before(at) {
+ delete(k.next, ref)
+ }
+ }
+ }
+ return nil
+}
diff --git a/internal/application/service/background_keepalive_test.go b/internal/application/service/background_keepalive_test.go
new file mode 100644
index 000000000..42fbb7225
--- /dev/null
+++ b/internal/application/service/background_keepalive_test.go
@@ -0,0 +1,76 @@
+package service
+
+import (
+ "context"
+ "errors"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+type fakeKeepAliveTarget struct {
+ next time.Duration
+ err error
+ calls []string
+ tenants []uint64
+}
+
+func (f *fakeKeepAliveTarget) KeepAliveSession(ctx context.Context, sessionID string) (time.Duration, error) {
+ f.calls = append(f.calls, sessionID)
+ tenant, _ := types.SandboxTenantIDFromContext(ctx)
+ f.tenants = append(f.tenants, tenant)
+ return f.next, f.err
+}
+
+func TestBackgroundKeepAliverThrottlesPerSession(t *testing.T) {
+ now := time.Date(2026, 10, 15, 10, 0, 0, 0, time.UTC)
+ target := &fakeKeepAliveTarget{next: 10 * time.Minute}
+ k := newBackgroundKeepAliver(target, func() time.Time { return now })
+ ctx := context.Background()
+ s1 := interfaces.SessionRef{TenantID: 7, SessionID: "s1"}
+ s2 := interfaces.SessionRef{TenantID: 7, SessionID: "s2"}
+
+ require.NoError(t, k.KeepAlive(ctx, s1))
+ require.NoError(t, k.KeepAlive(ctx, s1))
+ require.NoError(t, k.KeepAlive(ctx, s2))
+ require.Equal(t, []string{"s1", "s2"}, target.calls, "a refresh within the interval is skipped")
+
+ now = now.Add(10 * time.Minute)
+ require.NoError(t, k.KeepAlive(ctx, s1))
+ require.Equal(t, []string{"s1", "s2", "s1"}, target.calls)
+ require.Equal(t, []uint64{7, 7, 7}, target.tenants, "the session owner's tenant keys the binding")
+}
+
+func TestBackgroundKeepAliverRetriesAfterFailure(t *testing.T) {
+ now := time.Date(2026, 10, 15, 10, 0, 0, 0, time.UTC)
+ target := &fakeKeepAliveTarget{next: 10 * time.Minute, err: errors.New("provider down")}
+ k := newBackgroundKeepAliver(target, func() time.Time { return now })
+ s := interfaces.SessionRef{TenantID: 7, SessionID: "s1"}
+
+ require.Error(t, k.KeepAlive(context.Background(), s))
+ require.Error(t, k.KeepAlive(context.Background(), s))
+ require.Len(t, target.calls, 2, "a failed refresh must not start the quiet interval")
+}
+
+func TestBackgroundKeepAliverWithoutIntervalDoesNotThrottle(t *testing.T) {
+ now := time.Date(2026, 10, 15, 10, 0, 0, 0, time.UTC)
+ target := &fakeKeepAliveTarget{}
+ k := newBackgroundKeepAliver(target, func() time.Time { return now })
+ s := interfaces.SessionRef{TenantID: 7, SessionID: "s1"}
+
+ require.NoError(t, k.KeepAlive(context.Background(), s))
+ require.NoError(t, k.KeepAlive(context.Background(), s))
+ require.Len(t, target.calls, 2, "nothing was bound yet, so the next call must look again")
+}
+
+func TestBackgroundKeepAliverIgnoresIncompleteRef(t *testing.T) {
+ target := &fakeKeepAliveTarget{next: time.Minute}
+ k := newBackgroundKeepAliver(target, time.Now)
+
+ require.NoError(t, k.KeepAlive(context.Background(), interfaces.SessionRef{SessionID: "s1"}))
+ require.NoError(t, k.KeepAlive(context.Background(), interfaces.SessionRef{TenantID: 7}))
+ require.Empty(t, target.calls)
+}
diff --git a/internal/application/service/background_work.go b/internal/application/service/background_work.go
new file mode 100644
index 000000000..b1d3d616e
--- /dev/null
+++ b/internal/application/service/background_work.go
@@ -0,0 +1,32 @@
+package service
+
+import (
+ "github.com/Tencent/WeKnora/internal/background"
+ "github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+// BackgroundWork bundles the shared background-work parts the agent, the
+// session handlers and features use. Handler is nil until a feature that
+// submits background work is wired; until then Enabled is false and nothing
+// background-related reaches a run.
+type BackgroundWork struct {
+ Registry interfaces.ActivityRegistry
+ Inbox interfaces.NoticeInbox
+ Listeners *background.Listeners
+ Handler interfaces.WorkHandler
+}
+
+// NewBackgroundWork returns the bundle with an empty listener set.
+func NewBackgroundWork(registry interfaces.ActivityRegistry, inbox interfaces.NoticeInbox) *BackgroundWork {
+ return &BackgroundWork{
+ Registry: registry,
+ Inbox: inbox,
+ Listeners: background.NewListeners(registry, inbox, logger.Warnf),
+ }
+}
+
+// Enabled reports whether background work can exist at all.
+func (b *BackgroundWork) Enabled() bool {
+ return b != nil && b.Handler != nil && b.Registry != nil && b.Inbox != nil
+}
diff --git a/internal/application/service/pinned_session_sandbox.go b/internal/application/service/pinned_session_sandbox.go
index 9edd2d2c7..af03a4a12 100644
--- a/internal/application/service/pinned_session_sandbox.go
+++ b/internal/application/service/pinned_session_sandbox.go
@@ -74,6 +74,17 @@ func (a *PinnedSessionSandbox) manager(ctx context.Context, sessionID string) sa
return mgr
}
+// BackendIsDocker reports whether the session's pinned manager is the Docker
+// backend. The container id is not a backend name, so callers must not infer
+// this from the sandbox id. A missing pin is not Docker.
+func (a *PinnedSessionSandbox) BackendIsDocker(ctx context.Context, sessionID string) bool {
+ mgr := a.manager(ctx, sessionID)
+ if mgr == nil {
+ return false
+ }
+ return mgr.GetType() == sandbox.SandboxTypeDocker
+}
+
// BoundSandboxID reports the sandbox currently bound to sessionID without
// provisioning one. A session with no pin or no live binding reports
// ok=false.
@@ -129,6 +140,52 @@ func (a *PinnedSessionSandbox) HasActiveTurn(ctx context.Context, sessionID stri
return false, nil
}
+// KeepAliveSession refreshes the provider idle timer of the session's pinned
+// sandbox. Managers without the method (host, disabled) have nothing to keep
+// alive and report (0, nil).
+func (a *PinnedSessionSandbox) KeepAliveSession(ctx context.Context, sessionID string) (time.Duration, error) {
+ mgr := a.manager(ctx, sessionID)
+ type keepAliver interface {
+ KeepAliveSession(context.Context, string) (time.Duration, error)
+ }
+ if k, ok := mgr.(keepAliver); ok {
+ return k.KeepAliveSession(ctx, sessionID)
+ }
+ return 0, nil
+}
+
+// KeepAliveSessionExpected refreshes the provider timer of the bound sandbox
+// only when it still is expectedID. Managers without the method have nothing
+// to refresh and report (0, nil).
+func (a *PinnedSessionSandbox) KeepAliveSessionExpected(
+ ctx context.Context, sessionID, expectedID string,
+) (time.Duration, error) {
+ mgr := a.manager(ctx, sessionID)
+ type expectedKeepAliver interface {
+ KeepAliveSessionExpected(context.Context, string, string) (time.Duration, error)
+ }
+ if k, ok := mgr.(expectedKeepAliver); ok {
+ return k.KeepAliveSessionExpected(ctx, sessionID, expectedID)
+ }
+ return 0, nil
+}
+
+// ListSessionFiles walks dir in the session sandbox. A manager without the
+// method reports an error so callers skip output-file inspection.
+func (a *PinnedSessionSandbox) ListSessionFiles(
+ ctx context.Context, sessionID, dir string,
+) ([]sandbox.RemoteDirEntry, error) {
+ mgr := a.manager(ctx, sessionID)
+ type lister interface {
+ ListSessionFiles(context.Context, string, string) ([]sandbox.RemoteDirEntry, error)
+ }
+ files, ok := mgr.(lister)
+ if !ok || files == nil {
+ return nil, errors.New("sandbox: session files cannot be listed")
+ }
+ return files.ListSessionFiles(ctx, sessionID, dir)
+}
+
// TryLockRewind locks rewind through the session's pinned manager. A manager
// that can take the session exclusively returns the context to reset with;
// one with only the rewind lock keeps ctx; others succeed as a no-op.
diff --git a/internal/application/service/pinned_session_sandbox_test.go b/internal/application/service/pinned_session_sandbox_test.go
index 18972bfa6..920ef0f20 100644
--- a/internal/application/service/pinned_session_sandbox_test.go
+++ b/internal/application/service/pinned_session_sandbox_test.go
@@ -235,3 +235,82 @@ func TestHostManagerForTreatsGlobalDefaultAsUnnamed(t *testing.T) {
require.Equal(t, host, r.HostManagerFor(context.Background(), "s1"))
}
+
+type stubKeepAlivePinnedManager struct {
+ stubPinnedManager
+ next time.Duration
+ calls int
+}
+
+func (m *stubKeepAlivePinnedManager) KeepAliveSession(context.Context, string) (time.Duration, error) {
+ m.calls++
+ return m.next, nil
+}
+
+func TestPinnedSessionSandboxKeepAliveSessionDelegates(t *testing.T) {
+ mgr := &stubKeepAlivePinnedManager{next: 4 * time.Minute}
+ access := NewPinnedSessionSandbox(stubPinReader{configID: "cfg-1"}, &stubTenantSandboxResolver{mgr: mgr}, nil, nil)
+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(9))
+
+ next, err := access.KeepAliveSession(ctx, "sess-1")
+
+ require.NoError(t, err)
+ require.Equal(t, 4*time.Minute, next)
+ require.Equal(t, 1, mgr.calls)
+}
+
+type stubExpectedKeepAliveManager struct {
+ stubKeepAlivePinnedManager
+ expectedID string
+ calls int
+}
+
+func (m *stubExpectedKeepAliveManager) KeepAliveSessionExpected(
+ _ context.Context, _ string, expectedID string,
+) (time.Duration, error) {
+ m.calls++
+ m.expectedID = expectedID
+ return 3 * time.Minute, nil
+}
+
+func TestPinnedSessionSandboxKeepAliveSessionExpectedDelegates(t *testing.T) {
+ mgr := &stubExpectedKeepAliveManager{}
+ access := NewPinnedSessionSandbox(stubPinReader{configID: "cfg-1"}, &stubTenantSandboxResolver{mgr: mgr}, nil, nil)
+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(9))
+
+ next, err := access.KeepAliveSessionExpected(ctx, "sess-1", "sbx-expected")
+
+ require.NoError(t, err)
+ require.Equal(t, 3*time.Minute, next)
+ require.Equal(t, 1, mgr.calls)
+ require.Equal(t, "sbx-expected", mgr.expectedID)
+}
+
+func TestPinnedSessionSandboxKeepAliveSessionExpectedWhenManagerLacksMethod(t *testing.T) {
+ access := NewPinnedSessionSandbox(
+ stubPinReader{configID: "cfg-1"},
+ &stubTenantSandboxResolver{mgr: &stubPinnedManager{id: "sbx-1", ok: true}},
+ nil, nil,
+ )
+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(9))
+
+ next, err := access.KeepAliveSessionExpected(ctx, "sess-1", "sbx-1")
+
+ require.NoError(t, err)
+ require.Zero(t, next)
+}
+
+func TestPinnedSessionSandboxKeepAliveSessionWhenManagerLacksMethod(t *testing.T) {
+ access := NewPinnedSessionSandbox(
+ stubPinReader{configID: "cfg-1"},
+ &stubTenantSandboxResolver{mgr: &stubPinnedManager{id: "sbx-1", ok: true}},
+ nil,
+ nil,
+ )
+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(9))
+
+ next, err := access.KeepAliveSession(ctx, "sess-1")
+
+ require.NoError(t, err)
+ require.Zero(t, next)
+}
diff --git a/internal/application/service/session.go b/internal/application/service/session.go
index 3dc384436..7e892c5f7 100644
--- a/internal/application/service/session.go
+++ b/internal/application/service/session.go
@@ -182,6 +182,7 @@ type sessionService struct {
// remote resolver) keeps the historical delete path and writes no
// allocation rows.
workspaceCleaner *sandboxalloc.WorkspaceCleaner
+ background *BackgroundWork
}
// NewSessionService creates a new session service instance with all required dependencies
@@ -208,6 +209,7 @@ func NewSessionService(cfg *config.Config,
sandboxConfigRepo repository.TenantSandboxConfigRepository,
tenantSkillRepo repository.TenantSkillRepository,
busyGate *SessionBusyGate,
+ backgroundWork *BackgroundWork,
) interfaces.SessionService {
return &sessionService{
cfg: cfg,
@@ -235,6 +237,7 @@ func NewSessionService(cfg *config.Config,
sandboxConfigRepo: sandboxConfigRepo,
tenantSkillRepo: tenantSkillRepo,
busyGate: busyGate,
+ background: backgroundWork,
}
}
@@ -590,6 +593,7 @@ func (s *sessionService) DeleteSession(ctx context.Context, id string) error {
}
}
+ s.notifyBackgroundDelete(ctx, session)
if !handled[id] {
s.destroyBoundSandbox(ctx, id)
}
@@ -692,6 +696,9 @@ func (s *sessionService) BatchDeleteSessions(ctx context.Context, ids []string)
// Tear down sandboxes while session rows (and pins) are still readable.
// A session on a shared machine keeps that machine; only its directory
// is removed after the rows are gone.
+ for _, session := range visible {
+ s.notifyBackgroundDelete(ctx, session)
+ }
for _, id := range visibleIDs {
if handled[id] {
continue
@@ -773,7 +780,11 @@ func (s *sessionService) DeleteAllSessions(ctx context.Context) error {
// Tear down sandboxes while session rows (and pins) are still readable.
for _, session := range sessions {
- if session == nil || session.ID == "" || handled[session.ID] {
+ if session == nil || session.ID == "" {
+ continue
+ }
+ s.notifyBackgroundDelete(ctx, session)
+ if handled[session.ID] {
continue
}
s.destroyBoundSandbox(ctx, session.ID)
@@ -876,6 +887,18 @@ func (s *sessionService) runUserWorkspaceCleanups(ctx context.Context, handled m
go s.workspaceCleaner.RunCleanups(bg, ids)
}
+// notifyBackgroundDelete lets background features stop their work for a
+// session about to be deleted, then clears its activity items and notices.
+// Call it before destroyBoundSandbox, while features can still reach the
+// sandbox, and while the session row is still readable.
+func (s *sessionService) notifyBackgroundDelete(ctx context.Context, session *types.Session) {
+ if s.background == nil || s.background.Listeners == nil || session == nil {
+ return
+ }
+ s.background.Listeners.NotifyDelete(context.WithoutCancel(ctx),
+ interfaces.SessionRef{TenantID: session.TenantID, SessionID: session.ID})
+}
+
// destroyBoundSandbox tears down the sandbox MicroVM bound to sessionID, if
// the configured sandbox backend supports session-scoped instances.
//
diff --git a/internal/application/service/session_delete_background_test.go b/internal/application/service/session_delete_background_test.go
new file mode 100644
index 000000000..8a018c415
--- /dev/null
+++ b/internal/application/service/session_delete_background_test.go
@@ -0,0 +1,79 @@
+package service
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/background"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+ "gorm.io/gorm"
+)
+
+type recordingDeleteListener struct {
+ deleted []interfaces.SessionRef
+}
+
+func (l *recordingDeleteListener) OnStop(context.Context, interfaces.SessionRef) error { return nil }
+
+func (l *recordingDeleteListener) OnDelete(_ context.Context, s interfaces.SessionRef) error {
+ l.deleted = append(l.deleted, s)
+ return nil
+}
+
+func newDeleteBackgroundFixture(t *testing.T, ids ...string) (*sessionService, *recordingDeleteListener) {
+ t.Helper()
+ svc, db, _ := newSessionServiceForForkDeleteTest(t)
+ work := NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ listener := &recordingDeleteListener{}
+ work.Listeners.Register(listener)
+ svc.background = work
+ for _, id := range ids {
+ require.NoError(t, db.Session(&gorm.Session{SkipHooks: true}).Create(&types.Session{
+ ID: id, TenantID: 1, UserID: "u1", Title: id,
+ }).Error)
+ ref := interfaces.SessionRef{TenantID: 1, SessionID: id}
+ require.NoError(t, work.Registry.Add(context.Background(), ref, interfaces.BackgroundActivity{
+ ID: "sj_" + id, Kind: "shell_job", StartedAt: time.Now(),
+ }, time.Hour, 0))
+ }
+ return svc, listener
+}
+
+func requireNoBackgroundLeft(t *testing.T, svc *sessionService, id string) {
+ t.Helper()
+ live, err := svc.background.Registry.List(context.Background(), interfaces.SessionRef{TenantID: 1, SessionID: id})
+ require.NoError(t, err)
+ require.Empty(t, live)
+}
+
+func TestDeleteSessionNotifiesBackgroundListeners(t *testing.T) {
+ svc, listener := newDeleteBackgroundFixture(t, "s1")
+
+ require.NoError(t, svc.DeleteSession(testSessionScopeContext(1, "u1"), "s1"))
+
+ require.Equal(t, []interfaces.SessionRef{{TenantID: 1, SessionID: "s1"}}, listener.deleted)
+ requireNoBackgroundLeft(t, svc, "s1")
+}
+
+func TestBatchDeleteSessionsNotifiesBackgroundListeners(t *testing.T) {
+ svc, listener := newDeleteBackgroundFixture(t, "s1", "s2")
+
+ require.NoError(t, svc.BatchDeleteSessions(testSessionScopeContext(1, "u1"), []string{"s1", "s2"}))
+
+ require.ElementsMatch(t, []interfaces.SessionRef{
+ {TenantID: 1, SessionID: "s1"}, {TenantID: 1, SessionID: "s2"},
+ }, listener.deleted)
+ requireNoBackgroundLeft(t, svc, "s2")
+}
+
+func TestDeleteAllSessionsNotifiesBackgroundListeners(t *testing.T) {
+ svc, listener := newDeleteBackgroundFixture(t, "s1", "s2")
+
+ require.NoError(t, svc.DeleteAllSessions(testSessionScopeContext(1, "u1")))
+
+ require.Len(t, listener.deleted, 2)
+ requireNoBackgroundLeft(t, svc, "s1")
+}
diff --git a/internal/application/service/session_fork.go b/internal/application/service/session_fork.go
index 678287a41..a2c385213 100644
--- a/internal/application/service/session_fork.go
+++ b/internal/application/service/session_fork.go
@@ -169,6 +169,7 @@ type SessionForkService struct {
messages forkMessageStore
sandbox SessionForkSandboxPort
allocations forkAllocations
+ activity interfaces.ActivityRegistry
}
// NewSessionForkService wires the service. A nil sandbox port makes every fork
@@ -187,8 +188,11 @@ func NewSessionForkServiceFromRepos(
sessions interfaces.SessionRepository,
messages interfaces.MessageRepository,
sandboxPort SessionForkSandboxPort,
+ activity interfaces.ActivityRegistry,
) *SessionForkService {
- return NewSessionForkService(sessions, messages, sandboxPort)
+ s := NewSessionForkService(sessions, messages, sandboxPort)
+ s.activity = activity
+ return s
}
// WithAllocations enables user-mode forks.
@@ -259,6 +263,9 @@ func (s *SessionForkService) Fork(
return nil, ErrForkSourceBusy
}
}
+ if err := s.rejectIfBackgroundActive(ctx, source); err != nil {
+ return nil, err
+ }
workCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), forkSnapshotTimeout)
defer cancel()
@@ -469,6 +476,22 @@ func forkLeaseLost(ctx context.Context, err error) error {
return nil
}
+// rejectIfBackgroundActive refuses while background work is live in the
+// session. Snapshotting pauses the source sandbox and would freeze the work.
+func (s *SessionForkService) rejectIfBackgroundActive(ctx context.Context, source *types.Session) error {
+ if s.activity == nil {
+ return nil
+ }
+ live, err := s.activity.List(ctx, interfaces.SessionRef{TenantID: source.TenantID, SessionID: source.ID})
+ if err != nil {
+ return fmt.Errorf("session fork: check background work: %w", err)
+ }
+ if len(live) > 0 {
+ return ErrForkSourceBusy
+ }
+ return nil
+}
+
// prepareBootstrap decides whether the fork can carry the source sandbox's
// workspace state. When every condition holds, it takes a snapshot.
//
@@ -632,27 +655,33 @@ func inheritUnconsumedForkSnapshot(source *types.Session, checkpoint *types.Sand
}
// latestCheckpoint returns the checkpoint of the last assistant message in
-// history, or nil when that message has none.
+// history, or nil when that message has none or took it while background work
+// was live.
func latestCheckpoint(history []*types.Message) *types.SandboxCheckpoint {
for i := len(history) - 1; i >= 0; i-- {
if history[i].Role != "assistant" {
continue
}
- return history[i].SandboxCheckpoint
+ cp := history[i].SandboxCheckpoint
+ if cp != nil && cp.BackgroundActive {
+ return nil
+ }
+ return cp
}
return nil
}
// latestReachableCheckpoint walks kept assistant messages until it finds one
-// with a non-empty commit SHA. Rewind uses this so a failed last-turn
-// checkpoint does not skip an earlier SHA that can still reset the workspace.
+// with a non-empty commit SHA that was not taken during live background work.
+// Rewind uses this so a failed or flagged last-turn checkpoint does not skip an
+// earlier SHA that can still reset the workspace.
func latestReachableCheckpoint(history []*types.Message) *types.SandboxCheckpoint {
for i := len(history) - 1; i >= 0; i-- {
if history[i] == nil || history[i].Role != "assistant" {
continue
}
cp := history[i].SandboxCheckpoint
- if cp == nil || strings.TrimSpace(cp.CommitSHA) == "" {
+ if cp == nil || strings.TrimSpace(cp.CommitSHA) == "" || cp.BackgroundActive {
continue
}
return cp
diff --git a/internal/application/service/session_fork_test.go b/internal/application/service/session_fork_test.go
index d23cad5c7..698415a80 100644
--- a/internal/application/service/session_fork_test.go
+++ b/internal/application/service/session_fork_test.go
@@ -12,6 +12,7 @@ import (
"github.com/Tencent/WeKnora/internal/sandbox/admission"
"github.com/Tencent/WeKnora/internal/sandbox/allocation"
"github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/stretchr/testify/require"
)
@@ -1510,3 +1511,38 @@ func TestForkOfSessionModeSourceStillSnapshots(t *testing.T) {
require.NotNil(t, sessions.created.ForkBootstrap)
require.Equal(t, "snap-1", sessions.created.ForkBootstrap.SnapshotID)
}
+
+func TestForkLiveBackgroundWorkIsBusy(t *testing.T) {
+ turn := checkpointedTurn("u-msg-1", "a-msg-1", "sbx-1", "sha1", 0)
+ forkPoint := &types.Message{
+ ID: "u-msg-2", SessionID: "src", Role: "user", CreatedAt: forkBase.Add(10 * time.Second),
+ }
+ port := &fakeForkSandboxPort{boundID: "sbx-1", bound: true, snapshotID: "snap-1"}
+ svc, sessions, _ := newForkFixture(t, port, append(turn, forkPoint))
+ svc.activity = liveActivityRegistry(t, interfaces.SessionRef{TenantID: 1, SessionID: "src"})
+
+ got, err := svc.Fork(context.Background(), 1, "u1", "src", "u-msg-2", "")
+
+ require.ErrorIs(t, err, ErrForkSourceBusy)
+ require.Nil(t, got)
+ require.Zero(t, port.snapshotCalls)
+ require.Nil(t, sessions.created)
+}
+
+func TestForkDegradesWhenCheckpointWasTakenWhileBackgroundWorkRan(t *testing.T) {
+ turn := checkpointedTurn("u-msg-1", "a-msg-1", "sbx-1", "sha1", 0)
+ turn[1].SandboxCheckpoint.BackgroundActive = true
+ forkPoint := &types.Message{
+ ID: "u-msg-2", SessionID: "src", Role: "user", CreatedAt: forkBase.Add(10 * time.Second),
+ }
+ port := &fakeForkSandboxPort{boundID: "sbx-1", bound: true, snapshotID: "snap-1"}
+ svc, sessions, _ := newForkFixture(t, port, append(turn, forkPoint))
+
+ got, err := svc.Fork(context.Background(), 1, "u1", "src", "u-msg-2", "")
+
+ require.NoError(t, err)
+ require.True(t, got.Degraded)
+ require.Equal(t, ForkDegradeNoCheckpoint, got.Reason)
+ require.Zero(t, port.snapshotCalls)
+ require.Nil(t, sessions.created.ForkBootstrap)
+}
diff --git a/internal/application/service/session_rewind.go b/internal/application/service/session_rewind.go
index a9db1d3c4..76cdcb50f 100644
--- a/internal/application/service/session_rewind.go
+++ b/internal/application/service/session_rewind.go
@@ -175,6 +175,7 @@ type SessionRewindService struct {
busyGate *SessionBusyGate
// suspendedRuns closes runs suspended on prompts before the cut.
suspendedRuns *SuspendedRunService
+ activity interfaces.ActivityRegistry
}
// SetSuspendedRuns lets rewind close the session's suspended runs.
@@ -211,6 +212,7 @@ func NewSessionRewindServiceFromRepos(
fallback sandbox.Manager,
streams interfaces.StreamManager,
busyGate *SessionBusyGate,
+ activity interfaces.ActivityRegistry,
) *SessionRewindService {
s := NewSessionRewindService(sessions, messages, sandboxPort, knowledge, suggestions)
s.snapshots = NewResolverForkSnapshotDeleter(resolver, fallback)
@@ -218,6 +220,7 @@ func NewSessionRewindServiceFromRepos(
if busyGate != nil {
s.busyGate = busyGate
}
+ s.activity = activity
return s
}
@@ -279,6 +282,9 @@ func (s *SessionRewindService) Rewind(
if err := s.rejectIfBusy(ctx, sessionID); err != nil {
return nil, err
}
+ if err := s.rejectIfBackgroundActive(ctx, source); err != nil {
+ return nil, err
+ }
history, err := s.keptCheckpointHistory(ctx, sessionID, rewindPoint)
if err != nil {
@@ -294,6 +300,11 @@ func (s *SessionRewindService) Rewind(
return nil, err
}
defer unlock()
+ // Re-check under the lock. The busy gate now holds off new turns, so no
+ // new background work can start before the truncate lands.
+ if err := s.rejectIfBackgroundActive(persistCtx, source); err != nil {
+ return nil, err
+ }
var abandonedBootstrap *types.ForkBootstrap
workspaceReset, reason, resetErr := s.resetWorkspaceIfPossible(workCtx, sessionID, history)
@@ -476,6 +487,7 @@ func (s *SessionRewindService) lockRewind(ctx context.Context, sessionID string)
if errors.Is(err, sandbox.ErrSessionRewindLocked) ||
errors.Is(err, sandbox.ErrSessionTurnActive) ||
errors.Is(err, sandbox.ErrSessionWorkspaceBusy) ||
+ errors.Is(err, sandbox.ErrSessionWorkActive) ||
errors.Is(err, ErrRewindSourceBusy) {
return nil, nil, ErrRewindSourceBusy
}
@@ -494,6 +506,24 @@ func (s *SessionRewindService) lockRewind(ctx context.Context, sessionID string)
}, nil
}
+// rejectIfBackgroundActive refuses while background work is live in the
+// session. On Web the rewind lock refuses as well (tryLockRewindScript); this
+// check is what covers Lite, which has no turn leases, and where rewind would
+// truncate the conversation a pending completion notice still refers to.
+func (s *SessionRewindService) rejectIfBackgroundActive(ctx context.Context, source *types.Session) error {
+ if s.activity == nil {
+ return nil
+ }
+ live, err := s.activity.List(ctx, interfaces.SessionRef{TenantID: source.TenantID, SessionID: source.ID})
+ if err != nil {
+ return fmt.Errorf("session rewind: check background work: %w", err)
+ }
+ if len(live) > 0 {
+ return ErrRewindSourceBusy
+ }
+ return nil
+}
+
func (s *SessionRewindService) rejectIfBusy(ctx context.Context, sessionID string) error {
if s.liveRuns != nil {
liveID, _, err := s.liveRuns.GetLiveRun(ctx, sessionID)
diff --git a/internal/application/service/session_rewind_test.go b/internal/application/service/session_rewind_test.go
index 0cae00f38..abd7d0f0c 100644
--- a/internal/application/service/session_rewind_test.go
+++ b/internal/application/service/session_rewind_test.go
@@ -12,6 +12,7 @@ import (
"github.com/Tencent/WeKnora/internal/sandbox"
"github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/stretchr/testify/require"
)
@@ -949,3 +950,82 @@ func TestRewindKeepsForkSnapshotWhenDeleteFails(t *testing.T) {
require.True(t, sessions.bootstrapCleared,
"the column write stays ahead of the delete so the fork cannot boot ahead of the transcript")
}
+
+func rewindBusyFixture(t *testing.T) (*SessionRewindService, *fakeMessageStore, *fakeRewindSandboxPort) {
+ t.Helper()
+ turn := rewindCompletedTurn("u-1", "a-1", "sbx-1", rewindSHA1, 0)
+ laterUser := &types.Message{
+ ID: "u-2", SessionID: "src", Role: "user", CreatedAt: forkBase.Add(10 * time.Second),
+ }
+ port := newFakeRewindPort()
+ svc, msgs := newRewindFixture(t, port, append(turn, laterUser))
+ return svc, msgs, port
+}
+
+func TestRewindLiveBackgroundWorkIsBusy(t *testing.T) {
+ svc, msgs, port := rewindBusyFixture(t)
+ svc.activity = liveActivityRegistry(t, interfaces.SessionRef{TenantID: 1, SessionID: "src"})
+
+ got, err := svc.Rewind(context.Background(), 1, "u1", "src", "u-2")
+
+ require.ErrorIs(t, err, ErrRewindSourceBusy)
+ require.Nil(t, got)
+ require.Equal(t, 0, msgs.deleteFromCalls)
+ require.Empty(t, port.runner.calls)
+}
+
+func TestRewindBackgroundWorkElsewhereIsNotBusy(t *testing.T) {
+ svc, _, _ := rewindBusyFixture(t)
+ svc.activity = liveActivityRegistry(t,
+ interfaces.SessionRef{TenantID: 1, SessionID: "other"},
+ interfaces.SessionRef{TenantID: 2, SessionID: "src"},
+ )
+
+ got, err := svc.Rewind(context.Background(), 1, "u1", "src", "u-2")
+
+ require.NoError(t, err)
+ require.Equal(t, 1, got.DeletedMessages)
+}
+
+func TestRewindBackgroundLookupFailureIsAnError(t *testing.T) {
+ svc, msgs, _ := rewindBusyFixture(t)
+ svc.activity = failingActivityRegistry{err: errors.New("redis down")}
+
+ _, err := svc.Rewind(context.Background(), 1, "u1", "src", "u-2")
+
+ require.Error(t, err)
+ require.NotErrorIs(t, err, ErrRewindSourceBusy)
+ require.Equal(t, 0, msgs.deleteFromCalls)
+}
+
+func TestRewindLockRefusedByBackgroundWorkIsBusy(t *testing.T) {
+ svc, msgs, port := rewindBusyFixture(t)
+ port.lockErr = sandbox.ErrSessionWorkActive
+
+ got, err := svc.Rewind(context.Background(), 1, "u1", "src", "u-2")
+
+ require.ErrorIs(t, err, ErrRewindSourceBusy)
+ require.Nil(t, got)
+ require.Equal(t, 0, msgs.deleteFromCalls)
+}
+
+func TestRewindSkipsCheckpointTakenWhileBackgroundWorkRan(t *testing.T) {
+ turn1 := rewindCompletedTurn("u-1", "a-1", "sbx-1", rewindSHA1, 0)
+ turn2 := rewindCompletedTurn("u-2", "a-2", "sbx-1", rewindSHA2, 10*time.Second)
+ turn2[1].SandboxCheckpoint.BackgroundActive = true
+ laterUser := &types.Message{
+ ID: "u-3", SessionID: "src", Role: "user", CreatedAt: forkBase.Add(20 * time.Second),
+ }
+ port := newFakeRewindPort()
+ messages := append(append(turn1, turn2...), laterUser)
+ svc, msgs := newRewindFixture(t, port, messages)
+
+ got, err := svc.Rewind(context.Background(), 1, "u1", "src", "u-3")
+
+ require.NoError(t, err)
+ require.True(t, got.WorkspaceReset)
+ require.Len(t, port.runner.calls, 1)
+ require.Contains(t, port.runner.calls[0], rewindSHA1)
+ require.NotContains(t, port.runner.calls[0], rewindSHA2)
+ require.Equal(t, []string{"u-1", "a-1", "u-2", "a-2"}, messageIDs(msgs.messages))
+}
diff --git a/internal/application/service/workspace_checkpointer.go b/internal/application/service/workspace_checkpointer.go
index 6f0db9e48..7e77285b4 100644
--- a/internal/application/service/workspace_checkpointer.go
+++ b/internal/application/service/workspace_checkpointer.go
@@ -26,6 +26,7 @@ import (
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/sandbox"
"github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
)
// SandboxShellRunner is the narrow subset of sandbox behaviour the
@@ -52,7 +53,8 @@ var gitSHAPattern = regexp.MustCompile(`^[0-9a-f]{40}$`)
// WorkspaceCheckpointer commits /workspace after each agent turn.
type WorkspaceCheckpointer struct {
- runner SandboxShellRunner
+ runner SandboxShellRunner
+ activity interfaces.ActivityRegistry
}
// NewWorkspaceCheckpointer wires a checkpointer. A nil runner yields a
@@ -62,6 +64,34 @@ func NewWorkspaceCheckpointer(runner SandboxShellRunner) *WorkspaceCheckpointer
return &WorkspaceCheckpointer{runner: runner}
}
+// WithActivityRegistry makes Checkpoint flag commits taken while background
+// work is live. Without a registry every checkpoint is unflagged.
+func (c *WorkspaceCheckpointer) WithActivityRegistry(activity interfaces.ActivityRegistry) *WorkspaceCheckpointer {
+ if c != nil {
+ c.activity = activity
+ }
+ return c
+}
+
+// backgroundActive reports whether background work is live in the session. A
+// failed lookup counts as live: an unusable fork point is recoverable, a rewind
+// onto a half-written tree is not.
+func (c *WorkspaceCheckpointer) backgroundActive(ctx context.Context, sessionID string) bool {
+ if c.activity == nil {
+ return false
+ }
+ tenantID, ok := types.SandboxTenantIDFromContext(ctx)
+ if !ok || tenantID == 0 {
+ return false
+ }
+ live, err := c.activity.List(ctx, interfaces.SessionRef{TenantID: tenantID, SessionID: sessionID})
+ if err != nil {
+ logger.Warnf(ctx, "[WorkspaceCheckpointer] background lookup failed session=%s: %v", sessionID, err)
+ return true
+ }
+ return len(live) > 0
+}
+
// checkpointScript renders the idempotent init-and-commit script.
//
// Ordering matters in two places:
@@ -118,6 +148,9 @@ func (c *WorkspaceCheckpointer) Checkpoint(
return nil
}
command := checkpointScriptFor(layout, messageID)
+ // Read before committing: no tool runs while the turn is finishing, so no
+ // background work can start between this read and the commit.
+ backgroundActive := c.backgroundActive(ctx, sessionID)
var result *sandbox.ExecuteResult
if runner, ok := c.runner.(sandbox.SessionInstallShellExecutor); ok {
// Git operates on the existing workspace. Do not prepare artifact/input
@@ -156,9 +189,10 @@ func (c *WorkspaceCheckpointer) Checkpoint(
}
return &types.SandboxCheckpoint{
- SandboxID: sandboxID,
- CommitSHA: sha,
- CommittedAt: time.Now().UTC(),
+ SandboxID: sandboxID,
+ CommitSHA: sha,
+ CommittedAt: time.Now().UTC(),
+ BackgroundActive: backgroundActive,
}
}
diff --git a/internal/application/service/workspace_checkpointer_test.go b/internal/application/service/workspace_checkpointer_test.go
index 068ffaaa4..d5a14d372 100644
--- a/internal/application/service/workspace_checkpointer_test.go
+++ b/internal/application/service/workspace_checkpointer_test.go
@@ -8,6 +8,8 @@ import (
"time"
"github.com/Tencent/WeKnora/internal/sandbox"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/stretchr/testify/require"
)
@@ -132,3 +134,48 @@ func TestCheckpointSkippedWhenBackendDoesNotVersionWorkspace(t *testing.T) {
require.Nil(t, cp.Checkpoint(context.Background(), "s1", "sbx-1", "msg-1"))
require.Empty(t, runner.calls, "must not git commit the user's real project")
}
+
+func shaRunner() *fakeShellRunner {
+ return &fakeShellRunner{result: &sandbox.ExecuteResult{ExitCode: 0, Stdout: strings.Repeat("a", 40) + "\n"}}
+}
+
+func TestCheckpointFlagsCommitTakenWhileBackgroundWorkIsLive(t *testing.T) {
+ reg := liveActivityRegistry(t, interfaces.SessionRef{TenantID: 7, SessionID: "s1"})
+ cp := NewWorkspaceCheckpointer(shaRunner()).WithActivityRegistry(reg)
+
+ got := cp.Checkpoint(types.WithSandboxTenantID(context.Background(), 7), "s1", "sbx-1", "msg-1")
+
+ require.NotNil(t, got)
+ require.True(t, got.BackgroundActive)
+}
+
+func TestCheckpointLeavesFlagOffWithoutLiveBackgroundWork(t *testing.T) {
+ reg := liveActivityRegistry(t,
+ interfaces.SessionRef{TenantID: 7, SessionID: "s2"},
+ interfaces.SessionRef{TenantID: 8, SessionID: "s1"},
+ )
+ cp := NewWorkspaceCheckpointer(shaRunner()).WithActivityRegistry(reg)
+
+ got := cp.Checkpoint(types.WithSandboxTenantID(context.Background(), 7), "s1", "sbx-1", "msg-1")
+
+ require.NotNil(t, got)
+ require.False(t, got.BackgroundActive)
+}
+
+func TestCheckpointFlagsCommitWhenBackgroundLookupFails(t *testing.T) {
+ cp := NewWorkspaceCheckpointer(shaRunner()).
+ WithActivityRegistry(failingActivityRegistry{err: errors.New("redis down")})
+
+ got := cp.Checkpoint(types.WithSandboxTenantID(context.Background(), 7), "s1", "sbx-1", "msg-1")
+
+ require.NotNil(t, got)
+ require.True(t, got.BackgroundActive)
+}
+
+func TestCheckpointWithoutRegistryIsUnflagged(t *testing.T) {
+ got := NewWorkspaceCheckpointer(shaRunner()).
+ Checkpoint(types.WithSandboxTenantID(context.Background(), 7), "s1", "sbx-1", "msg-1")
+
+ require.NotNil(t, got)
+ require.False(t, got.BackgroundActive)
+}
diff --git a/internal/background/backgroundtest/contract.go b/internal/background/backgroundtest/contract.go
new file mode 100644
index 000000000..76d8ff17b
--- /dev/null
+++ b/internal/background/backgroundtest/contract.go
@@ -0,0 +1,287 @@
+// Package backgroundtest holds contract tests shared by every implementation of
+// the background-work interfaces.
+package backgroundtest
+
+import (
+ "context"
+ "fmt"
+ "sort"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+// Advance moves the implementation's notion of "now" forward by d.
+type Advance func(d time.Duration)
+
+var (
+ sessA = interfaces.SessionRef{TenantID: 1, SessionID: "sess-a"}
+ sessB = interfaces.SessionRef{TenantID: 1, SessionID: "sess-b"}
+)
+
+func activity(id, kind string) interfaces.BackgroundActivity {
+ return interfaces.BackgroundActivity{
+ ID: id, Kind: kind, Title: "title " + id,
+ StartedAt: time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC),
+ }
+}
+
+func listIDs(t *testing.T, r interfaces.ActivityRegistry, s interfaces.SessionRef) []string {
+ t.Helper()
+ items, err := r.List(context.Background(), s)
+ require.NoError(t, err)
+ ids := make([]string, 0, len(items))
+ for _, it := range items {
+ ids = append(ids, it.ID)
+ }
+ sort.Strings(ids)
+ return ids
+}
+
+// RunActivityRegistryContract runs the shared ActivityRegistry cases. newRegistry
+// must return an empty registry and a function that advances its clock.
+func RunActivityRegistryContract(
+ t *testing.T,
+ newRegistry func(t *testing.T) (interfaces.ActivityRegistry, Advance),
+) {
+ ctx := context.Background()
+
+ t.Run("AddThenList", func(t *testing.T) {
+ r, _ := newRegistry(t)
+ require.NoError(t, r.Add(ctx, sessA, activity("sj_1", "shell_job"), time.Minute, 0))
+ items, err := r.List(ctx, sessA)
+ require.NoError(t, err)
+ require.Len(t, items, 1)
+ require.Equal(t, "sj_1", items[0].ID)
+ require.Equal(t, "shell_job", items[0].Kind)
+ require.Equal(t, "title sj_1", items[0].Title)
+ require.True(t, items[0].StartedAt.Equal(activity("sj_1", "").StartedAt))
+ })
+
+ t.Run("SessionsAreIsolated", func(t *testing.T) {
+ r, _ := newRegistry(t)
+ require.NoError(t, r.Add(ctx, sessA, activity("sj_1", "shell_job"), time.Minute, 0))
+ require.Empty(t, listIDs(t, r, sessB))
+ })
+
+ t.Run("CapIsPerKind", func(t *testing.T) {
+ r, _ := newRegistry(t)
+ require.NoError(t, r.Add(ctx, sessA, activity("a1", "k1"), time.Minute, 2))
+ require.NoError(t, r.Add(ctx, sessA, activity("a2", "k1"), time.Minute, 2))
+ err := r.Add(ctx, sessA, activity("a3", "k1"), time.Minute, 2)
+ require.ErrorIs(t, err, interfaces.ErrActivityLimit)
+ require.NoError(t, r.Add(ctx, sessA, activity("b1", "k2"), time.Minute, 2))
+ require.Equal(t, []string{"a1", "a2", "b1"}, listIDs(t, r, sessA))
+ })
+
+ t.Run("NonPositiveCapMeansUnlimited", func(t *testing.T) {
+ r, _ := newRegistry(t)
+ for i := 0; i < 10; i++ {
+ require.NoError(t, r.Add(ctx, sessA, activity(fmt.Sprintf("a%d", i), "k"), time.Minute, 0))
+ }
+ require.NoError(t, r.Add(ctx, sessA, activity("neg", "k"), time.Minute, -1))
+ require.Len(t, listIDs(t, r, sessA), 11)
+ })
+
+ t.Run("ReAddExtendsWithoutCountingAgainstCap", func(t *testing.T) {
+ r, advance := newRegistry(t)
+ require.NoError(t, r.Add(ctx, sessA, activity("a1", "k"), 10*time.Second, 1))
+ advance(8 * time.Second)
+ require.NoError(t, r.Add(ctx, sessA, activity("a1", "k"), 10*time.Second, 1))
+ advance(8 * time.Second) // 16s after the first Add, 8s after the second
+ require.Equal(t, []string{"a1"}, listIDs(t, r, sessA))
+ })
+
+ t.Run("ConcurrentAddRespectsCap", func(t *testing.T) {
+ r, _ := newRegistry(t)
+ const workers = 20
+ var wg sync.WaitGroup
+ errs := make([]error, workers)
+ for i := 0; i < workers; i++ {
+ wg.Add(1)
+ go func(i int) {
+ defer wg.Done()
+ errs[i] = r.Add(ctx, sessA, activity(fmt.Sprintf("c%d", i), "k"), time.Minute, 3)
+ }(i)
+ }
+ wg.Wait()
+ ok := 0
+ for _, err := range errs {
+ if err == nil {
+ ok++
+ continue
+ }
+ require.ErrorIs(t, err, interfaces.ErrActivityLimit)
+ }
+ require.Equal(t, 3, ok)
+ require.Len(t, listIDs(t, r, sessA), 3)
+ })
+
+ t.Run("ExpiredItemsDisappearAndFreeTheCap", func(t *testing.T) {
+ r, advance := newRegistry(t)
+ require.NoError(t, r.Add(ctx, sessA, activity("a1", "k"), 10*time.Second, 1))
+ advance(11 * time.Second)
+ require.Empty(t, listIDs(t, r, sessA))
+ require.NoError(t, r.Add(ctx, sessA, activity("a2", "k"), 10*time.Second, 1))
+ require.Equal(t, []string{"a2"}, listIDs(t, r, sessA))
+ })
+
+ t.Run("RenewExtends", func(t *testing.T) {
+ r, advance := newRegistry(t)
+ require.NoError(t, r.Add(ctx, sessA, activity("a1", "k"), 10*time.Second, 0))
+ advance(8 * time.Second)
+ found, err := r.Renew(ctx, sessA, "a1", 10*time.Second)
+ require.NoError(t, err)
+ require.True(t, found)
+ advance(8 * time.Second)
+ require.Equal(t, []string{"a1"}, listIDs(t, r, sessA))
+ })
+
+ t.Run("RenewDoesNotRecreateRemoved", func(t *testing.T) {
+ r, _ := newRegistry(t)
+ require.NoError(t, r.Add(ctx, sessA, activity("a1", "k"), time.Minute, 0))
+ require.NoError(t, r.Remove(ctx, sessA, "a1"))
+ found, err := r.Renew(ctx, sessA, "a1", time.Minute)
+ require.NoError(t, err)
+ require.False(t, found)
+ require.Empty(t, listIDs(t, r, sessA))
+ })
+
+ t.Run("RenewDoesNotRecreateExpired", func(t *testing.T) {
+ r, advance := newRegistry(t)
+ require.NoError(t, r.Add(ctx, sessA, activity("a1", "k"), 10*time.Second, 0))
+ advance(11 * time.Second)
+ found, err := r.Renew(ctx, sessA, "a1", time.Minute)
+ require.NoError(t, err)
+ require.False(t, found)
+ require.Empty(t, listIDs(t, r, sessA))
+ })
+
+ t.Run("RenewUnknownIsNotFound", func(t *testing.T) {
+ r, _ := newRegistry(t)
+ found, err := r.Renew(ctx, sessA, "missing", time.Minute)
+ require.NoError(t, err)
+ require.False(t, found)
+ })
+
+ t.Run("RemoveMissingIsNotAnError", func(t *testing.T) {
+ r, _ := newRegistry(t)
+ require.NoError(t, r.Remove(ctx, sessA, "missing"))
+ })
+}
+
+func notice(id string, finishedAt time.Time) interfaces.BackgroundNotice {
+ return interfaces.BackgroundNotice{ID: id, Kind: "shell_job", Text: "done " + id, FinishedAt: finishedAt}
+}
+
+func pendingIDs(t *testing.T, in interfaces.NoticeInbox, s interfaces.SessionRef) []string {
+ t.Helper()
+ items, err := in.Pending(context.Background(), s)
+ require.NoError(t, err)
+ ids := make([]string, 0, len(items))
+ for _, it := range items {
+ ids = append(ids, it.ID)
+ }
+ return ids
+}
+
+// RunNoticeInboxContract runs the shared NoticeInbox cases. newInbox must return
+// an empty inbox whose clock starts at 2026-10-14T10:00:00Z, and a function that
+// advances that clock.
+func RunNoticeInboxContract(
+ t *testing.T,
+ newInbox func(t *testing.T) (interfaces.NoticeInbox, Advance),
+) {
+ ctx := context.Background()
+ start := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC)
+
+ t.Run("FirstPushWins", func(t *testing.T) {
+ in, _ := newInbox(t)
+ first, err := in.Push(ctx, sessA, notice("n1", start))
+ require.NoError(t, err)
+ require.True(t, first)
+ second := notice("n1", start)
+ second.Text = "overwritten"
+ first, err = in.Push(ctx, sessA, second)
+ require.NoError(t, err)
+ require.False(t, first)
+ items, err := in.Pending(ctx, sessA)
+ require.NoError(t, err)
+ require.Len(t, items, 1)
+ require.Equal(t, "done n1", items[0].Text)
+ })
+
+ t.Run("PendingIsOldestFirst", func(t *testing.T) {
+ in, _ := newInbox(t)
+ _, err := in.Push(ctx, sessA, notice("late", start.Add(2*time.Minute)))
+ require.NoError(t, err)
+ _, err = in.Push(ctx, sessA, notice("early", start))
+ require.NoError(t, err)
+ _, err = in.Push(ctx, sessA, notice("mid", start.Add(time.Minute)))
+ require.NoError(t, err)
+ require.Equal(t, []string{"early", "mid", "late"}, pendingIDs(t, in, sessA))
+ })
+
+ t.Run("AckRemoves", func(t *testing.T) {
+ in, _ := newInbox(t)
+ _, err := in.Push(ctx, sessA, notice("n1", start))
+ require.NoError(t, err)
+ _, err = in.Push(ctx, sessA, notice("n2", start))
+ require.NoError(t, err)
+ require.NoError(t, in.Ack(ctx, sessA, []string{"n1", "unknown"}))
+ require.Equal(t, []string{"n2"}, pendingIDs(t, in, sessA))
+ })
+
+ t.Run("AckEmptyIsNoop", func(t *testing.T) {
+ in, _ := newInbox(t)
+ require.NoError(t, in.Ack(ctx, sessA, nil))
+ })
+
+ t.Run("AckedIDCanBePushedAgain", func(t *testing.T) {
+ in, _ := newInbox(t)
+ _, err := in.Push(ctx, sessA, notice("n1", start))
+ require.NoError(t, err)
+ require.NoError(t, in.Ack(ctx, sessA, []string{"n1"}))
+ first, err := in.Push(ctx, sessA, notice("n1", start))
+ require.NoError(t, err)
+ require.True(t, first)
+ })
+
+ t.Run("RetentionHidesOldNotices", func(t *testing.T) {
+ in, advance := newInbox(t)
+ _, err := in.Push(ctx, sessA, notice("n1", start))
+ require.NoError(t, err)
+ advance(interfaces.NoticeRetention - time.Minute)
+ require.Equal(t, []string{"n1"}, pendingIDs(t, in, sessA))
+ advance(2 * time.Minute)
+ require.Empty(t, pendingIDs(t, in, sessA))
+ })
+
+ t.Run("ClearRemovesSessionOnly", func(t *testing.T) {
+ in, _ := newInbox(t)
+ _, err := in.Push(ctx, sessA, notice("n1", start))
+ require.NoError(t, err)
+ _, err = in.Push(ctx, sessB, notice("n2", start))
+ require.NoError(t, err)
+ require.NoError(t, in.Clear(ctx, sessA))
+ require.Empty(t, pendingIDs(t, in, sessA))
+ require.Equal(t, []string{"n2"}, pendingIDs(t, in, sessB))
+ })
+
+ t.Run("PreservesFields", func(t *testing.T) {
+ in, _ := newInbox(t)
+ n := notice("n1", start)
+ _, err := in.Push(ctx, sessA, n)
+ require.NoError(t, err)
+ items, err := in.Pending(ctx, sessA)
+ require.NoError(t, err)
+ require.Len(t, items, 1)
+ require.Equal(t, n.ID, items[0].ID)
+ require.Equal(t, n.Kind, items[0].Kind)
+ require.Equal(t, n.Text, items[0].Text)
+ require.True(t, n.FinishedAt.Equal(items[0].FinishedAt))
+ })
+}
diff --git a/internal/background/lifecycle.go b/internal/background/lifecycle.go
new file mode 100644
index 000000000..aef115004
--- /dev/null
+++ b/internal/background/lifecycle.go
@@ -0,0 +1,98 @@
+package background
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+// Lifecycle combines the shared parts in the only orders that are safe.
+// Features call these four methods and never the underlying interfaces.
+type Lifecycle struct {
+ Registry interfaces.ActivityRegistry
+ Inbox interfaces.NoticeInbox
+ KeepAlive interfaces.SessionKeepAliver // optional
+ Events interfaces.WorkEventEmitter // optional
+ Logf func(ctx context.Context, format string, args ...any)
+}
+
+func (l *Lifecycle) logf(ctx context.Context, format string, args ...any) {
+ if l.Logf != nil {
+ l.Logf(ctx, format, args...)
+ }
+}
+
+func (l *Lifecycle) keepAlive(ctx context.Context, s interfaces.SessionRef) {
+ if l.KeepAlive == nil {
+ return
+ }
+ if err := l.KeepAlive.KeepAlive(ctx, s); err != nil {
+ l.logf(ctx, "[Background] keep-alive failed session=%s: %v", s.SessionID, err)
+ }
+}
+
+// Begin registers a, then runs start. Registering first means lifecycle
+// protection is in force the moment the work exists. If start fails the
+// registration is removed. On success the sandbox is kept alive once at once,
+// because nothing renewed it before the work began.
+func (l *Lifecycle) Begin(
+ ctx context.Context, s interfaces.SessionRef, a interfaces.BackgroundActivity,
+ lease time.Duration, kindCap int, start func(context.Context) error,
+) error {
+ if err := l.Registry.Add(ctx, s, a, lease, kindCap); err != nil {
+ return err
+ }
+ if err := start(ctx); err != nil {
+ if rmErr := l.Registry.Remove(context.WithoutCancel(ctx), s, a.ID); rmErr != nil {
+ l.logf(ctx, "[Background] remove after failed start session=%s id=%s: %v", s.SessionID, a.ID, rmErr)
+ }
+ return err
+ }
+ l.keepAlive(ctx, s)
+ return nil
+}
+
+// Alive renews the registration and, when it still exists, keeps the sandbox
+// alive. found=false means the item was removed or expired; the caller decides
+// whether to re-register.
+func (l *Lifecycle) Alive(
+ ctx context.Context, s interfaces.SessionRef, id string, lease time.Duration,
+) (bool, error) {
+ found, err := l.Registry.Renew(ctx, s, id, lease)
+ if err != nil {
+ return false, err
+ }
+ if found {
+ l.keepAlive(ctx, s)
+ }
+ return found, nil
+}
+
+// Finish records the completion notice, emits work_finished only if this call
+// wrote the notice first, then removes the registration. Push comes first: in
+// the other order there is a window where neither the registry nor the inbox
+// knows the item. If Push fails the registration is kept so the item stays
+// queryable and a later Finish can retry.
+func (l *Lifecycle) Finish(
+ ctx context.Context, s interfaces.SessionRef, n interfaces.BackgroundNotice,
+ r interfaces.WorkResult, parentMessageID string,
+) error {
+ first, err := l.Inbox.Push(ctx, s, n)
+ if err != nil {
+ return fmt.Errorf("push background notice: %w", err)
+ }
+ if first && l.Events != nil {
+ if err := l.Events.EmitWorkFinished(ctx, s, parentMessageID, r); err != nil {
+ l.logf(ctx, "[Background] emit work_finished failed session=%s id=%s: %v", s.SessionID, n.ID, err)
+ }
+ }
+ return l.Registry.Remove(ctx, s, n.ID)
+}
+
+// Discard removes the registration without a notice or event. Use it only when
+// the result was already handed to the model directly.
+func (l *Lifecycle) Discard(ctx context.Context, s interfaces.SessionRef, id string) error {
+ return l.Registry.Remove(ctx, s, id)
+}
diff --git a/internal/background/lifecycle_test.go b/internal/background/lifecycle_test.go
new file mode 100644
index 000000000..d2ac5e603
--- /dev/null
+++ b/internal/background/lifecycle_test.go
@@ -0,0 +1,182 @@
+package background
+
+import (
+ "context"
+ "errors"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+type countingKeepAliver struct{ calls int }
+
+func (k *countingKeepAliver) KeepAlive(context.Context, interfaces.SessionRef) error {
+ k.calls++
+ return nil
+}
+
+type recordingEmitter struct{ finished []interfaces.WorkResult }
+
+func (e *recordingEmitter) EmitWorkFinished(
+ _ context.Context, _ interfaces.SessionRef, _ string, r interfaces.WorkResult,
+) error {
+ e.finished = append(e.finished, r)
+ return nil
+}
+
+type lifecycleFixture struct {
+ l *Lifecycle
+ registry *MemoryActivityRegistry
+ inbox *MemoryNoticeInbox
+ keep *countingKeepAliver
+ events *recordingEmitter
+}
+
+func newLifecycleFixture() lifecycleFixture {
+ f := lifecycleFixture{
+ registry: NewMemoryActivityRegistry(nil),
+ inbox: NewMemoryNoticeInbox(nil),
+ keep: &countingKeepAliver{},
+ events: &recordingEmitter{},
+ }
+ f.l = &Lifecycle{Registry: f.registry, Inbox: f.inbox, KeepAlive: f.keep, Events: f.events}
+ return f
+}
+
+var lcSess = interfaces.SessionRef{TenantID: 1, SessionID: "s"}
+
+func lcActivity(id string) interfaces.BackgroundActivity {
+ return interfaces.BackgroundActivity{ID: id, Kind: "shell_job", Title: "make build", StartedAt: time.Now()}
+}
+
+func liveIDs(t *testing.T, r interfaces.ActivityRegistry) []string {
+ t.Helper()
+ items, err := r.List(context.Background(), lcSess)
+ require.NoError(t, err)
+ ids := make([]string, 0, len(items))
+ for _, it := range items {
+ ids = append(ids, it.ID)
+ }
+ return ids
+}
+
+func TestBeginRegistersBeforeStartAndKeepsAlive(t *testing.T) {
+ f := newLifecycleFixture()
+ var registeredAtStart []string
+ err := f.l.Begin(context.Background(), lcSess, lcActivity("sj_1"), time.Minute, 0,
+ func(context.Context) error {
+ registeredAtStart = liveIDs(t, f.registry)
+ return nil
+ })
+ require.NoError(t, err)
+ require.Equal(t, []string{"sj_1"}, registeredAtStart)
+ require.Equal(t, []string{"sj_1"}, liveIDs(t, f.registry))
+ require.Equal(t, 1, f.keep.calls)
+}
+
+func TestBeginRemovesRegistrationWhenStartFails(t *testing.T) {
+ f := newLifecycleFixture()
+ boom := errors.New("start failed")
+ err := f.l.Begin(context.Background(), lcSess, lcActivity("sj_1"), time.Minute, 0,
+ func(context.Context) error { return boom })
+ require.ErrorIs(t, err, boom)
+ require.Empty(t, liveIDs(t, f.registry))
+ require.Zero(t, f.keep.calls)
+}
+
+func TestBeginReturnsLimitWithoutStarting(t *testing.T) {
+ f := newLifecycleFixture()
+ require.NoError(t, f.l.Begin(context.Background(), lcSess, lcActivity("sj_1"), time.Minute, 1,
+ func(context.Context) error { return nil }))
+ started := false
+ err := f.l.Begin(context.Background(), lcSess, lcActivity("sj_2"), time.Minute, 1,
+ func(context.Context) error { started = true; return nil })
+ require.ErrorIs(t, err, interfaces.ErrActivityLimit)
+ require.False(t, started)
+}
+
+func TestAliveRenewsAndKeepsAliveOnlyWhenFound(t *testing.T) {
+ f := newLifecycleFixture()
+ require.NoError(t, f.registry.Add(context.Background(), lcSess, lcActivity("sj_1"), time.Minute, 0))
+
+ found, err := f.l.Alive(context.Background(), lcSess, "sj_1", time.Minute)
+ require.NoError(t, err)
+ require.True(t, found)
+ require.Equal(t, 1, f.keep.calls)
+
+ found, err = f.l.Alive(context.Background(), lcSess, "missing", time.Minute)
+ require.NoError(t, err)
+ require.False(t, found)
+ require.Equal(t, 1, f.keep.calls)
+}
+
+func TestFinishPushesEmitsOnceAndRemoves(t *testing.T) {
+ f := newLifecycleFixture()
+ ctx := context.Background()
+ require.NoError(t, f.registry.Add(ctx, lcSess, lcActivity("sj_1"), time.Minute, 0))
+ n := interfaces.BackgroundNotice{ID: "sj_1", Kind: "shell_job", Text: "done", FinishedAt: time.Now()}
+ r := interfaces.WorkResult{ID: "sj_1", State: interfaces.WorkStateSucceeded, Terminal: true}
+
+ require.NoError(t, f.l.Finish(ctx, lcSess, n, r, "msg-1"))
+ require.NoError(t, f.l.Finish(ctx, lcSess, n, r, "msg-1")) // second reporter
+
+ require.Len(t, f.events.finished, 1)
+ require.Empty(t, liveIDs(t, f.registry))
+ pending, err := f.inbox.Pending(ctx, lcSess)
+ require.NoError(t, err)
+ require.Len(t, pending, 1)
+}
+
+func TestFinishKeepsRegistrationWhenPushFails(t *testing.T) {
+ f := newLifecycleFixture()
+ ctx := context.Background()
+ require.NoError(t, f.registry.Add(ctx, lcSess, lcActivity("sj_1"), time.Minute, 0))
+ f.l.Inbox = failingInbox{}
+
+ err := f.l.Finish(ctx, lcSess, interfaces.BackgroundNotice{ID: "sj_1"}, interfaces.WorkResult{ID: "sj_1"}, "")
+ require.Error(t, err)
+ require.Equal(t, []string{"sj_1"}, liveIDs(t, f.registry),
+ "without a notice the item must stay visible, or queries would find it nowhere")
+ require.Empty(t, f.events.finished)
+}
+
+func TestDiscardRemovesWithoutNoticeOrEvent(t *testing.T) {
+ f := newLifecycleFixture()
+ ctx := context.Background()
+ require.NoError(t, f.registry.Add(ctx, lcSess, lcActivity("sj_1"), time.Minute, 0))
+
+ require.NoError(t, f.l.Discard(ctx, lcSess, "sj_1"))
+
+ require.Empty(t, liveIDs(t, f.registry))
+ pending, err := f.inbox.Pending(ctx, lcSess)
+ require.NoError(t, err)
+ require.Empty(t, pending)
+ require.Empty(t, f.events.finished)
+}
+
+func TestLifecycleToleratesNilOptionalParts(t *testing.T) {
+ l := &Lifecycle{Registry: NewMemoryActivityRegistry(nil), Inbox: NewMemoryNoticeInbox(nil)}
+ ctx := context.Background()
+ require.NoError(t, l.Begin(ctx, lcSess, lcActivity("sj_1"), time.Minute, 0,
+ func(context.Context) error { return nil }))
+ _, err := l.Alive(ctx, lcSess, "sj_1", time.Minute)
+ require.NoError(t, err)
+ require.NoError(t, l.Finish(ctx, lcSess, interfaces.BackgroundNotice{ID: "sj_1", FinishedAt: time.Now()},
+ interfaces.WorkResult{ID: "sj_1"}, ""))
+}
+
+type failingInbox struct{}
+
+func (failingInbox) Push(context.Context, interfaces.SessionRef, interfaces.BackgroundNotice) (bool, error) {
+ return false, errors.New("redis down")
+}
+
+func (failingInbox) Pending(context.Context, interfaces.SessionRef) ([]interfaces.BackgroundNotice, error) {
+ return nil, nil
+}
+
+func (failingInbox) Ack(context.Context, interfaces.SessionRef, []string) error { return nil }
+
+func (failingInbox) Clear(context.Context, interfaces.SessionRef) error { return nil }
diff --git a/internal/background/listeners.go b/internal/background/listeners.go
new file mode 100644
index 000000000..457841535
--- /dev/null
+++ b/internal/background/listeners.go
@@ -0,0 +1,81 @@
+package background
+
+import (
+ "context"
+ "sync"
+
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+// Listeners is the ordered set of session lifecycle listeners. Feature
+// listeners run in registration order; on delete the shared activity
+// registry and inbox are cleared after all of them have run.
+type Listeners struct {
+ registry interfaces.ActivityRegistry
+ inbox interfaces.NoticeInbox
+ logf func(ctx context.Context, format string, args ...any)
+
+ mu sync.RWMutex
+ listeners []interfaces.SessionLifecycleListener
+}
+
+// NewListeners returns an empty listener set. A nil logf discards warnings.
+func NewListeners(
+ registry interfaces.ActivityRegistry,
+ inbox interfaces.NoticeInbox,
+ logf func(ctx context.Context, format string, args ...any),
+) *Listeners {
+ if logf == nil {
+ logf = func(context.Context, string, ...any) {}
+ }
+ return &Listeners{registry: registry, inbox: inbox, logf: logf}
+}
+
+// Register appends a listener.
+func (l *Listeners) Register(listener interfaces.SessionLifecycleListener) {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ l.listeners = append(l.listeners, listener)
+}
+
+func (l *Listeners) snapshot() []interfaces.SessionLifecycleListener {
+ l.mu.RLock()
+ defer l.mu.RUnlock()
+ return append([]interfaces.SessionLifecycleListener(nil), l.listeners...)
+}
+
+// NotifyStop calls OnStop on every listener. Errors are logged, never returned.
+func (l *Listeners) NotifyStop(ctx context.Context, s interfaces.SessionRef) {
+ for _, listener := range l.snapshot() {
+ if err := listener.OnStop(ctx, s); err != nil {
+ l.logf(ctx, "[Background] OnStop listener failed session=%s: %v", s.SessionID, err)
+ }
+ }
+}
+
+// NotifyDelete calls OnDelete on every listener, then clears the session's
+// activity items and notices. Errors are logged, never returned.
+func (l *Listeners) NotifyDelete(ctx context.Context, s interfaces.SessionRef) {
+ for _, listener := range l.snapshot() {
+ if err := listener.OnDelete(ctx, s); err != nil {
+ l.logf(ctx, "[Background] OnDelete listener failed session=%s: %v", s.SessionID, err)
+ }
+ }
+ if l.registry != nil {
+ items, err := l.registry.List(ctx, s)
+ if err != nil {
+ l.logf(ctx, "[Background] list activity on delete failed session=%s: %v", s.SessionID, err)
+ }
+ for _, it := range items {
+ if err := l.registry.Remove(ctx, s, it.ID); err != nil {
+ l.logf(ctx, "[Background] remove activity on delete failed session=%s id=%s: %v",
+ s.SessionID, it.ID, err)
+ }
+ }
+ }
+ if l.inbox != nil {
+ if err := l.inbox.Clear(ctx, s); err != nil {
+ l.logf(ctx, "[Background] clear inbox on delete failed session=%s: %v", s.SessionID, err)
+ }
+ }
+}
diff --git a/internal/background/listeners_test.go b/internal/background/listeners_test.go
new file mode 100644
index 000000000..1fd10521f
--- /dev/null
+++ b/internal/background/listeners_test.go
@@ -0,0 +1,95 @@
+package background
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+type recordingListener struct {
+ name string
+ log *[]string
+ err error
+}
+
+func (l recordingListener) OnStop(context.Context, interfaces.SessionRef) error {
+ *l.log = append(*l.log, l.name+":stop")
+ return l.err
+}
+
+func (l recordingListener) OnDelete(context.Context, interfaces.SessionRef) error {
+ *l.log = append(*l.log, l.name+":delete")
+ return l.err
+}
+
+func TestListenersRunInOrderAndSurviveErrors(t *testing.T) {
+ var log []string
+ var warnings []string
+ logf := func(_ context.Context, format string, args ...any) {
+ warnings = append(warnings, fmt.Sprintf(format, args...))
+ }
+ l := NewListeners(NewMemoryActivityRegistry(nil), NewMemoryNoticeInbox(nil), logf)
+ l.Register(recordingListener{name: "a", log: &log, err: errors.New("a failed")})
+ l.Register(recordingListener{name: "b", log: &log})
+
+ s := interfaces.SessionRef{TenantID: 1, SessionID: "s"}
+ l.NotifyStop(context.Background(), s)
+ l.NotifyDelete(context.Background(), s)
+
+ require.Equal(t, []string{"a:stop", "b:stop", "a:delete", "b:delete"}, log)
+ require.Len(t, warnings, 2)
+}
+
+func TestNotifyDeleteClearsSharedStateLast(t *testing.T) {
+ ctx := context.Background()
+ s := interfaces.SessionRef{TenantID: 1, SessionID: "s"}
+ registry := NewMemoryActivityRegistry(nil)
+ inbox := NewMemoryNoticeInbox(nil)
+ require.NoError(t, registry.Add(ctx, s, interfaces.BackgroundActivity{ID: "sj_1", Kind: "k"}, time.Hour, 0))
+ _, err := inbox.Push(ctx, s, interfaces.BackgroundNotice{ID: "sj_0", FinishedAt: time.Now()})
+ require.NoError(t, err)
+
+ var sawDuringListener int
+ l := NewListeners(registry, inbox, nil)
+ l.Register(listenerFunc(func(ctx context.Context, s interfaces.SessionRef) {
+ items, _ := registry.List(ctx, s)
+ sawDuringListener = len(items)
+ }))
+
+ l.NotifyDelete(ctx, s)
+
+ require.Equal(t, 1, sawDuringListener, "feature listeners must run before shared state is cleared")
+ items, err := registry.List(ctx, s)
+ require.NoError(t, err)
+ require.Empty(t, items)
+ pending, err := inbox.Pending(ctx, s)
+ require.NoError(t, err)
+ require.Empty(t, pending)
+}
+
+func TestNotifyStopKeepsSharedState(t *testing.T) {
+ ctx := context.Background()
+ s := interfaces.SessionRef{TenantID: 1, SessionID: "s"}
+ registry := NewMemoryActivityRegistry(nil)
+ require.NoError(t, registry.Add(ctx, s, interfaces.BackgroundActivity{ID: "sj_1", Kind: "k"}, time.Hour, 0))
+
+ NewListeners(registry, NewMemoryNoticeInbox(nil), nil).NotifyStop(ctx, s)
+
+ items, err := registry.List(ctx, s)
+ require.NoError(t, err)
+ require.Len(t, items, 1)
+}
+
+type listenerFunc func(ctx context.Context, s interfaces.SessionRef)
+
+func (f listenerFunc) OnStop(context.Context, interfaces.SessionRef) error { return nil }
+
+func (f listenerFunc) OnDelete(ctx context.Context, s interfaces.SessionRef) error {
+ f(ctx, s)
+ return nil
+}
diff --git a/internal/background/memory_inbox.go b/internal/background/memory_inbox.go
new file mode 100644
index 000000000..3254cbf37
--- /dev/null
+++ b/internal/background/memory_inbox.go
@@ -0,0 +1,97 @@
+package background
+
+import (
+ "context"
+ "sort"
+ "sync"
+
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+// MemoryNoticeInbox is the single-process NoticeInbox used by Lite.
+type MemoryNoticeInbox struct {
+ clock Clock
+ mu sync.Mutex
+ notices map[interfaces.SessionRef]map[string]interfaces.BackgroundNotice
+}
+
+var _ interfaces.NoticeInbox = (*MemoryNoticeInbox)(nil)
+
+// NewMemoryNoticeInbox returns an empty inbox. A nil clock means SystemClock.
+func NewMemoryNoticeInbox(clock Clock) *MemoryNoticeInbox {
+ if clock == nil {
+ clock = SystemClock{}
+ }
+ return &MemoryNoticeInbox{
+ clock: clock,
+ notices: make(map[interfaces.SessionRef]map[string]interfaces.BackgroundNotice),
+ }
+}
+
+// Push implements interfaces.NoticeInbox.
+func (b *MemoryNoticeInbox) Push(
+ _ context.Context, s interfaces.SessionRef, n interfaces.BackgroundNotice,
+) (bool, error) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ m := b.notices[s]
+ if _, ok := m[n.ID]; ok {
+ return false, nil
+ }
+ if m == nil {
+ m = make(map[string]interfaces.BackgroundNotice)
+ b.notices[s] = m
+ }
+ m[n.ID] = n
+ return true, nil
+}
+
+// Pending implements interfaces.NoticeInbox.
+func (b *MemoryNoticeInbox) Pending(
+ _ context.Context, s interfaces.SessionRef,
+) ([]interfaces.BackgroundNotice, error) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ cutoff := b.clock.Now().Add(-interfaces.NoticeRetention)
+ m := b.notices[s]
+ out := make([]interfaces.BackgroundNotice, 0, len(m))
+ for id, n := range m {
+ if n.FinishedAt.Before(cutoff) {
+ delete(m, id)
+ continue
+ }
+ out = append(out, n)
+ }
+ if len(m) == 0 {
+ delete(b.notices, s)
+ }
+ sort.Slice(out, func(i, j int) bool {
+ if out[i].FinishedAt.Equal(out[j].FinishedAt) {
+ return out[i].ID < out[j].ID
+ }
+ return out[i].FinishedAt.Before(out[j].FinishedAt)
+ })
+ return out, nil
+}
+
+// Ack implements interfaces.NoticeInbox.
+func (b *MemoryNoticeInbox) Ack(_ context.Context, s interfaces.SessionRef, ids []string) error {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ m := b.notices[s]
+ for _, id := range ids {
+ delete(m, id)
+ }
+ if len(m) == 0 {
+ delete(b.notices, s)
+ }
+ return nil
+}
+
+// Clear implements interfaces.NoticeInbox.
+func (b *MemoryNoticeInbox) Clear(_ context.Context, s interfaces.SessionRef) error {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ delete(b.notices, s)
+ return nil
+}
diff --git a/internal/background/memory_registry.go b/internal/background/memory_registry.go
new file mode 100644
index 000000000..18f698944
--- /dev/null
+++ b/internal/background/memory_registry.go
@@ -0,0 +1,139 @@
+// Package background holds the shared background-work parts: in-memory
+// implementations, the lifecycle-listener registry and the lifecycle helpers.
+// It must not branch on activity kind.
+package background
+
+import (
+ "context"
+ "sync"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+// Clock abstracts time so tests can move it.
+type Clock interface {
+ Now() time.Time
+}
+
+// SystemClock is the real wall clock.
+type SystemClock struct{}
+
+// Now returns time.Now().
+func (SystemClock) Now() time.Time { return time.Now() }
+
+type memoryActivity struct {
+ item interfaces.BackgroundActivity
+ deadline time.Time
+}
+
+// MemoryActivityRegistry is the single-process ActivityRegistry used by Lite.
+type MemoryActivityRegistry struct {
+ clock Clock
+ mu sync.Mutex
+ items map[interfaces.SessionRef]map[string]memoryActivity
+}
+
+var _ interfaces.ActivityRegistry = (*MemoryActivityRegistry)(nil)
+
+// NewMemoryActivityRegistry returns an empty registry. A nil clock means SystemClock.
+func NewMemoryActivityRegistry(clock Clock) *MemoryActivityRegistry {
+ if clock == nil {
+ clock = SystemClock{}
+ }
+ return &MemoryActivityRegistry{
+ clock: clock,
+ items: make(map[interfaces.SessionRef]map[string]memoryActivity),
+ }
+}
+
+// liveLocked drops expired items of s and returns what remains. Caller holds mu.
+func (r *MemoryActivityRegistry) liveLocked(s interfaces.SessionRef, now time.Time) map[string]memoryActivity {
+ m := r.items[s]
+ for id, it := range m {
+ if !now.Before(it.deadline) {
+ delete(m, id)
+ }
+ }
+ if len(m) == 0 {
+ delete(r.items, s)
+ return nil
+ }
+ return m
+}
+
+// Add implements interfaces.ActivityRegistry.
+func (r *MemoryActivityRegistry) Add(
+ _ context.Context, s interfaces.SessionRef, a interfaces.BackgroundActivity, lease time.Duration, kindCap int,
+) error {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ now := r.clock.Now()
+ m := r.liveLocked(s, now)
+ if existing, ok := m[a.ID]; ok {
+ existing.deadline = now.Add(lease)
+ m[a.ID] = existing
+ return nil
+ }
+ if kindCap > 0 {
+ count := 0
+ for _, it := range m {
+ if it.item.Kind == a.Kind {
+ count++
+ }
+ }
+ if count >= kindCap {
+ return interfaces.ErrActivityLimit
+ }
+ }
+ if m == nil {
+ m = make(map[string]memoryActivity)
+ r.items[s] = m
+ }
+ m[a.ID] = memoryActivity{item: a, deadline: now.Add(lease)}
+ return nil
+}
+
+// Renew implements interfaces.ActivityRegistry.
+func (r *MemoryActivityRegistry) Renew(
+ _ context.Context, s interfaces.SessionRef, id string, lease time.Duration,
+) (bool, error) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ now := r.clock.Now()
+ m := r.liveLocked(s, now)
+ it, ok := m[id]
+ if !ok {
+ return false, nil
+ }
+ it.deadline = now.Add(lease)
+ m[id] = it
+ return true, nil
+}
+
+// Remove implements interfaces.ActivityRegistry.
+func (r *MemoryActivityRegistry) Remove(_ context.Context, s interfaces.SessionRef, id string) error {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ if m := r.items[s]; m != nil {
+ delete(m, id)
+ if len(m) == 0 {
+ delete(r.items, s)
+ }
+ }
+ return nil
+}
+
+// List implements interfaces.ActivityRegistry.
+func (r *MemoryActivityRegistry) List(
+ _ context.Context, s interfaces.SessionRef,
+) ([]interfaces.BackgroundActivity, error) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ m := r.liveLocked(s, r.clock.Now())
+ out := make([]interfaces.BackgroundActivity, 0, len(m))
+ for _, it := range m {
+ out = append(out, it.item)
+ }
+ return out, nil
+}
diff --git a/internal/background/memory_test.go b/internal/background/memory_test.go
new file mode 100644
index 000000000..06a5ea433
--- /dev/null
+++ b/internal/background/memory_test.go
@@ -0,0 +1,47 @@
+package background
+
+import (
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/background/backgroundtest"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+type fakeClock struct {
+ mu sync.Mutex
+ now time.Time
+}
+
+func newFakeClock() *fakeClock {
+ return &fakeClock{now: time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC)}
+}
+
+func (c *fakeClock) Now() time.Time {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ return c.now
+}
+
+func (c *fakeClock) Advance(d time.Duration) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ c.now = c.now.Add(d)
+}
+
+func TestMemoryActivityRegistryContract(t *testing.T) {
+ backgroundtest.RunActivityRegistryContract(t,
+ func(*testing.T) (interfaces.ActivityRegistry, backgroundtest.Advance) {
+ clock := newFakeClock()
+ return NewMemoryActivityRegistry(clock), clock.Advance
+ })
+}
+
+func TestMemoryNoticeInboxContract(t *testing.T) {
+ backgroundtest.RunNoticeInboxContract(t,
+ func(*testing.T) (interfaces.NoticeInbox, backgroundtest.Advance) {
+ clock := newFakeClock()
+ return NewMemoryNoticeInbox(clock), clock.Advance
+ })
+}
diff --git a/internal/background/run_notices.go b/internal/background/run_notices.go
new file mode 100644
index 000000000..94637e770
--- /dev/null
+++ b/internal/background/run_notices.go
@@ -0,0 +1,185 @@
+package background
+
+import (
+ "context"
+ "fmt"
+ "sort"
+ "strings"
+ "sync"
+
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+// Delivery is background-work context for the model at one round boundary.
+type Delivery struct {
+ // Text is the model-facing body, one line per entry, without a wrapper.
+ Text string
+ // AckIDs are the completion notices Text delivers. They are acknowledged
+ // once the assistant message carrying them is persisted.
+ AckIDs []string
+}
+
+// RunNotices decides, for one agent run, what the model is told about
+// background work and when. A run executes in one process, so its sets live in
+// memory; the shared registry and inbox are only read here.
+type RunNotices struct {
+ session interfaces.SessionRef
+ registry interfaces.ActivityRegistry
+ inbox interfaces.NoticeInbox
+
+ mu sync.Mutex
+ listed bool
+ seen map[string]bool
+ started map[string]bool
+ reminded map[string]bool
+ newSeen []string
+}
+
+// NewRunNotices returns a tracker for one run of session s.
+func NewRunNotices(
+ s interfaces.SessionRef, registry interfaces.ActivityRegistry, inbox interfaces.NoticeInbox,
+) *RunNotices {
+ return &RunNotices{
+ session: s,
+ registry: registry,
+ inbox: inbox,
+ seen: make(map[string]bool),
+ started: make(map[string]bool),
+ reminded: make(map[string]bool),
+ }
+}
+
+// MarkStarted records work this run submitted and got back as still running.
+// Only such work is reminded about before the final answer.
+func (r *RunNotices) MarkStarted(id string) {
+ if id == "" {
+ return
+ }
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ r.started[id] = true
+}
+
+// MarkSeen records finished IDs a tool already showed the model, so their
+// notices are not injected again and are acknowledged with the turn.
+func (r *RunNotices) MarkSeen(ids ...string) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ for _, id := range ids {
+ if id == "" || r.seen[id] {
+ continue
+ }
+ r.seen[id] = true
+ r.newSeen = append(r.newSeen, id)
+ }
+}
+
+// TakeSeen returns the IDs MarkSeen recorded since the previous call.
+func (r *RunNotices) TakeSeen() []string {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ out := r.newSeen
+ r.newSeen = nil
+ return out
+}
+
+// RoundStart returns what to tell the model before a round's LLM call: on the
+// run's first boundary, the work still running; on every boundary, completion
+// notices this run has not seen.
+func (r *RunNotices) RoundStart(ctx context.Context) (Delivery, error) {
+ r.mu.Lock()
+ first := !r.listed
+ r.mu.Unlock()
+
+ var lines []string
+ if first {
+ live, err := r.registry.List(ctx, r.session)
+ if err != nil {
+ return Delivery{}, fmt.Errorf("list background work: %w", err)
+ }
+ if len(live) > 0 {
+ lines = append(lines, "Background work still running in this session: "+describeActivities(live)+".")
+ }
+ }
+ notices, err := r.inbox.Pending(ctx, r.session)
+ if err != nil {
+ return Delivery{}, fmt.Errorf("read background notices: %w", err)
+ }
+
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ r.listed = true
+ return r.deliverLocked(notices, lines), nil
+}
+
+// BeforeFinal returns what to tell the model before it may finish: unseen
+// completion notices, and one reminder for each item this run started that is
+// still running and has not been reminded about.
+func (r *RunNotices) BeforeFinal(ctx context.Context) (Delivery, error) {
+ notices, err := r.inbox.Pending(ctx, r.session)
+ if err != nil {
+ return Delivery{}, fmt.Errorf("read background notices: %w", err)
+ }
+ live, err := r.registry.List(ctx, r.session)
+ if err != nil {
+ return Delivery{}, fmt.Errorf("list background work: %w", err)
+ }
+
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ var remind []interfaces.BackgroundActivity
+ for _, item := range live {
+ if r.started[item.ID] && !r.reminded[item.ID] {
+ remind = append(remind, item)
+ }
+ }
+ var lines []string
+ if len(remind) > 0 {
+ for _, item := range remind {
+ r.reminded[item.ID] = true
+ }
+ lines = append(lines, "Background work you started in this turn is still running: "+
+ describeActivities(remind)+
+ ". Finish your answer. Its result will arrive when it completes. "+
+ "Call work_wait only if you cannot answer without that output.")
+ }
+ return r.deliverLocked(notices, lines), nil
+}
+
+// deliverLocked puts unseen notices first, then lines, and marks the notices
+// seen. Caller holds mu.
+func (r *RunNotices) deliverLocked(notices []interfaces.BackgroundNotice, lines []string) Delivery {
+ var d Delivery
+ text := make([]string, 0, len(notices)+len(lines))
+ for _, n := range notices {
+ if r.seen[n.ID] || strings.TrimSpace(n.Text) == "" {
+ continue
+ }
+ r.seen[n.ID] = true
+ text = append(text, n.Text)
+ d.AckIDs = append(d.AckIDs, n.ID)
+ }
+ text = append(text, lines...)
+ d.Text = strings.Join(text, "\n")
+ return d
+}
+
+// describeActivities renders items as "title (id)" in start order.
+func describeActivities(items []interfaces.BackgroundActivity) string {
+ sorted := append([]interfaces.BackgroundActivity(nil), items...)
+ sort.Slice(sorted, func(i, j int) bool {
+ if sorted[i].StartedAt.Equal(sorted[j].StartedAt) {
+ return sorted[i].ID < sorted[j].ID
+ }
+ return sorted[i].StartedAt.Before(sorted[j].StartedAt)
+ })
+ parts := make([]string, 0, len(sorted))
+ for _, item := range sorted {
+ if item.Title == "" {
+ parts = append(parts, item.ID)
+ continue
+ }
+ parts = append(parts, fmt.Sprintf("%s (%s)", item.Title, item.ID))
+ }
+ return strings.Join(parts, ", ")
+}
diff --git a/internal/background/run_notices_test.go b/internal/background/run_notices_test.go
new file mode 100644
index 000000000..9c3753abf
--- /dev/null
+++ b/internal/background/run_notices_test.go
@@ -0,0 +1,133 @@
+package background
+
+import (
+ "context"
+ "errors"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+var noticeSession = interfaces.SessionRef{TenantID: 1, SessionID: "s"}
+
+func newNoticeFixture(t *testing.T) (*RunNotices, *MemoryActivityRegistry, *MemoryNoticeInbox) {
+ t.Helper()
+ reg := NewMemoryActivityRegistry(nil)
+ inbox := NewMemoryNoticeInbox(nil)
+ return NewRunNotices(noticeSession, reg, inbox), reg, inbox
+}
+
+func addRunning(t *testing.T, reg *MemoryActivityRegistry, id, title string, started time.Time) {
+ t.Helper()
+ require.NoError(t, reg.Add(context.Background(), noticeSession, interfaces.BackgroundActivity{
+ ID: id, Kind: "shell_job", Title: title, StartedAt: started,
+ }, time.Hour, 0))
+}
+
+func pushNotice(t *testing.T, inbox *MemoryNoticeInbox, id, text string) {
+ t.Helper()
+ _, err := inbox.Push(context.Background(), noticeSession, interfaces.BackgroundNotice{
+ ID: id, Kind: "shell_job", Text: text, FinishedAt: time.Now(),
+ })
+ require.NoError(t, err)
+}
+
+func TestRunNoticesListsRunningWorkOnFirstRoundOnly(t *testing.T) {
+ r, reg, _ := newNoticeFixture(t)
+ base := time.Now()
+ addRunning(t, reg, "sj_b", "tests", base.Add(time.Second))
+ addRunning(t, reg, "sj_a", "build", base)
+
+ first, err := r.RoundStart(context.Background())
+ require.NoError(t, err)
+ require.Equal(t, "Background work still running in this session: build (sj_a), tests (sj_b).", first.Text)
+ require.Empty(t, first.AckIDs)
+
+ second, err := r.RoundStart(context.Background())
+ require.NoError(t, err)
+ require.Empty(t, second.Text)
+}
+
+func TestRunNoticesDeliversEachNoticeOnce(t *testing.T) {
+ r, _, inbox := newNoticeFixture(t)
+ pushNotice(t, inbox, "sj_1", "build (sj_1) succeeded.")
+
+ got, err := r.RoundStart(context.Background())
+ require.NoError(t, err)
+ require.Equal(t, "build (sj_1) succeeded.", got.Text)
+ require.Equal(t, []string{"sj_1"}, got.AckIDs)
+
+ pushNotice(t, inbox, "sj_2", "tests (sj_2) failed.")
+ got, err = r.BeforeFinal(context.Background())
+ require.NoError(t, err)
+ require.Equal(t, "tests (sj_2) failed.", got.Text, "sj_1 was already delivered this run")
+ require.Equal(t, []string{"sj_2"}, got.AckIDs)
+}
+
+func TestRunNoticesSkipsNoticesAToolAlreadyShowed(t *testing.T) {
+ r, _, inbox := newNoticeFixture(t)
+ pushNotice(t, inbox, "sj_1", "build (sj_1) succeeded.")
+
+ r.MarkSeen("sj_1", "sj_1", "")
+ require.Equal(t, []string{"sj_1"}, r.TakeSeen())
+ require.Empty(t, r.TakeSeen())
+
+ got, err := r.RoundStart(context.Background())
+ require.NoError(t, err)
+ require.Empty(t, got.Text)
+ require.Empty(t, got.AckIDs)
+}
+
+func TestRunNoticesRemindsOnlyWorkThisRunStarted(t *testing.T) {
+ r, reg, _ := newNoticeFixture(t)
+ base := time.Now()
+ addRunning(t, reg, "sj_old", "dev server", base)
+ addRunning(t, reg, "sj_new", "build", base.Add(time.Second))
+ r.MarkStarted("sj_new")
+
+ got, err := r.BeforeFinal(context.Background())
+ require.NoError(t, err)
+ require.Equal(t, "Background work you started in this turn is still running: build (sj_new). "+
+ "Finish your answer. Its result will arrive when it completes. "+
+ "Call work_wait only if you cannot answer without that output.", got.Text)
+
+ again, err := r.BeforeFinal(context.Background())
+ require.NoError(t, err)
+ require.Empty(t, again.Text, "each item is reminded once")
+}
+
+func TestRunNoticesPutsNoticesBeforeOtherLines(t *testing.T) {
+ r, reg, inbox := newNoticeFixture(t)
+ addRunning(t, reg, "sj_a", "build", time.Now())
+ pushNotice(t, inbox, "sj_0", "lint (sj_0) succeeded.")
+
+ got, err := r.RoundStart(context.Background())
+ require.NoError(t, err)
+ require.Equal(t, "lint (sj_0) succeeded.\nBackground work still running in this session: build (sj_a).", got.Text)
+}
+
+type failingRegistry struct {
+ interfaces.ActivityRegistry
+ err error
+}
+
+func (f failingRegistry) List(context.Context, interfaces.SessionRef) ([]interfaces.BackgroundActivity, error) {
+ return nil, f.err
+}
+
+func TestRunNoticesRetriesTheListingAfterAnError(t *testing.T) {
+ reg := NewMemoryActivityRegistry(nil)
+ r := NewRunNotices(noticeSession, failingRegistry{ActivityRegistry: reg, err: errors.New("redis down")},
+ NewMemoryNoticeInbox(nil))
+ addRunning(t, reg, "sj_a", "build", time.Now())
+
+ _, err := r.RoundStart(context.Background())
+ require.Error(t, err)
+
+ r.registry = reg // the store recovers
+ got, err := r.RoundStart(context.Background())
+ require.NoError(t, err)
+ require.Contains(t, got.Text, "build (sj_a)", "a failed first boundary must not use up the listing")
+}
diff --git a/internal/container/background.go b/internal/container/background.go
new file mode 100644
index 000000000..57f4c8b2c
--- /dev/null
+++ b/internal/container/background.go
@@ -0,0 +1,40 @@
+// Package container - background-work wiring.
+package container
+
+import (
+ "github.com/Tencent/WeKnora/internal/application/service/sandboxalloc"
+ "github.com/Tencent/WeKnora/internal/background"
+ "github.com/Tencent/WeKnora/internal/sandbox"
+ "github.com/Tencent/WeKnora/internal/stream"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+// newBackgroundActivityRegistry returns the registry of live background work.
+// With Redis it lives on the binding store, so turn leases, the rebuild check
+// and the rewind lock read it in the same Lua call. The memory store uses
+// the same lock for activity registration and lifecycle checks.
+func newBackgroundActivityRegistry(
+ store sandbox.SessionSandboxBindingStore, allocations *sandboxalloc.Service,
+) interfaces.ActivityRegistry {
+ switch store := store.(type) {
+ case *sandbox.RedisSessionSandboxBindingStore:
+ registry := sandbox.NewRedisActivityRegistry(store)
+ if allocations != nil {
+ registry.WithAllocations(allocations)
+ }
+ return registry
+ case *sandbox.MemorySessionSandboxBindingStore:
+ return sandbox.NewMemoryActivityRegistry(store)
+ }
+ return background.NewMemoryActivityRegistry(nil)
+}
+
+// newBackgroundNoticeInbox returns the completion-notice inbox. With Redis it
+// lives next to the steer lists; otherwise it is process-local, like the
+// in-memory stream manager it follows.
+func newBackgroundNoticeInbox(streams interfaces.StreamManager) interfaces.NoticeInbox {
+ if redisStreams, ok := streams.(*stream.RedisStreamManager); ok {
+ return stream.NewRedisNoticeInbox(redisStreams, nil)
+ }
+ return background.NewMemoryNoticeInbox(nil)
+}
diff --git a/internal/container/background_test.go b/internal/container/background_test.go
new file mode 100644
index 000000000..6f6cdc593
--- /dev/null
+++ b/internal/container/background_test.go
@@ -0,0 +1,72 @@
+package container
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/application/service/sandboxalloc"
+ "github.com/Tencent/WeKnora/internal/sandbox"
+ "github.com/Tencent/WeKnora/internal/sandbox/allocation"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/alicebob/miniredis/v2"
+ "github.com/redis/go-redis/v9"
+ "github.com/stretchr/testify/require"
+)
+
+func TestBackgroundActivityRegistrySharesMemoryBindingState(t *testing.T) {
+ ctx := t.Context()
+ store := sandbox.NewMemorySessionSandboxBindingStore()
+ registry := newBackgroundActivityRegistry(store, nil)
+ ref := interfaces.SessionRef{TenantID: 42, SessionID: "s1"}
+ key := sandbox.SessionSandboxKey{TenantID: ref.TenantID, SessionID: ref.SessionID}
+ require.NoError(t, registry.Add(ctx, ref, interfaces.BackgroundActivity{ID: "sj_1"}, time.Minute, 0))
+ require.NoError(t, store.BeginTurn(ctx, key))
+ active, rebuild, err := store.TurnState(ctx, key)
+ require.NoError(t, err)
+ require.True(t, active)
+ require.False(t, rebuild)
+ require.NoError(t, store.EndTurn(ctx, key))
+ _, err = store.TryLockRewind(ctx, key)
+ require.ErrorIs(t, err, sandbox.ErrSessionWorkActive)
+}
+
+type backgroundAllocationRepo struct {
+ interfaces.SandboxAllocationRepository
+ assignment *types.SessionSandboxAssignment
+ machine *types.SandboxUserMachine
+}
+
+func (r *backgroundAllocationRepo) GetAssignment(context.Context, string) (*types.SessionSandboxAssignment, error) {
+ return r.assignment, nil
+}
+
+func (r *backgroundAllocationRepo) GetUserMachine(context.Context, string) (*types.SandboxUserMachine, error) {
+ return r.machine, nil
+}
+
+func TestBackgroundActivityRegistryUsesRecordedUserAllocation(t *testing.T) {
+ ctx := t.Context()
+ mini := miniredis.RunT(t)
+ client := redis.NewClient(&redis.Options{Addr: mini.Addr()})
+ t.Cleanup(func() { require.NoError(t, client.Close()) })
+ store, err := sandbox.NewRedisSessionSandboxBindingStore(client, "test")
+ require.NoError(t, err)
+ a := allocation.Assignment{
+ SessionID: "0f8fad5b-d9cb-469f-a165-70867728950e", ConfigTenantID: 42, ConfigID: "cfg-1",
+ Machine: allocation.UserMachine{OwnerTenantID: 42, OwnerID: "u1", ConfigTenantID: 42, ConfigID: "cfg-1"},
+ }
+ row, machine := sandboxalloc.ToEntity(a, a.Machine.Digest("test"))
+ allocations := sandboxalloc.New(&backgroundAllocationRepo{assignment: row, machine: machine}, nil, nil, "test")
+ registry := newBackgroundActivityRegistry(store, allocations)
+ ref := interfaces.SessionRef{TenantID: 42, SessionID: a.SessionID}
+ require.NoError(t, registry.Add(ctx, ref, interfaces.BackgroundActivity{ID: "sj_1"}, time.Minute, 0))
+ key := sandbox.SessionSandboxKey{TenantID: ref.TenantID, SessionID: ref.SessionID, User: a.Machine}
+ active, rebuild, err := store.TurnState(ctx, key)
+ require.NoError(t, err)
+ require.True(t, active, "the production factory must address the same leases as the shared binding store")
+ require.False(t, rebuild)
+ _, err = store.TryLockRewind(ctx, key)
+ require.ErrorIs(t, err, sandbox.ErrSessionWorkActive)
+}
diff --git a/internal/container/container.go b/internal/container/container.go
index 50493e5b4..3151e2875 100644
--- a/internal/container/container.go
+++ b/internal/container/container.go
@@ -236,6 +236,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
return b
}))
must(container.Provide(newAdmissionRegistry))
+ must(container.Provide(newSessionBindingStore))
must(container.Provide(newTenantSandboxResolver))
// The resolver admits user-mode sessions, so the allocation service has to
// exist before the first Invoke that builds a TenantSandboxResolver
@@ -247,6 +248,9 @@ func BuildContainer(container *dig.Container) *dig.Container {
) *sandboxalloc.Service {
return sandboxalloc.New(repo, configs, sessions, sandboxRedisNamespace())
}))
+ must(container.Provide(newBackgroundActivityRegistry))
+ must(container.Provide(newBackgroundNoticeInbox))
+ must(container.Provide(service.NewBackgroundWork))
// Business service layer
logger.Debugf(ctx, "[Container] Registering business services...")
@@ -436,11 +440,12 @@ func BuildContainer(container *dig.Container) *dig.Container {
must(container.Provide(func(
mgr sandbox.Manager,
pinned *service.PinnedSessionSandbox,
+ activity interfaces.ActivityRegistry,
) *service.WorkspaceCheckpointer {
if runner, ok := mgr.(service.SandboxShellRunner); ok {
- return service.NewWorkspaceCheckpointer(runner)
+ return service.NewWorkspaceCheckpointer(runner).WithActivityRegistry(activity)
}
- return service.NewWorkspaceCheckpointer(pinned)
+ return service.NewWorkspaceCheckpointer(pinned).WithActivityRegistry(activity)
}))
must(container.Provide(func(
mgr sandbox.Manager,
diff --git a/internal/container/sandbox.go b/internal/container/sandbox.go
index b0c2080f9..7b86a9527 100644
--- a/internal/container/sandbox.go
+++ b/internal/container/sandbox.go
@@ -71,6 +71,22 @@ func selectSessionBindingStore(
return sandbox.NewMemorySessionSandboxBindingStore(), "memory", nil
}
+// newSessionBindingStore selects the process-wide session→sandbox binding
+// store. The tenant sandbox resolver and the background activity registry
+// share it: the registry's Redis hash must use the binding store's client,
+// namespace and hash tag so the turn-lease scripts can read it. A nil result
+// disables per-tenant sandbox configs, as a failed selection always has.
+func newSessionBindingStore(redisClient *redis.Client) sandbox.SessionSandboxBindingStore {
+ ctx := context.Background()
+ store, storeKind, err := selectSessionBindingStore(redisClient, true)
+ if err != nil {
+ logger.Warnf(ctx, "Per-tenant sandbox config disabled: %v", err)
+ return nil
+ }
+ logger.Infof(ctx, "Session sandbox binding store: %s", storeKind)
+ return store
+}
+
// sessionExistenceLookup is the narrow slice of SessionRepository the
// session existence checker actually needs. Declaring it here (rather than
// depending on interfaces.SessionRepository) keeps the checker easy to test
@@ -133,7 +149,7 @@ func buildGlobalSandboxConfig() *sandbox.Config {
func newTenantSandboxResolver(
defaultManager sandbox.Manager,
loader sandbox.TenantSandboxConfigLoader,
- redisClient *redis.Client,
+ store sandbox.SessionSandboxBindingStore,
sessionRepo interfaces.SessionRepository,
bootstrapper sandbox.SessionBootstrapper,
alloc *sandboxalloc.Service,
@@ -143,10 +159,8 @@ func newTenantSandboxResolver(
// Tenants may configure any supported backend regardless of process startup
// mode. Remote configs use this binding store for session persistence.
- store, storeKind, err := selectSessionBindingStore(redisClient, true)
- if err != nil {
- logger.Warnf(ctx,
- "Per-tenant sandbox config disabled: %v", err)
+ if store == nil {
+ logger.Warnf(ctx, "Per-tenant sandbox config disabled: no session binding store")
return nil
}
resolver, err := sandbox.NewTenantSandboxResolver(sandbox.TenantSandboxResolverDeps{
@@ -167,7 +181,7 @@ func newTenantSandboxResolver(
"(per-tenant sandbox config disabled)", err)
return nil
}
- logger.Infof(ctx, "Tenant sandbox resolver configured: binding=%s", storeKind)
+ logger.Infof(ctx, "Tenant sandbox resolver configured")
return resolver
}
diff --git a/internal/handler/session/background.go b/internal/handler/session/background.go
new file mode 100644
index 000000000..dd47516eb
--- /dev/null
+++ b/internal/handler/session/background.go
@@ -0,0 +1,66 @@
+package session
+
+import (
+ "context"
+
+ "github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+// ackBackgroundNotices acknowledges the background notices a persisted
+// assistant message carries: the ones injected into its turn and the ones its
+// work tools returned. Until the message is saved they stay in the inbox, so a
+// turn that never reached history delivers them again next run. ctx carries
+// the session's tenant, as on every completeAssistantMessage path.
+func (h *Handler) ackBackgroundNotices(ctx context.Context, msg *types.Message) {
+ if h.background == nil || h.background.Inbox == nil || msg == nil {
+ return
+ }
+ ids := backgroundNoticeIDs(msg.AgentSteps)
+ if len(ids) == 0 {
+ return
+ }
+ tenantID, ok := types.TenantIDFromContext(ctx)
+ if !ok || tenantID == 0 {
+ return
+ }
+ ref := interfaces.SessionRef{TenantID: tenantID, SessionID: msg.SessionID}
+ if err := h.background.Inbox.Ack(ctx, ref, ids); err != nil {
+ logger.Warnf(ctx, "ack background notices for message %s failed: %v", msg.ID, err)
+ }
+}
+
+// backgroundNoticeIDs collects every notice ID the steps delivered or showed,
+// once each, in order.
+func backgroundNoticeIDs(steps []types.AgentStep) []string {
+ var ids []string
+ seen := make(map[string]bool)
+ add := func(id string) {
+ if id != "" && !seen[id] {
+ seen[id] = true
+ ids = append(ids, id)
+ }
+ }
+ for _, step := range steps {
+ for _, injection := range step.BackgroundInjections {
+ for _, id := range injection.AckIDs {
+ add(id)
+ }
+ }
+ for _, id := range step.BackgroundSeenIDs {
+ add(id)
+ }
+ }
+ return ids
+}
+
+// notifyBackgroundStop tells background features the user stopped the turn.
+// Each feature decides what that means for its own work.
+func (h *Handler) notifyBackgroundStop(ctx context.Context, session *types.Session) {
+ if h.background == nil || h.background.Listeners == nil || session == nil {
+ return
+ }
+ h.background.Listeners.NotifyStop(context.WithoutCancel(ctx),
+ interfaces.SessionRef{TenantID: session.TenantID, SessionID: session.ID})
+}
diff --git a/internal/handler/session/background_test.go b/internal/handler/session/background_test.go
new file mode 100644
index 000000000..9505b43e0
--- /dev/null
+++ b/internal/handler/session/background_test.go
@@ -0,0 +1,96 @@
+package session
+
+import (
+ "context"
+ "errors"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/application/service"
+ "github.com/Tencent/WeKnora/internal/background"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+type recordingSessionListener struct {
+ stopped []interfaces.SessionRef
+}
+
+func (l *recordingSessionListener) OnStop(_ context.Context, s interfaces.SessionRef) error {
+ l.stopped = append(l.stopped, s)
+ return nil
+}
+
+func (l *recordingSessionListener) OnDelete(context.Context, interfaces.SessionRef) error { return nil }
+
+func newAckFixture(t *testing.T) (*service.BackgroundWork, interfaces.SessionRef) {
+ t.Helper()
+ work := service.NewBackgroundWork(background.NewMemoryActivityRegistry(nil), background.NewMemoryNoticeInbox(nil))
+ ref := interfaces.SessionRef{TenantID: 1, SessionID: "s"}
+ for _, id := range []string{"sj_1", "sj_2", "sj_3"} {
+ _, err := work.Inbox.Push(context.Background(), ref, interfaces.BackgroundNotice{
+ ID: id, Kind: "shell_job", Text: id, FinishedAt: time.Now(),
+ })
+ require.NoError(t, err)
+ }
+ return work, ref
+}
+
+func messageCarryingNotices() *types.Message {
+ return &types.Message{
+ ID: "m", SessionID: "s", Role: "assistant", Content: "done",
+ AgentSteps: []types.AgentStep{
+ {BackgroundInjections: []types.BackgroundInjection{{Content: "x", AckIDs: []string{"sj_1"}}}},
+ {BackgroundSeenIDs: []string{"sj_2", "sj_1"}},
+ },
+ }
+}
+
+func pendingIDs(t *testing.T, work *service.BackgroundWork, ref interfaces.SessionRef) []string {
+ t.Helper()
+ notices, err := work.Inbox.Pending(context.Background(), ref)
+ require.NoError(t, err)
+ ids := make([]string, 0, len(notices))
+ for _, n := range notices {
+ ids = append(ids, n.ID)
+ }
+ return ids
+}
+
+func TestPersistedMessageAcknowledgesTheNoticesItCarries(t *testing.T) {
+ work, ref := newAckFixture(t)
+ h := &Handler{messageService: &imageCompletionMessages{}, background: work}
+ ctx := types.WithExecutionTenant(context.Background(), 1)
+
+ require.NoError(t, h.completeAssistantMessage(ctx, messageCarryingNotices(), "", ""))
+
+ require.ElementsMatch(t, []string{"sj_3"}, pendingIDs(t, work, ref))
+}
+
+func TestFailedPersistLeavesNoticesForTheNextRun(t *testing.T) {
+ work, ref := newAckFixture(t)
+ h := &Handler{messageService: &imageCompletionMessages{err: errors.New("db down")}, background: work}
+ ctx := types.WithExecutionTenant(context.Background(), 1)
+
+ require.Error(t, h.completeAssistantMessage(ctx, messageCarryingNotices(), "", ""))
+
+ require.ElementsMatch(t, []string{"sj_1", "sj_2", "sj_3"}, pendingIDs(t, work, ref))
+}
+
+func TestBackgroundNoticeIDsAreDeduplicatedInOrder(t *testing.T) {
+ require.Equal(t, []string{"sj_1", "sj_2"}, backgroundNoticeIDs(messageCarryingNotices().AgentSteps))
+ require.Empty(t, backgroundNoticeIDs(nil))
+}
+
+func TestStopNotifiesBackgroundListeners(t *testing.T) {
+ work := service.NewBackgroundWork(nil, nil)
+ listener := &recordingSessionListener{}
+ work.Listeners.Register(listener)
+ h := &Handler{background: work}
+
+ h.notifyBackgroundStop(context.Background(), &types.Session{ID: "s", TenantID: 3})
+ (&Handler{}).notifyBackgroundStop(context.Background(), &types.Session{ID: "s", TenantID: 3})
+
+ require.Equal(t, []interfaces.SessionRef{{TenantID: 3, SessionID: "s"}}, listener.stopped)
+}
diff --git a/internal/handler/session/handler.go b/internal/handler/session/handler.go
index e0c0b0fd1..2ff476758 100644
--- a/internal/handler/session/handler.go
+++ b/internal/handler/session/handler.go
@@ -80,6 +80,7 @@ type Handler struct {
suspendedRuns *service.SuspendedRunService
stopCollaboration func(context.Context, string, string) error
delegatedPrompt func(context.Context, *types.AgentPrompt) bool
+ background *service.BackgroundWork
}
// NewHandler creates a new instance of Handler with all necessary dependencies
@@ -116,6 +117,7 @@ func NewHandler(
forkService *service.SessionForkService,
rewindService *service.SessionRewindService,
approvedProjectDirs HostProjectDirsLoader,
+ backgroundWork *service.BackgroundWork,
) *Handler {
h := &Handler{
browserSkill: browserSkill,
@@ -146,6 +148,7 @@ func NewHandler(
desktopLast: desktopLast,
redis: rdb,
approvedProjectDirs: approvedProjectDirs,
+ background: backgroundWork,
attachmentProcessor: NewAttachmentProcessor(
fileService,
documentReader,
diff --git a/internal/handler/session/qa.go b/internal/handler/session/qa.go
index 23d532f8e..74724bdb6 100644
--- a/internal/handler/session/qa.go
+++ b/internal/handler/session/qa.go
@@ -2168,6 +2168,7 @@ func (h *Handler) completeAssistantMessage(
logger.Errorf(ctx, "Failed to persist assistant message %s: %v", assistantMessage.ID, err)
return err
}
+ h.ackBackgroundNotices(ctx, assistantMessage)
// Asynchronously index the Q&A pair into the chat history knowledge base for vector search.
// Use WithoutCancel so the goroutine survives after the HTTP request context is done.
diff --git a/internal/handler/session/stream.go b/internal/handler/session/stream.go
index 4dd2c94d7..9c052cf0b 100644
--- a/internal/handler/session/stream.go
+++ b/internal/handler/session/stream.go
@@ -349,6 +349,7 @@ func (h *Handler) StopSession(c *gin.Context) {
}
logger.Infof(ctx, "Stop event written successfully for session: %s, message: %s", sessionID, assistantMessageID)
+ h.notifyBackgroundStop(ctx, session)
c.JSON(200, gin.H{
"success": true,
"message": "Generation stopped",
diff --git a/internal/modelcontext/tool_policy.go b/internal/modelcontext/tool_policy.go
index f4657cb57..688debcb5 100644
--- a/internal/modelcontext/tool_policy.go
+++ b/internal/modelcontext/tool_policy.go
@@ -222,7 +222,10 @@ var toolHandlePolicies = map[string]toolHandlePolicy{
"execute_skill_script": {},
// Sandbox shell/file tools return whatever a skill script produced, so they
// carry the same unknown-shape caveat as skill output above.
- "shell_exec": {},
+ "shell_exec": {},
+ // shell_exec_async results carry opaque work handles (sj_...) that the
+ // work_wait / work_cancel tools consume; they are not source identities.
+ "shell_exec_async": {},
"list_sandbox_files": {},
"read_sandbox_file": {},
"write_sandbox_file": {},
@@ -231,6 +234,10 @@ var toolHandlePolicies = map[string]toolHandlePolicy{
// caveat: they echo paths and byte counts, never knowledge identifiers.
"write_skill_file": {},
"edit_skill_file": {},
+ // Background-work tools echo work IDs, states and command output; they
+ // never carry knowledge identifiers.
+ "work_wait": {},
+ "work_cancel": {},
}
// HasToolPolicy reports whether a tool has an explicit model-handle policy.
diff --git a/internal/sandbox/activity_registry_memory.go b/internal/sandbox/activity_registry_memory.go
new file mode 100644
index 000000000..9a45d7c2a
--- /dev/null
+++ b/internal/sandbox/activity_registry_memory.go
@@ -0,0 +1,85 @@
+package sandbox
+
+import (
+ "context"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+)
+
+// MemoryActivityRegistry shares the binding store's background leases and lock,
+// so single-process deployments also protect live work from rebuild and rewind.
+type MemoryActivityRegistry struct {
+ store *MemorySessionSandboxBindingStore
+}
+
+var _ interfaces.ActivityRegistry = (*MemoryActivityRegistry)(nil)
+
+// NewMemoryActivityRegistry returns the activity registry of store.
+func NewMemoryActivityRegistry(store *MemorySessionSandboxBindingStore) *MemoryActivityRegistry {
+ return &MemoryActivityRegistry{store: store}
+}
+
+func memoryActivityKey(ctx context.Context, s interfaces.SessionRef) (SessionSandboxKey, error) {
+ key := SessionSandboxKey{TenantID: s.TenantID, SessionID: s.SessionID}
+ if err := key.Validate(); err != nil {
+ return key, err
+ }
+ return key, ctx.Err()
+}
+
+// Add implements interfaces.ActivityRegistry.
+func (r *MemoryActivityRegistry) Add(
+ ctx context.Context, s interfaces.SessionRef, a interfaces.BackgroundActivity, lease time.Duration, kindCap int,
+) error {
+ key, err := memoryActivityKey(ctx, s)
+ if err != nil {
+ return err
+ }
+ r.store.mu.Lock()
+ defer r.store.mu.Unlock()
+ if _, held := r.store.rewinds[key]; held {
+ return ErrSessionRewindLocked
+ }
+ return r.store.activity.Add(ctx, s, a, lease, kindCap)
+}
+
+// Renew implements interfaces.ActivityRegistry.
+func (r *MemoryActivityRegistry) Renew(
+ ctx context.Context, s interfaces.SessionRef, id string, lease time.Duration,
+) (bool, error) {
+ if _, err := memoryActivityKey(ctx, s); err != nil {
+ return false, err
+ }
+ r.store.mu.Lock()
+ defer r.store.mu.Unlock()
+ return r.store.activity.Renew(ctx, s, id, lease)
+}
+
+// Remove implements interfaces.ActivityRegistry.
+func (r *MemoryActivityRegistry) Remove(ctx context.Context, s interfaces.SessionRef, id string) error {
+ if _, err := memoryActivityKey(ctx, s); err != nil {
+ return err
+ }
+ r.store.mu.Lock()
+ defer r.store.mu.Unlock()
+ return r.store.activity.Remove(ctx, s, id)
+}
+
+// List implements interfaces.ActivityRegistry.
+func (r *MemoryActivityRegistry) List(
+ ctx context.Context, s interfaces.SessionRef,
+) ([]interfaces.BackgroundActivity, error) {
+ if _, err := memoryActivityKey(ctx, s); err != nil {
+ return nil, err
+ }
+ r.store.mu.Lock()
+ defer r.store.mu.Unlock()
+ return r.store.activity.List(ctx, s)
+}
+
+// Caller holds s.mu, serializing the lease check with activity registration.
+func (s *MemorySessionSandboxBindingStore) hasSessionActivityLocked(ctx context.Context, key SessionSandboxKey) bool {
+ items, err := s.activity.List(ctx, interfaces.SessionRef{TenantID: key.TenantID, SessionID: key.SessionID})
+ return err != nil || len(items) > 0
+}
diff --git a/internal/sandbox/activity_registry_memory_test.go b/internal/sandbox/activity_registry_memory_test.go
new file mode 100644
index 000000000..4c128e753
--- /dev/null
+++ b/internal/sandbox/activity_registry_memory_test.go
@@ -0,0 +1,100 @@
+package sandbox
+
+import (
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/background"
+ "github.com/Tencent/WeKnora/internal/background/backgroundtest"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+type activityTestClock struct {
+ mu sync.Mutex
+ now time.Time
+}
+
+func (c *activityTestClock) Now() time.Time {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ return c.now
+}
+
+func (c *activityTestClock) advance(d time.Duration) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ c.now = c.now.Add(d)
+}
+
+func TestBindingMemoryActivityRegistryContract(t *testing.T) {
+ backgroundtest.RunActivityRegistryContract(t,
+ func(_ *testing.T) (interfaces.ActivityRegistry, backgroundtest.Advance) {
+ store := NewMemorySessionSandboxBindingStore()
+ clock := &activityTestClock{now: time.Now()}
+ store.activity = background.NewMemoryActivityRegistry(clock)
+ return NewMemoryActivityRegistry(store), clock.advance
+ })
+}
+
+func TestMemoryActivityProtectsStaleSandboxAcrossTurns(t *testing.T) {
+ for _, release := range []string{"remove", "expire"} {
+ t.Run(release, func(t *testing.T) {
+ ctx := t.Context()
+ fx := newLifecycleFixture(t)
+ clock := &activityTestClock{now: time.Now()}
+ fx.bindings.activity = background.NewMemoryActivityRegistry(clock)
+ registry := NewMemoryActivityRegistry(fx.bindings)
+ ref := interfaces.SessionRef{TenantID: fx.key.TenantID, SessionID: fx.key.SessionID}
+ first, err := fx.lifecycle.Resolve(ctx, fx.key)
+ require.NoError(t, err)
+ require.NoError(t, registry.Add(ctx, ref, interfaces.BackgroundActivity{ID: "sj_1"}, time.Minute, 0))
+ _, err = fx.bindings.InvalidateByConfig(ctx, fx.key.TenantID, "cfg-1")
+ require.NoError(t, err)
+
+ active, rebuild, err := fx.bindings.TurnState(ctx, fx.key)
+ require.NoError(t, err)
+ require.True(t, active, "background work stays active after the submitting turn")
+ require.False(t, rebuild)
+ require.NoError(t, fx.bindings.BeginTurn(ctx, fx.key))
+ kept, err := fx.lifecycle.Resolve(ctx, fx.key)
+ require.NoError(t, err)
+ require.Equal(t, first.ID(), kept.ID())
+ require.Empty(t, fx.client.deleteIDs)
+ require.NoError(t, fx.bindings.EndTurn(ctx, fx.key))
+
+ if release == "remove" {
+ require.NoError(t, registry.Remove(ctx, ref, "sj_1"))
+ } else {
+ clock.advance(time.Minute)
+ }
+ require.NoError(t, fx.bindings.BeginTurn(ctx, fx.key))
+ replaced, err := fx.lifecycle.Resolve(ctx, fx.key)
+ require.NoError(t, err)
+ require.NotEqual(t, first.ID(), replaced.ID(), "completed or expired work must release the rebuild guard")
+ require.Equal(t, []string{first.ID()}, fx.client.deleteIDs)
+ require.NoError(t, fx.bindings.EndTurn(ctx, fx.key))
+ })
+ }
+}
+
+func TestMemoryActivityAndRewindAreExclusive(t *testing.T) {
+ ctx := t.Context()
+ store := NewMemorySessionSandboxBindingStore()
+ registry := NewMemoryActivityRegistry(store)
+ key := SessionSandboxKey{TenantID: 42, SessionID: "session-a"}
+ ref := interfaces.SessionRef{TenantID: key.TenantID, SessionID: key.SessionID}
+ item := interfaces.BackgroundActivity{ID: "sj_1"}
+ require.NoError(t, registry.Add(ctx, ref, item, time.Minute, 0))
+ _, err := store.TryLockRewind(ctx, key)
+ require.ErrorIs(t, err, ErrSessionWorkActive)
+ require.NoError(t, registry.Remove(ctx, ref, item.ID))
+ unlock, err := store.TryLockRewind(ctx, key)
+ require.NoError(t, err)
+ defer unlock()
+ require.ErrorIs(t, registry.Add(ctx, ref, item, time.Minute, 0), ErrSessionRewindLocked)
+ items, err := registry.List(ctx, ref)
+ require.NoError(t, err)
+ require.Empty(t, items)
+}
diff --git a/internal/sandbox/activity_registry_redis.go b/internal/sandbox/activity_registry_redis.go
new file mode 100644
index 000000000..033496d92
--- /dev/null
+++ b/internal/sandbox/activity_registry_redis.go
@@ -0,0 +1,297 @@
+package sandbox
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "strings"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/redis/go-redis/v9"
+)
+
+// activityKeyGrace keeps the activity hash around briefly after its last
+// deadline so a read racing the expiry still sees a well-formed key.
+const activityKeyGrace = 60 * time.Second
+
+// activityLiveCountLua defines now_ms() and live_count(key, now_ms): the
+// latter deletes expired fields of the activity hash and returns how many
+// remain. The turn-lease scripts in session_binding_redis.go prepend it so
+// they can refuse a rebuild or a rewind while items are live.
+const activityLiveCountLua = `
+local function now_ms()
+ local t = redis.call('TIME')
+ return tonumber(t[1]) * 1000 + math.floor(tonumber(t[2]) / 1000)
+end
+local function live_count(key, now)
+ local raw = redis.call('HGETALL', key)
+ local live = 0
+ for i = 1, #raw, 2 do
+ local v = cjson.decode(raw[i + 1])
+ if tonumber(v['deadline_ms']) <= now then
+ redis.call('HDEL', key, raw[i])
+ else
+ live = live + 1
+ end
+ end
+ return live
+end
+`
+
+// Every write refreshes the hash until its latest member expires. A shared
+// machine mirrors each session's leases in a machine hash, in the same script
+// and Redis slot, so sibling sessions cannot rebuild underneath them.
+const activityWriteLua = `
+local function save_activity(key, id, value, now, grace)
+ redis.call('HSET', key, id, cjson.encode(value))
+ local raw = redis.call('HGETALL', key)
+ local maxd = now
+ for i = 1, #raw, 2 do
+ local d = tonumber(cjson.decode(raw[i + 1])['deadline_ms'])
+ if d > maxd then maxd = d end
+ end
+ redis.call('PEXPIRE', key, maxd - now + grace)
+end
+`
+
+// KEYS: session activity, machine activity, rewind, machine close, session close.
+// ARGV[1] id, ARGV[2] kind, ARGV[3] title,
+// ARGV[4] started_at_ms, ARGV[5] lease_ms, ARGV[6] kind cap (<=0 unlimited),
+// ARGV[7] key grace ms, ARGV[8] machine member. Returns 1 on success, 0 at the cap.
+var addActivityScript = redis.NewScript(activityLiveCountLua + activityWriteLua + `
+local now = now_ms()
+live_count(KEYS[1], now)
+local deadline = now + tonumber(ARGV[5])
+local existing = redis.call('HGET', KEYS[1], ARGV[1])
+local v
+if existing then
+ v = cjson.decode(existing)
+ v['deadline_ms'] = deadline
+else
+ if redis.call('EXISTS', KEYS[3]) == 1 then
+ return redis.error_reply('ERR_SESSION_REWIND_LOCKED')
+ end
+ if redis.call('EXISTS', KEYS[4]) == 1 or redis.call('EXISTS', KEYS[5]) == 1 then
+ return redis.error_reply('ERR_SCOPE_CLOSED')
+ end
+ local cap = tonumber(ARGV[6])
+ if cap > 0 then
+ local raw = redis.call('HGETALL', KEYS[1])
+ local same = 0
+ for i = 1, #raw, 2 do
+ if cjson.decode(raw[i + 1])['kind'] == ARGV[2] then
+ same = same + 1
+ end
+ end
+ if same >= cap then
+ return 0
+ end
+ end
+ v = {
+ kind = ARGV[2], title = ARGV[3],
+ started_at_ms = tonumber(ARGV[4]), deadline_ms = deadline,
+ }
+end
+save_activity(KEYS[1], ARGV[1], v, now, tonumber(ARGV[7]))
+if KEYS[2] ~= KEYS[1] then
+ live_count(KEYS[2], now)
+ save_activity(KEYS[2], ARGV[8], v, now, tonumber(ARGV[7]))
+end
+return 1
+`)
+
+// KEYS: session activity, machine activity.
+// ARGV: id, lease_ms, key grace ms, machine member.
+// Returns 1 when renewed, 0 when the item is missing or expired.
+var renewActivityScript = redis.NewScript(activityLiveCountLua + activityWriteLua + `
+local now = now_ms()
+live_count(KEYS[1], now)
+local existing = redis.call('HGET', KEYS[1], ARGV[1])
+if not existing then
+ if KEYS[2] ~= KEYS[1] then redis.call('HDEL', KEYS[2], ARGV[4]) end
+ return 0
+end
+local v = cjson.decode(existing)
+v['deadline_ms'] = now + tonumber(ARGV[2])
+save_activity(KEYS[1], ARGV[1], v, now, tonumber(ARGV[3]))
+if KEYS[2] ~= KEYS[1] then
+ live_count(KEYS[2], now)
+ save_activity(KEYS[2], ARGV[4], v, now, tonumber(ARGV[3]))
+end
+return 1
+`)
+
+var removeActivityScript = redis.NewScript(`
+redis.call('HDEL', KEYS[1], ARGV[1])
+if KEYS[2] ~= KEYS[1] then redis.call('HDEL', KEYS[2], ARGV[2]) end
+return 1
+`)
+
+var countActivityScript = redis.NewScript(activityLiveCountLua + `
+return live_count(KEYS[1], now_ms())
+`)
+
+// KEYS[1] activity hash. Returns the live fields as a flat [id, json, ...] list.
+var listActivityScript = redis.NewScript(activityLiveCountLua + `
+live_count(KEYS[1], now_ms())
+return redis.call('HGETALL', KEYS[1])
+`)
+
+type redisActivityValue struct {
+ Kind string `json:"kind"`
+ Title string `json:"title"`
+ StartedAtMS int64 `json:"started_at_ms"`
+ DeadlineMS int64 `json:"deadline_ms"`
+}
+
+// RedisActivityRegistry is the multi-replica ActivityRegistry. It shares the
+// binding store's client and hash tag so the turn-lease scripts can read it.
+type RedisActivityRegistry struct {
+ store *RedisSessionSandboxBindingStore
+ allocations AllocationDirectory
+}
+
+var _ interfaces.ActivityRegistry = (*RedisActivityRegistry)(nil)
+
+// NewRedisActivityRegistry returns a registry backed by store's Redis client.
+func NewRedisActivityRegistry(store *RedisSessionSandboxBindingStore) *RedisActivityRegistry {
+ return &RedisActivityRegistry{store: store}
+}
+
+// WithAllocations uses each session's recorded placement to address shared
+// machine leases. Configure it before using the registry.
+func (r *RedisActivityRegistry) WithAllocations(allocations AllocationDirectory) *RedisActivityRegistry {
+ r.allocations = allocations
+ return r
+}
+
+func (s *RedisSessionSandboxBindingStore) activityKey(key SessionSandboxKey) string {
+ if key.IsUserScoped() && key.SessionID != "" {
+ return s.keyBase(key) + ":session:" + key.SessionID + ":activity"
+ }
+ return s.keyBase(key) + ":activity"
+}
+
+func (r *RedisActivityRegistry) key(ctx context.Context, s interfaces.SessionRef) (SessionSandboxKey, error) {
+ k := SessionSandboxKey{TenantID: s.TenantID, SessionID: s.SessionID}
+ if err := k.Validate(); err != nil {
+ return k, err
+ }
+ if r.allocations != nil {
+ a, found, err := r.allocations.SessionAllocation(ctx, s.SessionID)
+ if err != nil {
+ return k, fmt.Errorf("read background work allocation: %w", err)
+ }
+ if found && a.IsUser() {
+ k.User = a.Machine
+ }
+ }
+ return k, k.Validate()
+}
+
+func (r *RedisActivityRegistry) keys(key SessionSandboxKey) []string {
+ return []string{r.store.activityKey(key), r.store.activityKey(key.Machine())}
+}
+
+func activityMachineMember(key SessionSandboxKey, id string) string {
+ return key.SessionID + ":" + id
+}
+
+// hasBackgroundActivity includes every sibling session on a shared machine.
+func (s *RedisSessionSandboxBindingStore) hasBackgroundActivity(
+ ctx context.Context, key SessionSandboxKey,
+) (bool, error) {
+ n, err := countActivityScript.Run(ctx, s.client, []string{s.activityKey(key.Machine())}).Int64()
+ return n > 0, err
+}
+
+// Add implements interfaces.ActivityRegistry.
+func (r *RedisActivityRegistry) Add(
+ ctx context.Context, s interfaces.SessionRef, a interfaces.BackgroundActivity, lease time.Duration, kindCap int,
+) error {
+ key, err := r.key(ctx, s)
+ if err != nil {
+ return err
+ }
+ keys := append(r.keys(key), r.store.rewindKey(key))
+ if key.IsUserScoped() {
+ base := r.store.keyBase(key)
+ keys = append(keys, base+":machine:closed", base+":session:"+key.SessionID+":closed")
+ } else {
+ keys = append(keys, r.store.rewindKey(key), r.store.rewindKey(key))
+ }
+ ok, err := addActivityScript.Run(ctx, r.store.client, keys,
+ a.ID, a.Kind, a.Title, a.StartedAt.UnixMilli(), lease.Milliseconds(), kindCap,
+ activityKeyGrace.Milliseconds(), activityMachineMember(key, a.ID),
+ ).Int64()
+ if err != nil {
+ if isRewindLockedRedisErr(err) {
+ return ErrSessionRewindLocked
+ }
+ if strings.Contains(err.Error(), "ERR_SCOPE_CLOSED") {
+ return ErrSessionWorkspaceBusy
+ }
+ return fmt.Errorf("add background activity: %w", err)
+ }
+ if ok == 0 {
+ return interfaces.ErrActivityLimit
+ }
+ return nil
+}
+
+// Renew implements interfaces.ActivityRegistry.
+func (r *RedisActivityRegistry) Renew(
+ ctx context.Context, s interfaces.SessionRef, id string, lease time.Duration,
+) (bool, error) {
+ key, err := r.key(ctx, s)
+ if err != nil {
+ return false, err
+ }
+ ok, err := renewActivityScript.Run(ctx, r.store.client, r.keys(key),
+ id, lease.Milliseconds(), activityKeyGrace.Milliseconds(), activityMachineMember(key, id),
+ ).Int64()
+ if err != nil {
+ return false, fmt.Errorf("renew background activity: %w", err)
+ }
+ return ok == 1, nil
+}
+
+// Remove implements interfaces.ActivityRegistry.
+func (r *RedisActivityRegistry) Remove(ctx context.Context, s interfaces.SessionRef, id string) error {
+ key, err := r.key(ctx, s)
+ if err != nil {
+ return err
+ }
+ err = removeActivityScript.Run(ctx, r.store.client, r.keys(key), id, activityMachineMember(key, id)).Err()
+ if err != nil {
+ return fmt.Errorf("remove background activity: %w", err)
+ }
+ return nil
+}
+
+// List implements interfaces.ActivityRegistry.
+func (r *RedisActivityRegistry) List(
+ ctx context.Context, s interfaces.SessionRef,
+) ([]interfaces.BackgroundActivity, error) {
+ key, err := r.key(ctx, s)
+ if err != nil {
+ return nil, err
+ }
+ flat, err := listActivityScript.Run(ctx, r.store.client, []string{r.store.activityKey(key)}).StringSlice()
+ if err != nil {
+ return nil, fmt.Errorf("list background activity: %w", err)
+ }
+ out := make([]interfaces.BackgroundActivity, 0, len(flat)/2)
+ for i := 0; i+1 < len(flat); i += 2 {
+ var v redisActivityValue
+ if err := json.Unmarshal([]byte(flat[i+1]), &v); err != nil {
+ return nil, fmt.Errorf("decode background activity %s: %w", flat[i], err)
+ }
+ out = append(out, interfaces.BackgroundActivity{
+ ID: flat[i], Kind: v.Kind, Title: v.Title,
+ StartedAt: time.UnixMilli(v.StartedAtMS).UTC(),
+ })
+ }
+ return out, nil
+}
diff --git a/internal/sandbox/activity_registry_redis_test.go b/internal/sandbox/activity_registry_redis_test.go
new file mode 100644
index 000000000..e89bce7f4
--- /dev/null
+++ b/internal/sandbox/activity_registry_redis_test.go
@@ -0,0 +1,215 @@
+package sandbox
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/background/backgroundtest"
+ "github.com/Tencent/WeKnora/internal/sandbox/admission"
+ "github.com/Tencent/WeKnora/internal/sandbox/allocation"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/alicebob/miniredis/v2"
+ "github.com/stretchr/testify/require"
+)
+
+func TestRedisActivityRegistryContract(t *testing.T) {
+ backgroundtest.RunActivityRegistryContract(t,
+ func(t *testing.T) (interfaces.ActivityRegistry, backgroundtest.Advance) {
+ store, _, mini := newRedisBindingTestStore(t)
+ now := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC)
+ mini.SetTime(now)
+ advance := func(d time.Duration) {
+ now = now.Add(d)
+ mini.SetTime(now)
+ mini.FastForward(d) // also expires key TTLs
+ }
+ return NewRedisActivityRegistry(store), advance
+ })
+}
+
+func TestRedisActivityRegistryUsesBindingHashTag(t *testing.T) {
+ store, _, _ := newRedisBindingTestStore(t)
+ key := SessionSandboxKey{TenantID: 7, SessionID: "s1"}
+ require.Equal(t, "weknora:sandbox:session:{test:7:s1}:activity", store.activityKey(key))
+ require.Equal(t, "weknora:sandbox:session:{test:7:s1}:turn", store.turnKey(key))
+}
+
+func TestRedisActivityRegistryRejectsInvalidSession(t *testing.T) {
+ store, _, _ := newRedisBindingTestStore(t)
+ r := NewRedisActivityRegistry(store)
+ err := r.Add(t.Context(), interfaces.SessionRef{}, interfaces.BackgroundActivity{ID: "x"}, time.Minute, 0)
+ require.Error(t, err)
+}
+
+// These managers use separate stores on the same Redis, as two replicas do.
+func newSharedActivityManagers(
+ t *testing.T,
+) (*userHarness, *userHarness, *RedisActivityRegistry, *miniredis.Miniredis) {
+ t.Helper()
+ store, client, mini := newRedisBindingTestStore(t)
+ other, err := NewRedisSessionSandboxBindingStore(client, store.namespace)
+ require.NoError(t, err)
+ allocs := newFakeAllocations(userAssignmentFor(userSessionA), userAssignmentFor(userSessionB))
+ remote := newFakeRemoteClient(SandboxTypeCube)
+ h1 := newUserManagerHarnessWithDirectory(t, remote, store,
+ admission.NewRedis(client, "weknora:sandbox:user:"), allocs, store.namespace)
+ h2 := newUserManagerHarnessWithDirectory(t, remote, other,
+ admission.NewRedis(client, "weknora:sandbox:user:"), allocs, store.namespace)
+ return h1, h2, NewRedisActivityRegistry(store).WithAllocations(allocs), mini
+}
+
+func useActivityTestSandbox(t *testing.T, mgr *SessionBoundManager, sessionID string) string {
+ t.Helper()
+ ctx, end, err := mgr.BeginSessionTurn(ownerCtx(), sessionID)
+ require.NoError(t, err)
+ defer end()
+ _, err = mgr.ExecShellCommand(ctx, sessionID, "true", "", time.Second, nil)
+ require.NoError(t, err)
+ id, bound := mgr.BoundSandboxID(ctx, sessionID)
+ require.True(t, bound)
+ return id
+}
+
+func TestRedisActivityProtectsSharedMachineFromSiblingRebuild(t *testing.T) {
+ for _, release := range []string{"remove", "expire"} {
+ t.Run(release, func(t *testing.T) {
+ h1, h2, registry, mini := newSharedActivityManagers(t)
+ now := time.Now()
+ mini.SetTime(now)
+ ctx := ownerCtx()
+ first := useActivityTestSandbox(t, h1.mgr, userSessionA)
+ ref := interfaces.SessionRef{TenantID: 42, SessionID: userSessionA}
+ require.NoError(t, registry.Add(ctx, ref, interfaces.BackgroundActivity{ID: "sj_1"}, time.Minute, 0))
+ n, err := h1.mgr.InvalidateConfigSandboxesForMachines(ctx, "cfg-1",
+ []allocation.UserMachine{userAssignmentFor(userSessionA).Machine})
+ require.NoError(t, err)
+ require.Equal(t, 1, n)
+
+ require.Equal(t, first, useActivityTestSandbox(t, h2.mgr, userSessionB),
+ "another replica's sibling turn must not destroy the background job's machine")
+ require.Equal(t, first, useActivityTestSandbox(t, h2.mgr, userSessionA),
+ "the submitting session's next turn must also keep its machine")
+ require.Empty(t, h1.client.deleteIDs)
+
+ if release == "remove" {
+ require.NoError(t, registry.Remove(ctx, ref, "sj_1"))
+ } else {
+ mini.SetTime(now.Add(time.Minute))
+ mini.FastForward(time.Minute)
+ }
+ require.NotEqual(t, first, useActivityTestSandbox(t, h2.mgr, userSessionB))
+ require.Equal(t, []string{first}, h1.client.deleteIDs)
+ })
+ }
+}
+
+type activityBeforeCloseRegistry struct {
+ admission.Registry
+ beforeClose func()
+}
+
+func (r *activityBeforeCloseRegistry) Close(
+ ctx context.Context, scope admission.Scope, ttl time.Duration,
+) (admission.Barrier, error) {
+ if r.beforeClose != nil {
+ f := r.beforeClose
+ r.beforeClose = nil
+ f()
+ }
+ return r.Registry.Close(ctx, scope, ttl)
+}
+
+func TestSharedRebuildRechecksActivityAfterClosingAdmission(t *testing.T) {
+ h1, h2, registry, _ := newSharedActivityManagers(t)
+ ctx := ownerCtx()
+ first := useActivityTestSandbox(t, h1.mgr, userSessionA)
+ _, err := h1.mgr.InvalidateConfigSandboxesForMachines(ctx, "cfg-1",
+ []allocation.UserMachine{userAssignmentFor(userSessionA).Machine})
+ require.NoError(t, err)
+ ref := interfaces.SessionRef{TenantID: 42, SessionID: userSessionA}
+ raced := false
+ h2.mgr.lifecycle.admission = &activityBeforeCloseRegistry{
+ Registry: h2.mgr.admission,
+ beforeClose: func() {
+ raced = true
+ require.NoError(t, registry.Add(ctx, ref, interfaces.BackgroundActivity{ID: "sj_1"}, time.Minute, 0))
+ },
+ }
+ require.Equal(t, first, useActivityTestSandbox(t, h2.mgr, userSessionB))
+ require.True(t, raced, "register between the optimistic check and the machine barrier")
+ require.Empty(t, h1.client.deleteIDs)
+}
+
+func TestRedisActivityCannotRegisterThroughClosedAdmission(t *testing.T) {
+ for _, scopeKind := range []string{"machine", "session", "rewind"} {
+ t.Run(scopeKind, func(t *testing.T) {
+ h, _, registry, _ := newSharedActivityManagers(t)
+ ctx := ownerCtx()
+ ref := interfaces.SessionRef{TenantID: 42, SessionID: userSessionA}
+ key, err := registry.key(ctx, ref)
+ require.NoError(t, err)
+ want := ErrSessionWorkspaceBusy
+ if scopeKind == "rewind" {
+ unlock, err := registry.store.TryLockRewind(ctx, key)
+ require.NoError(t, err)
+ defer unlock()
+ want = ErrSessionRewindLocked
+ } else {
+ digest := key.User.Digest(registry.store.namespace)
+ scope := admission.MachineScope(digest)
+ if scopeKind == "session" {
+ scope = admission.SessionScope(digest, ref.SessionID)
+ }
+ barrier, err := h.mgr.admission.Close(ctx, scope, time.Minute)
+ require.NoError(t, err)
+ defer barrier.Release()
+ }
+ require.ErrorIs(t, registry.Add(ctx, ref, interfaces.BackgroundActivity{ID: "sj_1"}, time.Minute, 0), want)
+ items, err := registry.List(ctx, ref)
+ require.NoError(t, err)
+ require.Empty(t, items)
+ active, err := registry.store.hasBackgroundActivity(ctx, key)
+ require.NoError(t, err)
+ require.False(t, active)
+ })
+ }
+}
+
+func TestRedisSharedActivityLeasesRemainSessionScoped(t *testing.T) {
+ _, _, registry, mini := newSharedActivityManagers(t)
+ ctx := ownerCtx()
+ now := time.Now()
+ mini.SetTime(now)
+ a := interfaces.SessionRef{TenantID: 42, SessionID: userSessionA}
+ b := interfaces.SessionRef{TenantID: 42, SessionID: userSessionB}
+ key, err := registry.key(ctx, a)
+ require.NoError(t, err)
+ item := interfaces.BackgroundActivity{ID: "sj_1", Kind: "shell_job"}
+ require.NoError(t, registry.Add(ctx, a, item, 10*time.Second, 1))
+ items, err := registry.List(ctx, b)
+ require.NoError(t, err)
+ require.Empty(t, items)
+ // Caps and IDs are session scoped even on the same machine.
+ require.NoError(t, registry.Add(ctx, b, item, 10*time.Second, 1))
+ _, err = registry.store.TryLockRewind(ctx, key)
+ require.ErrorIs(t, err, ErrSessionWorkActive)
+ require.NoError(t, registry.Remove(ctx, a, item.ID))
+ unlock, err := registry.store.TryLockRewind(ctx, key)
+ require.NoError(t, err, "a sibling's work must not block this session's rewind")
+ unlock()
+ mini.SetTime(now.Add(8 * time.Second))
+ mini.FastForward(8 * time.Second)
+ found, err := registry.Renew(ctx, b, item.ID, time.Minute)
+ require.NoError(t, err)
+ require.True(t, found)
+ mini.SetTime(now.Add(20 * time.Second))
+ mini.FastForward(12 * time.Second)
+ active, err := registry.store.hasBackgroundActivity(ctx, key)
+ require.NoError(t, err)
+ require.True(t, active, "renew must extend the shared machine guard too")
+ require.NoError(t, registry.Remove(ctx, b, item.ID))
+ active, err = registry.store.hasBackgroundActivity(ctx, key)
+ require.NoError(t, err)
+ require.False(t, active)
+}
diff --git a/internal/sandbox/cube_remote_client.go b/internal/sandbox/cube_remote_client.go
index 158bfc26a..ed82bae48 100644
--- a/internal/sandbox/cube_remote_client.go
+++ b/internal/sandbox/cube_remote_client.go
@@ -945,6 +945,14 @@ func (c *CubeRemoteClient) Exec(
"cube returned an empty command result", nil,
)
}
+ if request.OnOutput != nil {
+ if sdkResult.Stdout != "" {
+ request.OnOutput("stdout", []byte(sdkResult.Stdout))
+ }
+ if sdkResult.Stderr != "" {
+ request.OnOutput("stderr", []byte(sdkResult.Stderr))
+ }
+ }
return &RemoteExecResult{
Stdout: sdkResult.Stdout,
Stderr: sdkResult.Stderr,
@@ -1553,6 +1561,20 @@ func (c *CubeRemoteClient) StartDesktopTTLRefresh(ctx context.Context, handle Re
})
}
+// KeepAlive moves the sandbox's idle timeout to ttl from now. Cube already
+// counts commands as activity; the explicit call keeps background work from
+// depending on which requests the control plane happens to count.
+func (c *CubeRemoteClient) KeepAlive(ctx context.Context, handle RemoteSandboxHandle, ttl time.Duration) error {
+ sb, err := cubeHandleSandbox("KeepAlive", handle)
+ if err != nil {
+ return err
+ }
+ if ttl <= 0 {
+ return nil
+ }
+ return sb.SetTimeout(ctx, ttl)
+}
+
var (
_ RemoteSandboxClient = (*CubeRemoteClient)(nil)
_ RemoteSnapshotManager = (*CubeRemoteClient)(nil)
diff --git a/internal/sandbox/cube_remote_client_test.go b/internal/sandbox/cube_remote_client_test.go
index 03f9ad524..2096f41ee 100644
--- a/internal/sandbox/cube_remote_client_test.go
+++ b/internal/sandbox/cube_remote_client_test.go
@@ -532,6 +532,30 @@ func TestCubeRemoteClientExecArgvAndShell(t *testing.T) {
require.True(t, IsRemoteInvalidRequest(err))
}
+func TestCubeRemoteClientExecForwardsBufferedOutput(t *testing.T) {
+ mock := newCubeMockServer(t)
+ mock.SetExecutor(func(string, string, []string) (string, string, int) {
+ return "out-line\n", "err-line\n", 0
+ })
+ client := newTestCubeRemoteClient(t, mock)
+ ctx := context.Background()
+ handle, err := client.Create(ctx, RemoteCreateRequest{TemplateID: "template-a"})
+ require.NoError(t, err)
+
+ var got []string
+ result, err := client.Exec(ctx, handle, RemoteExecRequest{
+ Command: "printf '%s' ok",
+ Shell: true,
+ OnOutput: func(stream string, chunk []byte) {
+ got = append(got, stream+":"+string(chunk))
+ },
+ })
+ require.NoError(t, err)
+ require.Equal(t, "out-line\n", result.Stdout)
+ require.Equal(t, "err-line\n", result.Stderr)
+ require.Equal(t, []string{"stdout:out-line\n", "stderr:err-line\n"}, got)
+}
+
func TestCubeRemoteClientExecTimeoutIsKilled(t *testing.T) {
mock := newCubeMockServer(t)
// A slow executor lets the outer request timeout fire before the mock
diff --git a/internal/sandbox/docker_remote_client.go b/internal/sandbox/docker_remote_client.go
index 106b752e5..30d303b79 100644
--- a/internal/sandbox/docker_remote_client.go
+++ b/internal/sandbox/docker_remote_client.go
@@ -725,6 +725,19 @@ func (c *DockerRemoteClient) Exec(
return result, nil
}
+// dockerKeepAliveTimeout bounds the no-op exec KeepAlive runs.
+const dockerKeepAliveTimeout = 10 * time.Second
+
+// KeepAlive runs a no-op exec. dockerExecCommand touches the activity marker on
+// every exec, and the marker's mtime is all the idle sweeper reads, so ttl is
+// not needed here.
+func (c *DockerRemoteClient) KeepAlive(ctx context.Context, handle RemoteSandboxHandle, _ time.Duration) error {
+ _, err := c.Exec(ctx, handle, RemoteExecRequest{
+ Command: "true", Timeout: dockerKeepAliveTimeout, User: DefaultSandboxExecUser,
+ })
+ return err
+}
+
// dockerExecGrace is the slack between the in-container timeout and the
// client-side deadline. It covers the round-trip and the wrapper's own
// teardown so a script killed at the deadline is still reported as Killed
diff --git a/internal/sandbox/e2b_remote_client.go b/internal/sandbox/e2b_remote_client.go
index 6b0ef8be9..94b7ba5a0 100644
--- a/internal/sandbox/e2b_remote_client.go
+++ b/internal/sandbox/e2b_remote_client.go
@@ -1164,7 +1164,8 @@ func (c *E2BRemoteClient) MakeDir(
return e2bInvalidRequest("MakeDir", "path is required", nil)
}
return makeDirTree(dir, func(component string) error {
- return normalizeE2BError("MakeDir", sandbox.Filesystem.MakeDir(ctx, component, e2b.WithFileUser(remoteFileUser(ctx))))
+ err := sandbox.Filesystem.MakeDir(ctx, component, e2b.WithFileUser(remoteFileUser(ctx)))
+ return normalizeE2BError("MakeDir", err)
})
}
@@ -1598,6 +1599,24 @@ func (c *E2BRemoteClient) StartDesktopTTLRefresh(ctx context.Context, handle Rem
})
}
+// KeepAlive moves the sandbox's timeout to ttl from now. E2B does not count
+// commands, file access or URL requests as activity on a running sandbox; only
+// SetTimeout resets it.
+func (c *E2BRemoteClient) KeepAlive(ctx context.Context, handle RemoteSandboxHandle, ttl time.Duration) error {
+ sbx, err := e2bHandleSandbox("KeepAlive", handle)
+ if err != nil {
+ return err
+ }
+ seconds, err := e2bTimeoutSeconds(RemoteTimeoutPolicy{Mode: RemoteTimeoutExplicit, Value: ttl}, 0)
+ if err != nil {
+ return err
+ }
+ if seconds <= 0 {
+ return nil
+ }
+ return sbx.SetTimeoutWithContext(ctx, seconds)
+}
+
var (
_ RemoteSandboxClient = (*E2BRemoteClient)(nil)
_ RemoteSnapshotManager = (*E2BRemoteClient)(nil)
diff --git a/internal/sandbox/keepalive.go b/internal/sandbox/keepalive.go
new file mode 100644
index 000000000..0c9ddb319
--- /dev/null
+++ b/internal/sandbox/keepalive.go
@@ -0,0 +1,102 @@
+package sandbox
+
+import (
+ "context"
+ "fmt"
+ "strings"
+ "time"
+)
+
+// RemoteSandboxKeepAliver refreshes a sandbox's provider idle timer.
+// Background work runs outside chat turns, so nothing else tells the provider
+// the sandbox is still in use. Cube and E2B move their timeout to ttl from
+// now; Docker runs a no-op exec, which refreshes the activity marker its idle
+// sweeper reads.
+type RemoteSandboxKeepAliver interface {
+ KeepAlive(ctx context.Context, handle RemoteSandboxHandle, ttl time.Duration) error
+}
+
+// keepAliverFrom narrows client to keep-alive. It looks through the Langfuse
+// decorator rather than having the decorator forward: every client
+// NewSessionBoundManager holds is wrapped, and a forwarding method would make
+// a client without the capability look like it has one.
+func keepAliverFrom(client RemoteSandboxClient) (RemoteSandboxKeepAliver, bool) {
+ switch c := client.(type) {
+ case nil:
+ return nil, false
+ case *langfuseRemoteClient:
+ return keepAliverFrom(c.inner)
+ case *langfuseSnapshotClient:
+ return keepAliverFrom(c.inner)
+ }
+ keeper, ok := client.(RemoteSandboxKeepAliver)
+ return keeper, ok
+}
+
+var (
+ _ RemoteSandboxKeepAliver = (*CubeRemoteClient)(nil)
+ _ RemoteSandboxKeepAliver = (*E2BRemoteClient)(nil)
+ _ RemoteSandboxKeepAliver = (*DockerRemoteClient)(nil)
+)
+
+// keepAliveRefreshDivisor sets how often a caller refreshes: every TTL/3
+// (shared design §5), so two missed refreshes still land before expiry.
+const keepAliveRefreshDivisor = 3
+
+// KeepAliveSession refreshes the provider idle timer of the sandbox bound to
+// sessionID and returns how long the caller may wait before the next refresh.
+// It is lookup-only: it never provisions, never rebuilds and never connects to
+// anything but the bound sandbox, so a session with nothing bound, or a
+// provider with nothing to refresh, returns (0, nil).
+func (m *SessionBoundManager) KeepAliveSession(ctx context.Context, sessionID string) (time.Duration, error) {
+ return m.keepAliveSession(ctx, sessionID, "")
+}
+
+// KeepAliveSessionExpected is KeepAliveSession for callers that selected a
+// specific sandbox. Lookup is only against expectedSandboxID: a missing
+// binding or a different sandbox returns (0, nil) and never allocates.
+func (m *SessionBoundManager) KeepAliveSessionExpected(
+ ctx context.Context, sessionID, expectedSandboxID string,
+) (time.Duration, error) {
+ return m.keepAliveSession(ctx, sessionID, expectedSandboxID)
+}
+
+func (m *SessionBoundManager) keepAliveSession(
+ ctx context.Context, sessionID, expectedSandboxID string,
+) (time.Duration, error) {
+ if err := m.requireRemoteBackend(); err != nil {
+ return 0, err
+ }
+ keeper, ok := keepAliverFrom(m.client)
+ if !ok {
+ return 0, nil
+ }
+ req, err := buildSessionCreateRequest(m.client.Provider(), m.config)
+ if err != nil {
+ return 0, err
+ }
+ ttl := req.Timeout.Value
+ if ttl <= 0 {
+ return 0, nil
+ }
+ sc, err := m.scope(ctx, sessionID)
+ if err != nil {
+ return 0, err
+ }
+ handle, found, err := m.lookupSessionHandleForKey(ctx, sc.machine(), expectedSandboxID)
+ if err != nil {
+ // A mismatched expected id is "not this sandbox", not a failure to refresh
+ // and not a reason to allocate a replacement.
+ if expectedSandboxID != "" && strings.Contains(err.Error(), "bound sandbox changed") {
+ return 0, nil
+ }
+ return 0, err
+ }
+ if !found {
+ return 0, nil
+ }
+ if err := keeper.KeepAlive(ctx, handle, ttl); err != nil {
+ return 0, fmt.Errorf("sandbox: keep session sandbox alive: %w", err)
+ }
+ return ttl / keepAliveRefreshDivisor, nil
+}
diff --git a/internal/sandbox/keepalive_test.go b/internal/sandbox/keepalive_test.go
new file mode 100644
index 000000000..74db567fb
--- /dev/null
+++ b/internal/sandbox/keepalive_test.go
@@ -0,0 +1,244 @@
+package sandbox
+
+import (
+ "context"
+ "errors"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+type keepAliveCall struct {
+ sandboxID string
+ ttl time.Duration
+}
+
+// keepAliveFakeClient adds the keep-alive capability to the shared fake.
+type keepAliveFakeClient struct {
+ *fakeRemoteClient
+ keepMu sync.Mutex
+ keepCalls []keepAliveCall
+ keepErr error
+}
+
+func (c *keepAliveFakeClient) KeepAlive(_ context.Context, handle RemoteSandboxHandle, ttl time.Duration) error {
+ c.keepMu.Lock()
+ defer c.keepMu.Unlock()
+ id := ""
+ if handle != nil {
+ id = handle.ID()
+ }
+ c.keepCalls = append(c.keepCalls, keepAliveCall{sandboxID: id, ttl: ttl})
+ return c.keepErr
+}
+
+func (c *keepAliveFakeClient) calls() []keepAliveCall {
+ c.keepMu.Lock()
+ defer c.keepMu.Unlock()
+ return append([]keepAliveCall(nil), c.keepCalls...)
+}
+
+func pausingCreateRequest() RemoteCreateRequest {
+ return RemoteCreateRequest{
+ TemplateID: "template-a",
+ Timeout: RemoteTimeoutPolicy{
+ Mode: RemoteTimeoutExplicit, Value: 15 * time.Minute,
+ Action: RemoteOnTimeoutPause, AutoResume: true,
+ },
+ }
+}
+
+func TestCubeKeepAliveRefreshesTimeout(t *testing.T) {
+ mock := newCubeMockServer(t)
+ client := newTestCubeRemoteClient(t, mock)
+ handle, err := client.Create(context.Background(), pausingCreateRequest())
+ require.NoError(t, err)
+
+ before := mock.timeoutPOSTs.Load()
+ require.NoError(t, client.KeepAlive(context.Background(), handle, 30*time.Minute))
+ require.Equal(t, before+1, mock.timeoutPOSTs.Load())
+}
+
+func TestE2BKeepAliveRefreshesTimeout(t *testing.T) {
+ mock := newE2BMockServer(t)
+ client := newTestE2BRemoteClient(t, mock)
+ handle, err := client.Create(context.Background(), pausingCreateRequest())
+ require.NoError(t, err)
+
+ before := mock.timeoutPOSTs.Load()
+ require.NoError(t, client.KeepAlive(context.Background(), handle, 5*time.Minute))
+ require.Equal(t, before+1, mock.timeoutPOSTs.Load())
+}
+
+func TestDockerKeepAliveRunsANoOpExecThatTouchesTheActivityMarker(t *testing.T) {
+ engine := newFakeDockerEngine()
+ docker := newTestDockerClient(t, engine)
+
+ require.NoError(t, docker.KeepAlive(context.Background(), testHandle("container-1"), time.Minute))
+
+ require.Len(t, engine.execOptions, 1)
+ cmd := engine.execOptions[0].Cmd
+ require.Contains(t, cmd[2], dockerActivityMarker)
+ require.Equal(t, []string{"weknora-exec", "true"}, cmd[3:])
+}
+
+func TestKeepAliverFromSeesThroughLangfuse(t *testing.T) {
+ inner := &keepAliveFakeClient{fakeRemoteClient: newFakeRemoteClient(SandboxTypeCube)}
+ keeper, ok := keepAliverFrom(wrapLangfuseRemoteClient(inner))
+ require.True(t, ok)
+ require.NoError(t, keeper.KeepAlive(context.Background(), nil, time.Minute))
+ require.Equal(t, []keepAliveCall{{ttl: time.Minute}}, inner.calls())
+}
+
+func TestKeepAliverFromRejectsClientWithoutTheCapability(t *testing.T) {
+ _, ok := keepAliverFrom(wrapLangfuseRemoteClient(newFakeRemoteClient(SandboxTypeCube)))
+ require.False(t, ok, "wrapping must not invent keep-alive support")
+ _, ok = keepAliverFrom(nil)
+ require.False(t, ok)
+}
+
+const keepAliveTestTTL = 12 * time.Minute
+
+func newKeepAliveTestManager(t *testing.T, client RemoteSandboxClient) *SessionBoundManager {
+ t.Helper()
+ cfg := DefaultConfig()
+ cfg.CubeTemplate = "tpl-test"
+ cfg.CubeSandboxTTL = keepAliveTestTTL
+ mgr, err := NewSessionBoundManager(SessionBoundManagerConfig{
+ Config: cfg,
+ Client: client,
+ Store: NewMemorySessionSandboxBindingStore(),
+ Checker: &fakeSessionExistenceChecker{exists: true},
+ SkipHealthProbe: true,
+ })
+ require.NoError(t, err)
+ return mgr
+}
+
+func keepAliveTestCtx() context.Context {
+ return context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7))
+}
+
+func bindKeepAliveTestSession(t *testing.T, mgr *SessionBoundManager, sessionID string) string {
+ t.Helper()
+ ctx := keepAliveTestCtx()
+ _, err := mgr.Execute(ctx, &ExecuteConfig{
+ SessionID: sessionID, SkipValidation: true, ScriptContent: "print('ok')\n", Script: "hello.py",
+ })
+ require.NoError(t, err)
+ id, ok := mgr.BoundSandboxID(ctx, sessionID)
+ require.True(t, ok)
+ return id
+}
+
+func TestKeepAliveSessionRefreshesTheBoundSandbox(t *testing.T) {
+ client := &keepAliveFakeClient{fakeRemoteClient: newFakeRemoteClient(SandboxTypeCube)}
+ mgr := newKeepAliveTestManager(t, client)
+ sandboxID := bindKeepAliveTestSession(t, mgr, "s1")
+
+ next, err := mgr.KeepAliveSession(keepAliveTestCtx(), "s1")
+
+ require.NoError(t, err)
+ require.Equal(t, keepAliveTestTTL/3, next)
+ require.Equal(t, []keepAliveCall{{sandboxID: sandboxID, ttl: keepAliveTestTTL}}, client.calls())
+}
+
+func TestKeepAliveSessionNeverProvisions(t *testing.T) {
+ client := &keepAliveFakeClient{fakeRemoteClient: newFakeRemoteClient(SandboxTypeCube)}
+ mgr := newKeepAliveTestManager(t, client)
+
+ next, err := mgr.KeepAliveSession(keepAliveTestCtx(), "unbound")
+
+ require.NoError(t, err)
+ require.Zero(t, next)
+ require.Empty(t, client.calls())
+ creates, _, _, _, _ := client.counts()
+ require.Zero(t, creates, "keep-alive must never create a sandbox")
+}
+
+func TestKeepAliveSessionWithoutProviderSupportIsNoOp(t *testing.T) {
+ client := newFakeRemoteClient(SandboxTypeCube)
+ mgr := newKeepAliveTestManager(t, client)
+ bindKeepAliveTestSession(t, mgr, "s1")
+ _, connectsBefore, _, _, _ := client.counts()
+
+ next, err := mgr.KeepAliveSession(keepAliveTestCtx(), "s1")
+
+ require.NoError(t, err)
+ require.Zero(t, next)
+ _, connectsAfter, _, _, _ := client.counts()
+ require.Equal(t, connectsBefore, connectsAfter, "nothing to refresh, so nothing to connect to")
+}
+
+func TestKeepAliveSessionExpectedRefreshesOnlyTheMatchingSandbox(t *testing.T) {
+ client := &keepAliveFakeClient{fakeRemoteClient: newFakeRemoteClient(SandboxTypeCube)}
+ mgr := newKeepAliveTestManager(t, client)
+ sandboxID := bindKeepAliveTestSession(t, mgr, "s1")
+
+ next, err := mgr.KeepAliveSessionExpected(keepAliveTestCtx(), "s1", sandboxID)
+
+ require.NoError(t, err)
+ require.Equal(t, keepAliveTestTTL/3, next)
+ require.Equal(t, []keepAliveCall{{sandboxID: sandboxID, ttl: keepAliveTestTTL}}, client.calls())
+}
+
+func TestKeepAliveSessionExpectedDoesNotRefreshADifferentSandbox(t *testing.T) {
+ client := &keepAliveFakeClient{fakeRemoteClient: newFakeRemoteClient(SandboxTypeCube)}
+ mgr := newKeepAliveTestManager(t, client)
+ bindKeepAliveTestSession(t, mgr, "s1")
+
+ next, err := mgr.KeepAliveSessionExpected(keepAliveTestCtx(), "s1", "sbx-other")
+
+ require.NoError(t, err)
+ require.Zero(t, next)
+ require.Empty(t, client.calls(), "a different sandbox is not refreshed and not replaced")
+}
+
+func TestKeepAliveSessionRefreshesSharedUserMachine(t *testing.T) {
+ for _, expected := range []bool{false, true} {
+ t.Run(map[bool]string{false: "session", true: "expected"}[expected], func(t *testing.T) {
+ h := newUserManagerHarness(t, userAssignmentFor(userSessionA))
+ ctx := ownerCtx()
+ _, err := h.mgr.ExecShellCommand(ctx, userSessionA, "true", "", time.Second, nil)
+ require.NoError(t, err)
+ id, found := h.mgr.BoundSandboxID(ctx, userSessionA)
+ require.True(t, found)
+ keeper := &keepAliveFakeClient{fakeRemoteClient: h.client}
+ h.mgr.client = wrapLangfuseRemoteClient(keeper)
+ creates, _, _, _, _ := keeper.counts()
+
+ var next time.Duration
+ if expected {
+ next, err = h.mgr.KeepAliveSessionExpected(ctx, userSessionA, id)
+ } else {
+ next, err = h.mgr.KeepAliveSession(ctx, userSessionA)
+ }
+ require.NoError(t, err)
+ require.Positive(t, next)
+ require.Equal(t, []keepAliveCall{{sandboxID: id, ttl: next * keepAliveRefreshDivisor}}, keeper.calls())
+ after, _, _, _, _ := keeper.counts()
+ require.Equal(t, creates, after, "keeping a shared machine alive must not create a session sandbox")
+
+ next, err = h.mgr.KeepAliveSessionExpected(ctx, userSessionA, "another-machine")
+ require.NoError(t, err)
+ require.Zero(t, next)
+ require.Len(t, keeper.calls(), 1, "a mismatched shared binding must not be refreshed")
+ })
+ }
+}
+
+func TestKeepAliveSessionReportsProviderFailure(t *testing.T) {
+ client := &keepAliveFakeClient{fakeRemoteClient: newFakeRemoteClient(SandboxTypeCube)}
+ client.keepErr = errors.New("provider down")
+ mgr := newKeepAliveTestManager(t, client)
+ bindKeepAliveTestSession(t, mgr, "s1")
+
+ next, err := mgr.KeepAliveSession(keepAliveTestCtx(), "s1")
+
+ require.ErrorContains(t, err, "provider down")
+ require.Zero(t, next)
+}
diff --git a/internal/sandbox/sandbox.go b/internal/sandbox/sandbox.go
index 2b8a7314a..2060ff651 100644
--- a/internal/sandbox/sandbox.go
+++ b/internal/sandbox/sandbox.go
@@ -179,6 +179,9 @@ var (
// ErrSessionTurnActive means a chat turn already holds the session lease,
// so rewind must not take the exclusive lock or reset the workspace.
ErrSessionTurnActive = errors.New("sandbox: session turn is active")
+ // ErrSessionWorkActive means background work is still live in the session,
+ // so rewind must not take the exclusive lock or reset the workspace.
+ ErrSessionWorkActive = errors.New("sandbox: session has live background work")
// ErrTerminalUnsupported is returned when the active backend cannot
// stream PTYs (Docker, a disabled manager). Distinct from
// ErrNoLiveSessionSandbox: the session may well have a live sandbox,
diff --git a/internal/sandbox/session_binding.go b/internal/sandbox/session_binding.go
index cb4cddf21..0f5f2b6f8 100644
--- a/internal/sandbox/session_binding.go
+++ b/internal/sandbox/session_binding.go
@@ -9,6 +9,7 @@ import (
"time"
"unicode"
+ "github.com/Tencent/WeKnora/internal/background"
"github.com/Tencent/WeKnora/internal/sandbox/allocation"
"github.com/Tencent/WeKnora/internal/sandbox/workspace"
)
@@ -369,6 +370,7 @@ type MemorySessionSandboxBindingStore struct {
locks map[SessionSandboxKey]*memoryLifecycleLock
turns map[SessionSandboxKey]*memoryTurnLease
rewinds map[SessionSandboxKey]struct{}
+ activity *background.MemoryActivityRegistry
}
// NewMemorySessionSandboxBindingStore creates an empty in-memory store.
@@ -378,6 +380,7 @@ func NewMemorySessionSandboxBindingStore() *MemorySessionSandboxBindingStore {
locks: make(map[SessionSandboxKey]*memoryLifecycleLock),
turns: make(map[SessionSandboxKey]*memoryTurnLease),
rewinds: make(map[SessionSandboxKey]struct{}),
+ activity: background.NewMemoryActivityRegistry(nil),
}
}
@@ -632,7 +635,7 @@ func (s *MemorySessionSandboxBindingStore) BeginTurn(
s.turns[key] = lease
}
if lease.refs == 0 {
- lease.rebuildOnce = true
+ lease.rebuildOnce = !s.hasSessionActivityLocked(ctx, key)
}
lease.refs++
return nil
@@ -679,6 +682,9 @@ func (s *MemorySessionSandboxBindingStore) TryLockRewind(
if lease := s.turns[key]; lease != nil && lease.refs > 0 {
return nil, ErrSessionTurnActive
}
+ if s.hasSessionActivityLocked(ctx, key) {
+ return nil, ErrSessionWorkActive
+ }
if _, held := s.rewinds[key]; held {
return nil, ErrSessionRewindLocked
}
@@ -721,6 +727,9 @@ func (s *MemorySessionSandboxBindingStore) TurnState(
}
s.mu.Lock()
defer s.mu.Unlock()
+ if s.hasSessionActivityLocked(ctx, key) {
+ return true, false, nil
+ }
lease := s.turns[key]
if lease == nil || lease.refs <= 0 {
return false, false, nil
diff --git a/internal/sandbox/session_binding_redis.go b/internal/sandbox/session_binding_redis.go
index 26e66d209..924fcd1e1 100644
--- a/internal/sandbox/session_binding_redis.go
+++ b/internal/sandbox/session_binding_redis.go
@@ -5,7 +5,6 @@ import (
"encoding/json"
"errors"
"fmt"
- "strconv"
"strings"
"sync"
"time"
@@ -78,23 +77,48 @@ return 1
// image, which is what we want once no turn is actually using the sandbox.
const sessionTurnLeaseTTL = 30 * time.Minute
-var beginTurnScript = redis.NewScript(`
+// KEYS[1] turn lease, KEYS[2] rewind lock, KEYS[3] background activity.
+// ARGV[1] TTL ms. A turn that opens while background work is live does not
+// arm its one rebuild: replacing the sandbox would destroy that work.
+var beginTurnScript = redis.NewScript(activityLiveCountLua + `
if redis.call('EXISTS', KEYS[2]) == 1 then
return redis.error_reply('ERR_SESSION_REWIND_LOCKED')
end
local refs = redis.call('HINCRBY', KEYS[1], 'refs', 1)
if refs == 1 then
- redis.call('HSET', KEYS[1], 'rebuild', '1')
+ if live_count(KEYS[3], now_ms()) > 0 then
+ redis.call('HSET', KEYS[1], 'rebuild', '0')
+ else
+ redis.call('HSET', KEYS[1], 'rebuild', '1')
+ end
end
redis.call('PEXPIRE', KEYS[1], ARGV[1])
return refs
`)
-// KEYS[1] turn lease, KEYS[2] rewind lock. ARGV[1] owner token, ARGV[2] TTL ms.
-var tryLockRewindScript = redis.NewScript(`
+// KEYS[1] turn lease, KEYS[2] background activity. ARGV[1] turn TTL ms.
+// Returns {refs, rebuild, live background items}.
+var turnStateScript = redis.NewScript(activityLiveCountLua + `
+local live = live_count(KEYS[2], now_ms())
+local refs = 0
+local rebuild = '0'
+if redis.call('EXISTS', KEYS[1]) == 1 then
+ redis.call('PEXPIRE', KEYS[1], ARGV[1])
+ refs = tonumber(redis.call('HGET', KEYS[1], 'refs') or '0') or 0
+ rebuild = redis.call('HGET', KEYS[1], 'rebuild') or '0'
+end
+return {refs, rebuild, live}
+`)
+
+// KEYS[1] turn lease, KEYS[2] rewind lock, KEYS[3] background activity.
+// ARGV[1] owner token, ARGV[2] TTL ms.
+var tryLockRewindScript = redis.NewScript(activityLiveCountLua + `
if redis.call('EXISTS', KEYS[1]) == 1 then
return redis.error_reply('ERR_SESSION_TURN_ACTIVE')
end
+if live_count(KEYS[3], now_ms()) > 0 then
+ return redis.error_reply('ERR_SESSION_WORK_ACTIVE')
+end
if redis.call('EXISTS', KEYS[2]) == 1 then
return 0
end
@@ -395,7 +419,8 @@ func (s *RedisSessionSandboxBindingStore) BeginTurn(
if ttlMS <= 0 {
ttlMS = (30 * time.Minute).Milliseconds()
}
- if err := beginTurnScript.Run(ctx, s.client, []string{s.turnKey(key), s.rewindKey(key)}, ttlMS).Err(); err != nil {
+ keys := []string{s.turnKey(key), s.rewindKey(key), s.activityKey(key)}
+ if err := beginTurnScript.Run(ctx, s.client, keys, ttlMS).Err(); err != nil {
if isRewindLockedRedisErr(err) {
return ErrSessionRewindLocked
}
@@ -420,7 +445,9 @@ func (s *RedisSessionSandboxBindingStore) EndTurn(
}
// TurnState reports whether a chat turn is open and whether its first
-// resolve may still rebuild a stale sandbox.
+// resolve may still rebuild a stale sandbox. Live background work counts as an
+// open turn that may not rebuild: it runs outside any turn, and a resolve that
+// replaced the sandbox under it would destroy it.
func (s *RedisSessionSandboxBindingStore) TurnState(
ctx context.Context,
key SessionSandboxKey,
@@ -428,19 +455,25 @@ func (s *RedisSessionSandboxBindingStore) TurnState(
if err := key.Validate(); err != nil {
return false, false, err
}
- values, err := s.client.HGetAll(ctx, s.turnKey(key)).Result()
+ values, err := turnStateScript.Run(
+ ctx, s.client, []string{s.turnKey(key), s.activityKey(key)}, sessionTurnLeaseTTL.Milliseconds(),
+ ).Slice()
if err != nil {
return false, false, fmt.Errorf("read sandbox turn lease: %w", err)
}
- if len(values) == 0 {
- return false, false, nil
+ if len(values) != 3 {
+ return false, false, fmt.Errorf("read sandbox turn lease: unexpected reply %v", values)
+ }
+ refs, _ := values[0].(int64)
+ rebuild, _ := values[1].(string)
+ live, _ := values[2].(int64)
+ if live > 0 {
+ return true, false, nil
}
- _ = s.client.PExpire(ctx, s.turnKey(key), sessionTurnLeaseTTL).Err()
- refs, _ := strconv.Atoi(values["refs"])
if refs <= 0 {
return false, false, nil
}
- return true, values["rebuild"] == "1", nil
+ return true, rebuild == "1", nil
}
// ConsumeTurnRebuild spends the one rebuild allowed for the current turn.
@@ -493,12 +526,15 @@ func (s *RedisSessionSandboxBindingStore) TryLockRewind(
ttlMS = sessionRewindLockTTL.Milliseconds()
}
acquired, err := tryLockRewindScript.Run(
- ctx, s.client, []string{s.turnKey(key), s.rewindKey(key)}, token, ttlMS,
+ ctx, s.client, []string{s.turnKey(key), s.rewindKey(key), s.activityKey(key)}, token, ttlMS,
).Int64()
if err != nil {
if isTurnActiveRedisErr(err) {
return nil, ErrSessionTurnActive
}
+ if isWorkActiveRedisErr(err) {
+ return nil, ErrSessionWorkActive
+ }
return nil, fmt.Errorf("lock session rewind: %w", err)
}
if acquired == 0 {
@@ -543,6 +579,10 @@ func isTurnActiveRedisErr(err error) bool {
return err != nil && strings.Contains(err.Error(), "ERR_SESSION_TURN_ACTIVE")
}
+func isWorkActiveRedisErr(err error) bool {
+ return err != nil && strings.Contains(err.Error(), "ERR_SESSION_WORK_ACTIVE")
+}
+
func (s *RedisSessionSandboxBindingStore) rewindLease() time.Duration {
if s != nil && s.rewindLockTTL > 0 {
return s.rewindLockTTL
diff --git a/internal/sandbox/session_binding_redis_activity_test.go b/internal/sandbox/session_binding_redis_activity_test.go
new file mode 100644
index 000000000..93d411b6f
--- /dev/null
+++ b/internal/sandbox/session_binding_redis_activity_test.go
@@ -0,0 +1,87 @@
+package sandbox
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+// addLiveActivity registers one background item for key through the real
+// Redis registry, so the turn-lease scripts read exactly what features write.
+func addLiveActivity(t *testing.T, store *RedisSessionSandboxBindingStore, key SessionSandboxKey, lease time.Duration) {
+ t.Helper()
+ reg := NewRedisActivityRegistry(store)
+ ref := interfaces.SessionRef{TenantID: key.TenantID, SessionID: key.SessionID}
+ require.NoError(t, reg.Add(context.Background(), ref, interfaces.BackgroundActivity{
+ ID: "sj_1", Kind: "shell_job", Title: "build", StartedAt: time.Now(),
+ }, lease, 0))
+}
+
+func TestRedisTurnStateCountsLiveBackgroundWorkAsActiveWithoutRebuild(t *testing.T) {
+ store, _, mini := newRedisBindingTestStore(t)
+ ctx := context.Background()
+ key := SessionSandboxKey{TenantID: 42, SessionID: "session-bg"}
+ now := time.Date(2026, 10, 15, 10, 0, 0, 0, time.UTC)
+ mini.SetTime(now)
+
+ addLiveActivity(t, store, key, 30*time.Second)
+
+ active, rebuildOnce, err := store.TurnState(ctx, key)
+ require.NoError(t, err)
+ require.True(t, active, "live background work must keep a turn-less resolve from rebuilding")
+ require.False(t, rebuildOnce)
+
+ mini.SetTime(now.Add(31 * time.Second))
+ active, rebuildOnce, err = store.TurnState(ctx, key)
+ require.NoError(t, err)
+ require.False(t, active, "an expired lease no longer protects the sandbox")
+ require.False(t, rebuildOnce)
+}
+
+func TestRedisBeginTurnDoesNotArmRebuildWhileBackgroundWorkIsLive(t *testing.T) {
+ store, _, mini := newRedisBindingTestStore(t)
+ ctx := context.Background()
+ key := SessionSandboxKey{TenantID: 42, SessionID: "session-bg-turn"}
+ now := time.Date(2026, 10, 15, 10, 0, 0, 0, time.UTC)
+ mini.SetTime(now)
+
+ addLiveActivity(t, store, key, 30*time.Second)
+ require.NoError(t, store.BeginTurn(ctx, key))
+
+ active, rebuildOnce, err := store.TurnState(ctx, key)
+ require.NoError(t, err)
+ require.True(t, active)
+ require.False(t, rebuildOnce)
+
+ // The work ends mid-turn. The turn still must not rebuild: its first
+ // resolve already ran against this sandbox.
+ mini.SetTime(now.Add(31 * time.Second))
+ active, rebuildOnce, err = store.TurnState(ctx, key)
+ require.NoError(t, err)
+ require.True(t, active)
+ require.False(t, rebuildOnce)
+ require.NoError(t, store.EndTurn(ctx, key))
+}
+
+func TestRedisTryLockRewindFailsWhileBackgroundWorkIsLive(t *testing.T) {
+ store, _, mini := newRedisBindingTestStore(t)
+ ctx := context.Background()
+ key := SessionSandboxKey{TenantID: 42, SessionID: "session-bg-rewind"}
+ now := time.Date(2026, 10, 15, 10, 0, 0, 0, time.UTC)
+ mini.SetTime(now)
+
+ addLiveActivity(t, store, key, 30*time.Second)
+ _, err := store.TryLockRewind(ctx, key)
+ require.ErrorIs(t, err, ErrSessionWorkActive)
+ held, err := store.HasRewindLock(ctx, key)
+ require.NoError(t, err)
+ require.False(t, held, "a refused rewind must not leave its lock behind")
+
+ mini.SetTime(now.Add(31 * time.Second))
+ unlock, err := store.TryLockRewind(ctx, key)
+ require.NoError(t, err)
+ unlock()
+}
diff --git a/internal/sandbox/session_lifecycle.go b/internal/sandbox/session_lifecycle.go
index 09925ace4..53ea83f16 100644
--- a/internal/sandbox/session_lifecycle.go
+++ b/internal/sandbox/session_lifecycle.go
@@ -320,6 +320,12 @@ func (l *remoteSessionLifecycle) rebuildStale(
if err != nil || others > 0 {
return false, nil
}
+ // A sibling may have registered background work after the first check
+ // and ended its turn before we closed admission. Registration is now
+ // blocked, so recheck the machine's leases before destroying it.
+ if l.hasLiveMachineActivity(guarded, key) {
+ return false, nil
+ }
if err := barrier.Check(); err != nil {
return false, err
}
@@ -1161,6 +1167,9 @@ func (l *remoteSessionLifecycle) shouldRebuildStaleBinding(
if !key.IsUserScoped() {
return true
}
+ if l.hasLiveMachineActivity(ctx, key) {
+ return false
+ }
others, err := l.admission.ActiveExcept(ctx, admission.MachineScope(l.digest(key)), admissionExemptFrom(ctx)...)
if err != nil {
log.Printf("[sandbox] admission of %s unavailable (%v); keeping the current sandbox", key, err)
@@ -1169,6 +1178,21 @@ func (l *remoteSessionLifecycle) shouldRebuildStaleBinding(
return others == 0
}
+func (l *remoteSessionLifecycle) hasLiveMachineActivity(ctx context.Context, key SessionSandboxKey) bool {
+ store, ok := l.bindings.(interface {
+ hasBackgroundActivity(context.Context, SessionSandboxKey) (bool, error)
+ })
+ if !ok {
+ return false
+ }
+ active, err := store.hasBackgroundActivity(ctx, key)
+ if err != nil {
+ log.Printf("[sandbox] background activity of %s unavailable (%v); keeping the current sandbox", key, err)
+ return true
+ }
+ return active
+}
+
func (l *remoteSessionLifecycle) consumeTurnRebuild(
ctx context.Context,
key SessionSandboxKey,
diff --git a/internal/stream/notice_inbox_redis.go b/internal/stream/notice_inbox_redis.go
new file mode 100644
index 000000000..dbcd0da9f
--- /dev/null
+++ b/internal/stream/notice_inbox_redis.go
@@ -0,0 +1,156 @@
+package stream
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "sort"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/redis/go-redis/v9"
+)
+
+// noticeKeyTTL is a backstop so an abandoned session's inbox disappears.
+const noticeKeyTTL = interfaces.NoticeRetention + time.Hour
+
+// pushNoticeScript writes a notice and refreshes the key TTL in one round
+// trip. As two commands, a failed EXPIRE after a successful HSETNX would return
+// an error for a notice that is already stored; the caller's retry would then
+// see first=false and work_finished would never be emitted.
+//
+// KEYS[1] inbox key; ARGV[1] id, ARGV[2] encoded notice, ARGV[3] TTL in ms.
+var pushNoticeScript = redis.NewScript(`
+local first = redis.call('HSETNX', KEYS[1], ARGV[1], ARGV[2])
+redis.call('PEXPIRE', KEYS[1], ARGV[3])
+return first
+`)
+
+// dropNoticesScript deletes each field only if it still holds the value the
+// caller read. Deleting by ID alone could remove a notice that was acked and
+// pushed again between the caller's read and this delete.
+//
+// KEYS[1] inbox key; ARGV pairs of id, encoded notice as read.
+var dropNoticesScript = redis.NewScript(`
+local n = 0
+for i = 1, #ARGV, 2 do
+ if redis.call('HGET', KEYS[1], ARGV[i]) == ARGV[i + 1] then
+ n = n + redis.call('HDEL', KEYS[1], ARGV[i])
+ end
+end
+return n
+`)
+
+type redisNoticeValue struct {
+ Kind string `json:"kind"`
+ Text string `json:"text"`
+ FinishedAtMS int64 `json:"finished_at_ms"`
+}
+
+// RedisNoticeInbox is the multi-replica NoticeInbox. It lives next to the
+// steer lists and uses the same key prefix.
+type RedisNoticeInbox struct {
+ mgr *RedisStreamManager
+ clock func() time.Time
+}
+
+var _ interfaces.NoticeInbox = (*RedisNoticeInbox)(nil)
+
+// NewRedisNoticeInbox returns an inbox on m's Redis client. A nil clock means time.Now.
+func NewRedisNoticeInbox(m *RedisStreamManager, clock func() time.Time) *RedisNoticeInbox {
+ if clock == nil {
+ clock = time.Now
+ }
+ return &RedisNoticeInbox{mgr: m, clock: clock}
+}
+
+func (b *RedisNoticeInbox) key(s interfaces.SessionRef) string {
+ return fmt.Sprintf("%s:%s:bg-inbox", b.mgr.prefix, s.SessionID)
+}
+
+// Push implements interfaces.NoticeInbox. An error means the notice was not
+// stored. The one exception is a reply lost after the script ran: the retry
+// then sees first=false, and the UI's background-work polling catches up.
+func (b *RedisNoticeInbox) Push(
+ ctx context.Context, s interfaces.SessionRef, n interfaces.BackgroundNotice,
+) (bool, error) {
+ raw, err := json.Marshal(redisNoticeValue{
+ Kind: n.Kind, Text: n.Text, FinishedAtMS: n.FinishedAt.UnixMilli(),
+ })
+ if err != nil {
+ return false, fmt.Errorf("encode background notice: %w", err)
+ }
+ first, err := pushNoticeScript.Run(ctx, b.mgr.client, []string{b.key(s)},
+ n.ID, raw, noticeKeyTTL.Milliseconds()).Int()
+ if err != nil {
+ return false, fmt.Errorf("push background notice: %w", err)
+ }
+ return first == 1, nil
+}
+
+// Pending implements interfaces.NoticeInbox.
+func (b *RedisNoticeInbox) Pending(
+ ctx context.Context, s interfaces.SessionRef,
+) ([]interfaces.BackgroundNotice, error) {
+ key := b.key(s)
+ all, err := b.mgr.client.HGetAll(ctx, key).Result()
+ if err != nil {
+ return nil, fmt.Errorf("read background notices: %w", err)
+ }
+ cutoff := b.clock().Add(-interfaces.NoticeRetention)
+ out := make([]interfaces.BackgroundNotice, 0, len(all))
+ expired := make(map[string]string)
+ for id, raw := range all {
+ var v redisNoticeValue
+ if err := json.Unmarshal([]byte(raw), &v); err != nil {
+ return nil, fmt.Errorf("decode background notice %s: %w", id, err)
+ }
+ finished := time.UnixMilli(v.FinishedAtMS).UTC()
+ if finished.Before(cutoff) {
+ expired[id] = raw
+ continue
+ }
+ out = append(out, interfaces.BackgroundNotice{ID: id, Kind: v.Kind, Text: v.Text, FinishedAt: finished})
+ }
+ if err := b.dropIfUnchanged(ctx, key, expired); err != nil {
+ return nil, fmt.Errorf("drop expired background notices: %w", err)
+ }
+ sort.Slice(out, func(i, j int) bool {
+ if out[i].FinishedAt.Equal(out[j].FinishedAt) {
+ return out[i].ID < out[j].ID
+ }
+ return out[i].FinishedAt.Before(out[j].FinishedAt)
+ })
+ return out, nil
+}
+
+// dropIfUnchanged deletes each id whose stored value still equals read[id].
+func (b *RedisNoticeInbox) dropIfUnchanged(ctx context.Context, key string, read map[string]string) error {
+ if len(read) == 0 {
+ return nil
+ }
+ args := make([]any, 0, 2*len(read))
+ for id, raw := range read {
+ args = append(args, id, raw)
+ }
+ return dropNoticesScript.Run(ctx, b.mgr.client, []string{key}, args...).Err()
+}
+
+// Ack implements interfaces.NoticeInbox.
+func (b *RedisNoticeInbox) Ack(ctx context.Context, s interfaces.SessionRef, ids []string) error {
+ if len(ids) == 0 {
+ return nil
+ }
+ if err := b.mgr.client.HDel(ctx, b.key(s), ids...).Err(); err != nil {
+ return fmt.Errorf("ack background notices: %w", err)
+ }
+ return nil
+}
+
+// Clear implements interfaces.NoticeInbox.
+func (b *RedisNoticeInbox) Clear(ctx context.Context, s interfaces.SessionRef) error {
+ if err := b.mgr.client.Del(ctx, b.key(s)).Err(); err != nil {
+ return fmt.Errorf("clear background notices: %w", err)
+ }
+ return nil
+}
diff --git a/internal/stream/notice_inbox_redis_test.go b/internal/stream/notice_inbox_redis_test.go
new file mode 100644
index 000000000..38a09a78b
--- /dev/null
+++ b/internal/stream/notice_inbox_redis_test.go
@@ -0,0 +1,77 @@
+package stream
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/background/backgroundtest"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+func TestRedisNoticeInboxContract(t *testing.T) {
+ backgroundtest.RunNoticeInboxContract(t,
+ func(t *testing.T) (interfaces.NoticeInbox, backgroundtest.Advance) {
+ mgr, mini := newTestRedisStreamManager(t, time.Hour)
+ now := time.Date(2026, 10, 14, 10, 0, 0, 0, time.UTC)
+ advance := func(d time.Duration) {
+ now = now.Add(d)
+ mini.FastForward(d)
+ }
+ return NewRedisNoticeInbox(mgr, func() time.Time { return now }), advance
+ })
+}
+
+// The write and the TTL refresh happen in one script, so a stored notice
+// always carries the backstop TTL, and a repeated push still refreshes it.
+func TestRedisNoticeInboxPushSetsKeyTTL(t *testing.T) {
+ mgr, mini := newTestRedisStreamManager(t, time.Hour)
+ inbox := NewRedisNoticeInbox(mgr, nil)
+ ctx := context.Background()
+ s := interfaces.SessionRef{TenantID: 1, SessionID: "sess-1"}
+ n := interfaces.BackgroundNotice{ID: "sj_1", Kind: "shell", Text: "done", FinishedAt: time.Now()}
+
+ first, err := inbox.Push(ctx, s, n)
+ require.NoError(t, err)
+ require.True(t, first)
+ require.Equal(t, noticeKeyTTL, mini.TTL(inbox.key(s)))
+
+ mini.FastForward(time.Hour)
+ first, err = inbox.Push(ctx, s, n)
+ require.NoError(t, err)
+ require.False(t, first)
+ require.Equal(t, noticeKeyTTL, mini.TTL(inbox.key(s)))
+}
+
+// Pending reads, then deletes what it saw as expired. A notice acked and
+// pushed again under the same ID in between must survive that delete.
+func TestRedisNoticeInboxExpiryKeepsNoticeRewrittenAfterRead(t *testing.T) {
+ mgr, mini := newTestRedisStreamManager(t, time.Hour)
+ inbox := NewRedisNoticeInbox(mgr, nil)
+ ctx := context.Background()
+ s := interfaces.SessionRef{TenantID: 1, SessionID: "sess-1"}
+ key := inbox.key(s)
+
+ old := time.Now().Add(-2 * interfaces.NoticeRetention)
+ _, err := inbox.Push(ctx, s, interfaces.BackgroundNotice{ID: "sj_1", Kind: "shell", Text: "old", FinishedAt: old})
+ require.NoError(t, err)
+ _, err = inbox.Push(ctx, s, interfaces.BackgroundNotice{ID: "sj_2", Kind: "shell", Text: "old", FinishedAt: old})
+ require.NoError(t, err)
+ read := map[string]string{"sj_1": mini.HGet(key, "sj_1"), "sj_2": mini.HGet(key, "sj_2")}
+
+ require.NoError(t, inbox.Ack(ctx, s, []string{"sj_1"}))
+ fresh := interfaces.BackgroundNotice{ID: "sj_1", Kind: "shell", Text: "fresh", FinishedAt: time.Now()}
+ first, err := inbox.Push(ctx, s, fresh)
+ require.NoError(t, err)
+ require.True(t, first)
+
+ require.NoError(t, inbox.dropIfUnchanged(ctx, key, read))
+
+ got, err := inbox.Pending(ctx, s)
+ require.NoError(t, err)
+ require.Len(t, got, 1)
+ require.Equal(t, "sj_1", got[0].ID)
+ require.Equal(t, "fresh", got[0].Text)
+ require.Empty(t, mini.HGet(key, "sj_2"), "an expired notice that did not change must be dropped")
+}
diff --git a/internal/types/agent.go b/internal/types/agent.go
index a34086541..42bb3154d 100644
--- a/internal/types/agent.go
+++ b/internal/types/agent.go
@@ -470,6 +470,15 @@ func IsPipelineToolCallID(id string) bool {
return strings.HasPrefix(id, PipelineToolCallIDPrefix)
}
+// BackgroundInjection is one background-work message the engine put in front
+// of the model before a step's LLM call. History replays it in place; AckIDs
+// are the completion notices it delivered, acknowledged once the assistant
+// message carrying the step is persisted.
+type BackgroundInjection struct {
+ Content string `json:"content"`
+ AckIDs []string `json:"ack_ids,omitempty"`
+}
+
// AgentStep represents one iteration of the ReAct loop
type AgentStep struct {
Iteration int `json:"iteration"` // Iteration number (0-indexed)
@@ -480,6 +489,12 @@ type AgentStep struct {
// UserMessagesBefore records consumed steer rows in delivery order, before
// this model response. Unlike timestamps, this remains unambiguous on replay.
UserMessagesBefore []string `json:"user_messages_before,omitempty"`
+ // BackgroundInjections are background-work messages delivered before this
+ // model response, after the step's steered messages, in delivery order.
+ BackgroundInjections []BackgroundInjection `json:"background_injections,omitempty"`
+ // BackgroundSeenIDs are finished background items whose results this
+ // step's tool calls (work_wait, work_cancel) returned to the model.
+ BackgroundSeenIDs []string `json:"background_seen_ids,omitempty"`
// IntermediateAnswer preserves a plain answer followed by a loop-end steer.
// The canonical final answer is still stored in Message.Content.
IntermediateAnswer bool `json:"intermediate_answer,omitempty"`
@@ -520,13 +535,14 @@ func (s *AgentStep) GetObservations() []string {
// AgentState tracks the execution state of an agent across iterations
type AgentState struct {
- PendingSteerMessages []string `json:"-"`
- CurrentRound int `json:"current_round"` // Current round number
- RoundSteps []AgentStep `json:"round_steps"` // All steps taken so far in the current round
- IsComplete bool `json:"is_complete"` // Whether agent has finished
- FinalAnswer string `json:"final_answer"` // The final answer to the query
- KnowledgeRefs []*SearchResult `json:"knowledge_refs"` // Collected knowledge references
- TurnUsage TokenUsage `json:"turn_usage"` // LLM usage accumulated across this turn
+ PendingSteerMessages []string `json:"-"`
+ PendingBackground []BackgroundInjection `json:"-"`
+ CurrentRound int `json:"current_round"` // Current round number
+ RoundSteps []AgentStep `json:"round_steps"` // All steps taken so far in the current round
+ IsComplete bool `json:"is_complete"` // Whether agent has finished
+ FinalAnswer string `json:"final_answer"` // The final answer to the query
+ KnowledgeRefs []*SearchResult `json:"knowledge_refs"` // Collected knowledge references
+ TurnUsage TokenUsage `json:"turn_usage"` // LLM usage accumulated across this turn
// Suspended is set when the run ended waiting on a user prompt.
// SuspendedStep is the interrupted round: its thought and the calls
// that finished before the blocked one, kept for display.
diff --git a/internal/types/chat.go b/internal/types/chat.go
index 803948407..f7ed5d513 100644
--- a/internal/types/chat.go
+++ b/internal/types/chat.go
@@ -248,6 +248,15 @@ const (
ResponseTypeInstallOutput ResponseType = "install_output"
// ResponseTypeCommandOutput updates the pending tool card without completing it.
ResponseTypeCommandOutput ResponseType = "command_output"
+ // ResponseTypeWorkStarted is a background work event of shared design §7.
+ // Features emit it through service.BackgroundEventEmitter, which picks the stream.
+ ResponseTypeWorkStarted ResponseType = "work_started"
+ // ResponseTypeWorkEvent is a background work event of shared design §7.
+ // Features emit it through service.BackgroundEventEmitter, which picks the stream.
+ ResponseTypeWorkEvent ResponseType = "work_event"
+ // ResponseTypeWorkFinished is a background work event of shared design §7.
+ // Features emit it through service.BackgroundEventEmitter, which picks the stream.
+ ResponseTypeWorkFinished ResponseType = "work_finished"
// Error response type
ResponseTypeError ResponseType = "error"
// Reflection response type (for agent reflection)
diff --git a/internal/types/interfaces/background.go b/internal/types/interfaces/background.go
new file mode 100644
index 000000000..a3ad86be3
--- /dev/null
+++ b/internal/types/interfaces/background.go
@@ -0,0 +1,145 @@
+package interfaces
+
+import (
+ "context"
+ "errors"
+ "time"
+)
+
+// SessionRef identifies a session in every background-work interface.
+type SessionRef struct {
+ TenantID uint64
+ SessionID string
+}
+
+// External work states. The model and the UI learn only these strings; a
+// feature maps its own internal states onto them at the boundary.
+const (
+ WorkStateRunning = "running"
+ WorkStateUnknown = "unknown" // cheap lookup could not decide; never returned to the model
+ WorkStateSucceeded = "succeeded"
+ WorkStateFailed = "failed"
+ WorkStateCancelled = "cancelled"
+ WorkStateLost = "lost"
+)
+
+// IsTerminalWorkState reports whether s is one of the four terminal states.
+func IsTerminalWorkState(s string) bool {
+ switch s {
+ case WorkStateSucceeded, WorkStateFailed, WorkStateCancelled, WorkStateLost:
+ return true
+ default:
+ return false
+ }
+}
+
+// ErrActivityLimit is returned by ActivityRegistry.Add when the session already
+// holds kindCap live items of the same kind.
+var ErrActivityLimit = errors.New("background: activity limit reached")
+
+// ErrWorkNotFound is returned by a WorkHandler for an ID that does not exist in
+// the given session. Handlers must use it for IDs owned by another session too.
+var ErrWorkNotFound = errors.New("background: work not found in this session")
+
+// BackgroundActivity is one live background item registered for a session.
+type BackgroundActivity struct {
+ ID string // feature-prefixed, e.g. "sj_..."; opaque to shared code
+ Kind string // opaque to shared code; used only for per-kind caps and UI badges
+ Title string
+ StartedAt time.Time
+}
+
+// ActivityRegistry records which background items are live in a session. Every
+// entry is a lease: it disappears unless renewed before its deadline.
+type ActivityRegistry interface {
+ // Add registers a under a lease of length lease. If a.ID is already live it
+ // only extends the deadline and does not count against kindCap. Otherwise it
+ // fails with ErrActivityLimit when the session already holds kindCap live
+ // items of a.Kind. kindCap <= 0 means no limit.
+ Add(ctx context.Context, s SessionRef, a BackgroundActivity, lease time.Duration, kindCap int) error
+ // Renew moves a live item's deadline to now+lease. It returns found=false,
+ // and does nothing, when the item was removed or has expired.
+ Renew(ctx context.Context, s SessionRef, id string, lease time.Duration) (found bool, err error)
+ // Remove deletes the item. Removing a missing item is not an error.
+ Remove(ctx context.Context, s SessionRef, id string) error
+ // List returns live items only, in no particular order.
+ List(ctx context.Context, s SessionRef) ([]BackgroundActivity, error)
+}
+
+// NoticeRetention is how long an unacknowledged notice stays after FinishedAt.
+const NoticeRetention = 24 * time.Hour
+
+// BackgroundNotice is one completion notice waiting to be shown to the model.
+type BackgroundNotice struct {
+ ID string // equals the activity ID
+ Kind string
+ Text string // model-facing text written by the feature; the feature bounds its length
+ FinishedAt time.Time
+}
+
+// NoticeInbox holds completion notices until a persisted turn acknowledges them.
+type NoticeInbox interface {
+ // Push stores n unless a notice with the same ID already exists. Only the
+ // first push for an ID returns first=true; later ones change nothing.
+ Push(ctx context.Context, s SessionRef, n BackgroundNotice) (first bool, err error)
+ // Pending returns unacknowledged notices finished within NoticeRetention,
+ // oldest first.
+ Pending(ctx context.Context, s SessionRef) ([]BackgroundNotice, error)
+ // Ack deletes the given IDs. Unknown IDs are ignored.
+ Ack(ctx context.Context, s SessionRef, ids []string) error
+ // Clear deletes every notice of the session.
+ Clear(ctx context.Context, s SessionRef) error
+}
+
+// StatusOptions controls one WorkHandler.Status call.
+type StatusOptions struct {
+ // Detailed=false must be cheap: Redis, memory or a local file only, and
+ // never a sandbox exec. It may return WorkStateUnknown.
+ Detailed bool
+ // RunID scopes incremental output reads to one agent run. Empty means the
+ // read is not part of a run: return head and tail, move no read position.
+ RunID string
+ // OutputBudget caps output bytes for each item; 0 means the handler default.
+ OutputBudget int
+}
+
+// WorkResult is one item's state as returned by a WorkHandler.
+type WorkResult struct {
+ ID string
+ State string // one of the WorkState* constants
+ Terminal bool
+ Title string
+ Payload any // feature-defined; serialized into tool results as-is
+ // OutputFiles is the deliverable list for a finished shell job.
+ // Nil means the output directory was not inspected.
+ OutputFiles []string
+}
+
+// WorkHandler answers status and cancel requests for one feature's work IDs.
+type WorkHandler interface {
+ // Status returns one result per ID in input order. An ID that does not
+ // belong to s must yield ErrWorkNotFound for the whole call or be reported
+ // with State=WorkStateLost. A batch error does not confirm the states of
+ // the other requested IDs or acknowledge their completion notices.
+ Status(ctx context.Context, s SessionRef, ids []string, opts StatusOptions) ([]WorkResult, error)
+ // Cancel requests cancellation and returns the state observed at that time.
+ Cancel(ctx context.Context, s SessionRef, ids []string) ([]WorkResult, error)
+}
+
+// SessionLifecycleListener lets features react to /stop and session deletion.
+type SessionLifecycleListener interface {
+ OnStop(ctx context.Context, s SessionRef) error
+ OnDelete(ctx context.Context, s SessionRef) error
+}
+
+// SessionKeepAliver keeps a session's sandbox from being reclaimed by its
+// provider. Implementations throttle internally and never provision a sandbox.
+type SessionKeepAliver interface {
+ KeepAlive(ctx context.Context, s SessionRef) error
+}
+
+// WorkEventEmitter publishes the work_finished event. It is called only by the
+// writer whose NoticeInbox.Push returned first=true.
+type WorkEventEmitter interface {
+ EmitWorkFinished(ctx context.Context, s SessionRef, parentMessageID string, r WorkResult) error
+}
diff --git a/internal/types/interfaces/background_test.go b/internal/types/interfaces/background_test.go
new file mode 100644
index 000000000..766645344
--- /dev/null
+++ b/internal/types/interfaces/background_test.go
@@ -0,0 +1,20 @@
+package interfaces
+
+import "testing"
+
+func TestIsTerminalWorkState(t *testing.T) {
+ terminal := []string{
+ WorkStateSucceeded, WorkStateFailed, WorkStateCancelled, WorkStateLost,
+ }
+ for _, s := range terminal {
+ if !IsTerminalWorkState(s) {
+ t.Errorf("IsTerminalWorkState(%q) = false, want true", s)
+ }
+ }
+ nonTerminal := []string{WorkStateRunning, WorkStateUnknown, "queued", "timeout", "", "bogus"}
+ for _, s := range nonTerminal {
+ if IsTerminalWorkState(s) {
+ t.Errorf("IsTerminalWorkState(%q) = true, want false", s)
+ }
+ }
+}
diff --git a/internal/types/sandbox_checkpoint_test.go b/internal/types/sandbox_checkpoint_test.go
new file mode 100644
index 000000000..930a615a6
--- /dev/null
+++ b/internal/types/sandbox_checkpoint_test.go
@@ -0,0 +1,22 @@
+package types
+
+import (
+ "encoding/json"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestSandboxCheckpointBackgroundActiveJSON(t *testing.T) {
+ raw, err := json.Marshal(SandboxCheckpoint{SandboxID: "sbx", CommitSHA: "sha"})
+ require.NoError(t, err)
+ require.NotContains(t, string(raw), "background_active", "unflagged checkpoints keep their old shape")
+
+ var flagged SandboxCheckpoint
+ require.NoError(t, flagged.Scan([]byte(`{"sandbox_id":"sbx","commit_sha":"sha","background_active":true}`)))
+ require.True(t, flagged.BackgroundActive)
+
+ var old SandboxCheckpoint
+ require.NoError(t, old.Scan([]byte(`{"sandbox_id":"sbx","commit_sha":"sha"}`)))
+ require.False(t, old.BackgroundActive, "rows written before the flag existed stay usable")
+}
diff --git a/internal/types/session_fork.go b/internal/types/session_fork.go
index 10d07dcad..a6b07c814 100644
--- a/internal/types/session_fork.go
+++ b/internal/types/session_fork.go
@@ -20,6 +20,10 @@ type SandboxCheckpoint struct {
SandboxID string `json:"sandbox_id"`
CommitSHA string `json:"commit_sha"`
CommittedAt time.Time `json:"committed_at"`
+ // BackgroundActive marks a commit taken while background work was live in
+ // the session. The work may have been mid-write, so fork and rewind never
+ // roll a workspace back to it.
+ BackgroundActive bool `json:"background_active,omitempty"`
}
// Value implements the driver.Valuer interface for database serialization.