Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(automation): enforce execution identity, cancellation, and scheduling limits (#4506)

lyingbuga8eb7e28fa40
diff --git a/internal/application/service/agent_collaboration.go b/internal/application/service/agent_collaboration.go
index 197814623..6e9ca5124 100644
--- a/internal/application/service/agent_collaboration.go
+++ b/internal/application/service/agent_collaboration.go
@@ -184,12 +184,37 @@ func (s *AgentCollaboration) prune(ctx context.Context) error {
type delegatedConfig struct {
Agent types.CustomAgent `json:"agent"`
Access types.AssistantAccessLevel `json:"access"`
+ // Durable execution identity follows children across scheduler instances
+ // and approval resumes; it must not turn an automation into a human turn.
+ ActingVia string `json:"acting_via,omitempty"`
+ Mode types.RunMode `json:"mode,omitempty"`
// AssistantParent records that the delegating run was the WeKnora
// assistant, whose level stays under the workspace's assistant ceiling.
AssistantParent bool `json:"assistant_parent,omitempty"`
Inherited *inheritedAgentConfig `json:"inherited,omitempty"`
}
+func (d *delegatedConfig) captureExecution(ctx context.Context) {
+ d.ActingVia = types.ActingViaFromContext(ctx)
+ d.Mode = types.RunModeAttended
+ if mode, ok := ctx.Value(types.RunModeContextKey).(types.RunMode); ok {
+ d.Mode = mode.Normalize()
+ }
+ if d.ActingVia != "" {
+ d.Mode = types.RunModeDeferred
+ }
+}
+
+func (d *delegatedConfig) runMode() types.RunMode {
+ if d.ActingVia != "" {
+ return types.RunModeDeferred
+ }
+ if d.Mode == "" {
+ return types.RunModeAttended // snapshots created before mode was stored
+ }
+ return d.Mode.Normalize()
+}
+
// level is the child's access level: the delegating run's, capped by the
// child agent's own ceiling and, under an assistant parent, by the
// workspace's current assistant ceiling.
@@ -222,7 +247,9 @@ func (s *AgentCollaboration) resolveDelegated(
id == types.BuiltinSkillInstallerID {
return "", errors.New("only same-workspace reasoning agents without management privileges may be delegated")
}
- data, err := json.Marshal(delegatedConfig{Agent: *a, Access: level, AssistantParent: assistantParent})
+ saved := delegatedConfig{Agent: *a, Access: level, AssistantParent: assistantParent}
+ saved.captureExecution(ctx)
+ data, err := json.Marshal(saved)
return string(data), err
}
@@ -377,7 +404,11 @@ func (s *AgentCollaboration) workerContext(ctx context.Context, scope collaborat
ctx = context.WithValue(ctx, types.TenantRoleContextKey, membership.Role)
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant)
ctx = types.WithCaller(ctx, types.Caller{TenantID: scope.TenantID, UserID: scope.UserID, Role: membership.Role})
- ctx = types.WithPrincipal(ctx, types.Principal{Type: types.PrincipalWebUser, ID: scope.UserID})
+ principal := types.Principal{Type: types.PrincipalWebUser, ID: scope.UserID}
+ if previous, ok := types.PrincipalFromContext(ctx); ok && previous.ID == scope.UserID {
+ principal.ActingVia = previous.ActingVia
+ }
+ ctx = types.WithPrincipal(ctx, principal)
return ctx, nil
}
@@ -397,6 +428,10 @@ func (s *AgentCollaboration) run(ctx context.Context, w collaboration.Work) (col
if err = json.Unmarshal([]byte(w.Member.ConfigJSON), &saved); err != nil {
return result, err
}
+ ctx = types.WithPrincipal(ctx, types.Principal{
+ Type: types.PrincipalWebUser, ID: w.Scope.UserID, ActingVia: saved.ActingVia,
+ })
+ ctx = types.WithRunMode(ctx, saved.runMode())
// Both modes pin configuration, but reauthorize the source before each turn.
if saved.Inherited == nil {
if _, err = s.resolve(ctx, w.Member.AgentID, types.AssistantAccessReadOnly); err != nil {
@@ -610,7 +645,7 @@ func (s *AgentCollaboration) run(ctx context.Context, w collaboration.Work) (col
ctx,
&types.QARequest{
Resume: resume,
- Mode: types.RunModeAttended,
+ Mode: saved.runMode(),
GrantSessionID: w.Scope.SessionID,
Suspendable: true,
Session: session,
diff --git a/internal/application/service/agent_collaboration_inherited.go b/internal/application/service/agent_collaboration_inherited.go
index e2d8e8064..b2b5211bf 100644
--- a/internal/application/service/agent_collaboration_inherited.go
+++ b/internal/application/service/agent_collaboration_inherited.go
@@ -75,10 +75,12 @@ func inheritedChildSnapshot(ctx context.Context, cfg *types.AgentConfig, toolNam
}
}
// Marshal immediately: later changes to parent config cannot change queued work.
- data, err := json.Marshal(delegatedConfig{
+ saved := delegatedConfig{
Agent: *source.Agent, Access: cfg.AssistantAccess, AssistantParent: cfg.OperationsAgentID != "",
Inherited: inherited,
- })
+ }
+ saved.captureExecution(ctx)
+ data, err := json.Marshal(saved)
return string(data), err
}
diff --git a/internal/application/service/agent_collaboration_recovery.go b/internal/application/service/agent_collaboration_recovery.go
index 223f2c3c5..9ba5f3c71 100644
--- a/internal/application/service/agent_collaboration_recovery.go
+++ b/internal/application/service/agent_collaboration_recovery.go
@@ -223,6 +223,11 @@ func (s *AgentCollaboration) resumeParent(ctx context.Context, b *collaboration.
if err = json.Unmarshal(snap.Request, &saved); err != nil {
return err
}
+ if saved.ActingVia != "" {
+ // Automation leases, cancellation and duration belong to their own
+ // scheduler. Never revive one through ordinary parent recovery.
+ return errors.New("scheduler-owned parent cannot resume through collaboration recovery")
+ }
if saved.AgentID == "" || saved.SharedAgentReadOnly {
return errors.New("invalid parent recovery identity")
}
diff --git a/internal/application/service/automation.go b/internal/application/service/automation.go
index f5f07d7e9..4f24789e0 100644
--- a/internal/application/service/automation.go
+++ b/internal/application/service/automation.go
@@ -80,6 +80,8 @@ type AutomationService struct {
// Events publishes a run to its message's event stream; see
// AgentCollaboration.ChildEvents.
Events func(context.Context, *types.Session, *types.Message, *event.EventBus) func(context.Context) error
+ // StopDelegation fences children and discards recoverable parent waits.
+ StopDelegation func(context.Context, uint64, string, string, string) error
cfg AutomationConfig
instance string
@@ -621,16 +623,37 @@ func (s *AutomationService) cancelRun(ctx context.Context, run *types.Automation
Update("cancel_requested", true).Error; err != nil {
return err
}
+ stopErr := s.stopRunDelegation(ctx, run)
s.mu.Lock()
cancel := s.running[run.ID]
s.mu.Unlock()
if cancel != nil {
cancel()
}
+ return stopErr
}
return nil
}
+func (s *AutomationService) stopRunDelegation(ctx context.Context, run *types.AutomationRun) error {
+ if s.StopDelegation == nil {
+ return nil
+ }
+ // The caller may hold the row from before the executor created its turn.
+ var current types.AutomationRun
+ if err := s.db.WithContext(ctx).Where("id = ?", run.ID).Take(&current).Error; err != nil {
+ return err
+ }
+ if current.SessionID == "" || current.AssistantMessageID == "" {
+ return nil
+ }
+ a, err := s.automationOf(ctx, &current)
+ if err != nil {
+ return err
+ }
+ return s.StopDelegation(ctx, a.TenantID, a.OwnerUserID, current.SessionID, current.AssistantMessageID)
+}
+
// closePrompt ends the open prompt id a run waits on and completes the
// run's message, as the prompt reaper would.
func (s *AutomationService) closePrompt(ctx context.Context, id, status, reason string) {
diff --git a/internal/application/service/automation_regression_test.go b/internal/application/service/automation_regression_test.go
new file mode 100644
index 000000000..aa22ad3c2
--- /dev/null
+++ b/internal/application/service/automation_regression_test.go
@@ -0,0 +1,462 @@
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/agent"
+ "github.com/Tencent/WeKnora/internal/agent/collaboration"
+ "github.com/Tencent/WeKnora/internal/event"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/require"
+ "gorm.io/driver/sqlite"
+ "gorm.io/gorm"
+ "gorm.io/gorm/logger"
+)
+
+func TestAutomationDelegatedExecutionKeepsIdentity(t *testing.T) {
+ for _, inherited := range []bool{false, true} {
+ t.Run(fmt.Sprintf("inherited=%t", inherited), func(t *testing.T) {
+ svc, agents, _, ctx := collaborationService(t)
+ via := types.AutomationActingVia("scheduled-task")
+ parent := types.WithPrincipal(ctx, types.Principal{Type: types.PrincipalWebUser, ID: "u", ActingVia: via})
+ parent = types.WithRunMode(parent, types.RunModeDeferred)
+ var snapshot string
+ var err error
+ if inherited {
+ parent = context.WithValue(parent, childSourceKey{}, childSource{Agent: agents.agent, ModelID: "model"})
+ snapshot, err = inheritedChildSnapshot(
+ parent,
+ &types.AgentConfig{AssistantAccess: types.AssistantAccessAutoWrite},
+ nil,
+ )
+ } else {
+ snapshot, err = svc.resolve(parent, "a", types.AssistantAccessAutoWrite)
+ }
+ require.NoError(t, err)
+ scope := collaboration.Scope{TenantID: automationTenant, UserID: "u", SessionID: "parent", ID: "answer"}
+ require.NoError(t, svc.Runtime.Store.Ensure(ctx, scope))
+ _, err = svc.Runtime.Spawn(ctx, scope, "spawn", "child", "a", snapshot, "work")
+ require.NoError(t, err)
+ require.NoError(t, svc.Runtime.Store.Change(ctx, scope, func(g *collaboration.Group) error {
+ r := g.Members[0].Runs[0]
+ r.State = "running"
+ r.Owner = "worker"
+ r.Lease = time.Now().Add(time.Minute)
+ return nil
+ }))
+ called := false
+ svc.Bind(
+ &collaborationSessions{
+ run: func(child context.Context, req *types.QARequest, bus *event.EventBus) error {
+ called = true
+ require.Equal(t, via, types.ActingViaFromContext(child))
+ require.Equal(t, types.RunModeDeferred, req.Mode)
+ require.False(t, req.Mode.Attended(), "ask_user must stay unavailable")
+ _, _, err := automationOwner(child)
+ require.Error(t, err, "delegation must not bypass the automation management guard")
+ return complete(child, bus, event.AgentCompleteData{FinalAnswer: "done"})
+ },
+ },
+ )
+ g, err := svc.Runtime.Store.Read(ctx, scope)
+ require.NoError(t, err)
+ // A scheduler starts from its own context, not the parent's request context.
+ _, err = svc.run(
+ context.Background(),
+ collaboration.Work{Scope: scope, Member: *g.Members[0], Run: *g.Members[0].Runs[0]},
+ )
+ require.NoError(t, err)
+ require.True(t, called)
+ })
+ }
+}
+
+func TestAutomationCancelStopsDelegationAndRecovery(t *testing.T) {
+ f := newAutomationFixture(t)
+ require.NoError(t, f.db.AutoMigrate(&collaboration.Record{}))
+ delegated := NewAgentCollaboration(f.db, f.agents, f.messages, f.users, f.members, f.svc.tenants)
+ f.svc.StopDelegation = delegated.StopOwned
+ a := f.create(t, types.AutomationPatch{})
+ queued, err := f.svc.RunNow(f.ctx, a.ID)
+ require.NoError(t, err)
+ entered := make(chan collaboration.Scope, 1)
+ f.sessions.run = func(ctx context.Context, req *types.QARequest, _ *event.EventBus) error {
+ scope := collaboration.Scope{
+ TenantID: automationTenant,
+ UserID: "u",
+ SessionID: req.Session.ID,
+ ID: req.AssistantMessageID,
+ }
+ b, err := delegated.Runtime.AcquireParent(ctx, scope)
+ if err != nil {
+ return err
+ }
+ _, err = delegated.Runtime.Spawn(b.Context(ctx), scope, "spawn", "child", "a", "{}", "work")
+ if err != nil {
+ return err
+ }
+ _, err = delegated.Runtime.Spawn(b.Context(ctx), scope, "spawn-running", "running_child", "a", "{}", "work")
+ if err != nil {
+ return err
+ }
+ if err = delegated.Runtime.Store.Change(b.Context(ctx), scope, func(g *collaboration.Group) error {
+ r := g.Members[1].Runs[0]
+ r.State, r.Owner, r.Lease = "running", "remote-worker", time.Now().Add(time.Minute)
+ return nil
+ }); err != nil {
+ return err
+ }
+ saved := resumableAgentRequest(ctx, req)
+ data, err := json.Marshal(
+ agent.RunSnapshot{Version: agent.RunSnapshotVersion, Request: mustJSON(t, saved), CollaborationWait: true},
+ )
+ if err != nil {
+ return err
+ }
+ if err = b.SaveWait(ctx, data, time.Now().Add(time.Minute)); err != nil {
+ return err
+ }
+ entered <- scope
+ <-ctx.Done()
+ // Exercise the same cleanup as an engine cancelled while in wait_agent.
+ b.End(true)
+ return ctx.Err()
+ }
+ require.NoError(t, f.svc.startQueued(f.ctx, f.now))
+ var scope collaboration.Scope
+ select {
+ case scope = <-entered:
+ case <-time.After(5 * time.Second):
+ t.Fatal("run did not start")
+ }
+ _, err = f.svc.CancelRun(f.ctx, queued.ID)
+ require.NoError(t, err)
+ done := f.settleRun(t, queued.ID)
+ require.Equal(t, types.AutomationRunCancelled, done.Status)
+ g, err := delegated.Runtime.Store.Read(f.ctx, scope)
+ require.NoError(t, err)
+ require.NotEqual(t, "active", g.Status)
+ require.Nil(t, g.ParentWait, "cancelled checkpoints must not be recovered")
+ require.True(t, g.Members[0].Runs[0].CancelRequested)
+ require.Equal(t, "cancelled", g.Members[0].Runs[0].State)
+ require.True(t, g.Members[1].Runs[0].CancelRequested)
+ require.ErrorIs(t, delegated.Runtime.CheckOwner(f.ctx, collaboration.Work{
+ Scope: scope, Member: *g.Members[1], Run: *g.Members[1].Runs[0],
+ }), collaboration.ErrConflict, "a remote child must lose permission to execute tools immediately")
+}
+
+func mustJSON(t *testing.T, v any) json.RawMessage {
+ t.Helper()
+ data, err := json.Marshal(v)
+ require.NoError(t, err)
+ return data
+}
+
+func TestAutomationParentRecoveryDoesNotEscapeScheduler(t *testing.T) {
+ svc, _, messages, ctx := collaborationService(t)
+ scope := collaboration.Scope{TenantID: automationTenant, UserID: "u", SessionID: "parent", ID: "answer"}
+ b, err := svc.Runtime.AcquireParent(ctx, scope)
+ require.NoError(t, err)
+ t.Cleanup(func() { b.End(false) })
+ require.NoError(t, messages.UpdateMessage(ctx, &types.Message{ID: scope.ID, SessionID: scope.SessionID}))
+ svc.streams = &collaborationStreams{}
+ svc.ParentEvents = func(
+ context.Context, *types.Session, *types.Message, *event.EventBus,
+ ) func(context.Context) error {
+ return func(context.Context) error { return nil }
+ }
+ called := false
+ svc.Bind(
+ &collaborationSessions{
+ run: func(context.Context, *types.QARequest, *event.EventBus) error { called = true; return nil },
+ },
+ )
+ ctx = types.WithPrincipal(
+ ctx,
+ types.Principal{Type: types.PrincipalWebUser, ID: "u", ActingVia: types.AutomationActingVia("a")},
+ )
+ req := resumableAgentRequest(
+ ctx,
+ &types.QARequest{CustomAgent: &types.CustomAgent{ID: "a"}, Mode: types.RunModeDeferred},
+ )
+ snapshot := mustJSON(
+ t,
+ agent.RunSnapshot{Version: agent.RunSnapshotVersion, Request: mustJSON(t, req), CollaborationWait: true},
+ )
+ err = svc.resumeParent(context.Background(), b, snapshot)
+ require.ErrorContains(t, err, "scheduler-owned parent")
+ require.False(t, called)
+}
+
+func TestAutomationStartSerializesDifferentRunsAcrossInstances(t *testing.T) {
+ f := newAutomationFixture(t)
+ a := f.create(t, types.AutomationPatch{OverlapPolicy: ptr(types.AutomationPolicyQueue)})
+ rows := []*types.AutomationRun{
+ {
+ AutomationID: a.ID,
+ TenantID: automationTenant,
+ ScheduledAt: f.now.Add(-time.Hour),
+ TriggerKind: types.AutomationRunSchedule,
+ Status: types.AutomationRunQueued,
+ Resume: true,
+ },
+ {
+ AutomationID: a.ID,
+ TenantID: automationTenant,
+ ScheduledAt: f.now,
+ TriggerKind: types.AutomationRunSchedule,
+ Status: types.AutomationRunQueued,
+ },
+ }
+ for _, r := range rows {
+ require.NoError(t, f.db.Create(r).Error)
+ }
+ // A separate connection pool represents another scheduler process.
+ db, err := gorm.Open(
+ sqlite.Open(f.db.Dialector.(*sqlite.Dialector).DSN),
+ &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)},
+ )
+ require.NoError(t, err)
+ raw, err := db.DB()
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = raw.Close() })
+ other := &AutomationService{db: db, instance: "second-instance"}
+ arrived := make(chan struct{}, 2)
+ release := make(chan struct{})
+ for _, db := range []*gorm.DB{f.db, db} {
+ require.NoError(t, db.Callback().Query().After("gorm:query").Register("admission_barrier", func(db *gorm.DB) {
+ if n, ok := db.Statement.Dest.(*int64); ok && *n == 0 && db.Statement.Table == "automation_runs" &&
+ strings.Contains(db.Statement.SQL.String(), "count(*)") {
+ arrived <- struct{}{}
+ <-release
+ }
+ }))
+ }
+ var wg sync.WaitGroup
+ wg.Add(2)
+ started := make([]bool, 2)
+ errs := make([]error, 2)
+ go func() { defer wg.Done(); started[0], errs[0] = f.svc.startRun(f.ctx, rows[0], f.now) }()
+ select {
+ case <-arrived:
+ case <-time.After(5 * time.Second):
+ close(release)
+ t.Fatal("first admission did not reach check")
+ }
+ go func() { defer wg.Done(); started[1], errs[1] = other.startRun(f.ctx, rows[1], f.now) }()
+ // The old implementation lets both count queries observe zero. With the
+ // automation lock the second query cannot run until the first commits.
+ select {
+ case <-arrived:
+ case <-time.After(100 * time.Millisecond):
+ }
+ close(release)
+ wg.Wait()
+ require.NoError(t, errs[0])
+ require.NoError(t, errs[1])
+ require.True(t, started[0])
+ require.False(t, started[1])
+ var active int64
+ require.NoError(
+ t,
+ f.db.Model(&types.AutomationRun{}).
+ Where("automation_id = ? AND status = ?", a.ID, types.AutomationRunRunning).
+ Count(&active).
+ Error,
+ )
+ require.EqualValues(t, 1, active)
+}
+
+func TestAutomationQueuedRunsSkipBlockedHead(t *testing.T) {
+ f := newAutomationFixture(t)
+ for i := 0; i < 16; i++ {
+ a := f.create(
+ t,
+ types.AutomationPatch{
+ Name: ptr(fmt.Sprintf("blocked%d", i)),
+ OverlapPolicy: ptr(types.AutomationPolicyQueue),
+ },
+ )
+ for j, status := range []string{types.AutomationRunAwaitingApproval, types.AutomationRunQueued} {
+ require.NoError(
+ t,
+ f.db.Create(
+ &types.AutomationRun{
+ AutomationID: a.ID,
+ TenantID: automationTenant,
+ ScheduledAt: f.now.Add(time.Duration(j-2) * time.Hour),
+ TriggerKind: types.AutomationRunSchedule,
+ Status: status,
+ },
+ ).Error,
+ )
+ }
+ }
+ a := f.create(t, types.AutomationPatch{Name: ptr("runnable")})
+ r, err := f.svc.RunNow(f.ctx, a.ID)
+ require.NoError(t, err)
+ require.NoError(t, f.svc.startQueued(context.Background(), f.now))
+ require.Eventually(t, func() bool {
+ var row types.AutomationRun
+ return f.db.First(&row, "id = ?", r.ID).Error == nil && row.Status == types.AutomationRunSucceeded &&
+ len(f.svc.slots) == 0
+ }, 5*time.Second, 10*time.Millisecond)
+}
+
+func TestAutomationApprovalReconciliationVisitsEveryBatch(t *testing.T) {
+ f := newAutomationFixture(t)
+ f.svc.cfg.MaxPerTenant = 200
+ var expired, answered *types.AutomationRun
+ for i := 0; i < 102; i++ {
+ timeout := 10080
+ updated := f.now.Add(-time.Hour)
+ if i == 100 {
+ timeout = 5
+ updated = f.now.Add(-10 * time.Minute)
+ }
+ a := f.create(
+ t,
+ types.AutomationPatch{Name: ptr(fmt.Sprintf("approval%d", i)), ApprovalTimeoutMinutes: ptr(timeout)},
+ )
+ status := types.AgentPromptSuspended
+ if i == 0 || i == 101 {
+ status = types.AgentPromptAnswered
+ }
+ p := &types.AgentPrompt{ID: fmt.Sprintf("p%d", i), Status: status}
+ require.NoError(t, f.broker.Store().Create(f.ctx, p))
+ r := &types.AutomationRun{
+ ID: fmt.Sprintf("run-%03d", i),
+ AutomationID: a.ID,
+ TenantID: automationTenant,
+ ScheduledAt: updated,
+ TriggerKind: types.AutomationRunSchedule,
+ Status: types.AutomationRunAwaitingApproval,
+ PendingPromptID: p.ID,
+ UpdatedAt: updated,
+ }
+ require.NoError(t, f.db.Create(r).Error)
+ if i == 100 {
+ expired = r
+ }
+ if i == 101 {
+ answered = r
+ }
+ }
+ require.NoError(t, f.svc.reconcileAwaiting(f.ctx, f.now))
+ require.NoError(t, f.db.First(expired, "id = ?", expired.ID).Error)
+ require.Equal(t, types.AutomationRunFailed, expired.Status)
+ require.NoError(t, f.db.First(answered, "id = ?", answered.ID).Error)
+ require.Equal(t, types.AutomationRunQueued, answered.Status)
+ require.True(t, answered.Resume)
+}
+
+func TestAutomationApprovalResumeUsesRemainingExecutionBudget(t *testing.T) {
+ f := newAutomationFixture(t)
+ f.svc.cfg.DefaultMaxDuration = time.Second
+ a := f.create(t, types.AutomationPatch{})
+ f.suspendOnApproval(t, "op:x")
+ original := f.sessions.run
+ f.sessions.run = func(ctx context.Context, req *types.QARequest, bus *event.EventBus) error {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-time.After(600 * time.Millisecond):
+ }
+ return original(ctx, req, bus)
+ }
+ queued, err := f.svc.RunNow(f.ctx, a.ID)
+ require.NoError(t, err)
+ suspended := f.settleRun(t, queued.ID)
+ require.Equal(t, types.AutomationRunAwaitingApproval, suspended.Status)
+ // Approval waiting is free, even across service restarts.
+ f.now = f.now.Add(12 * time.Hour)
+ restarted := NewAutomationService(
+ f.db,
+ f.agents,
+ f.users,
+ f.members,
+ f.svc.tenants,
+ f.messages,
+ f.svc.grants,
+ f.svc.notices,
+ )
+ restarted.Configure(AutomationConfig{DefaultMaxDuration: time.Second})
+ restarted.now = func() time.Time { return f.now }
+ restarted.Bind(f.sessions, f.broker, nil, nil)
+ f.svc = restarted
+ ok, err := f.broker.Store().Transition(f.ctx, suspended.PendingPromptID, types.AgentPromptTransition{
+ From: []string{types.AgentPromptSuspended},
+ To: types.AgentPromptAnswered,
+ Response: types.JSON(`{"approved":true}`),
+ })
+ require.NoError(t, err)
+ require.True(t, ok)
+ resumed := f.settleRun(t, queued.ID)
+ require.Equal(t, types.AutomationRunFailed, resumed.Status)
+ require.Contains(t, resumed.Error, "longer than 1s")
+ require.NotNil(t, f.sessions.last().req.Resume, "waiting time must not exhaust the execution budget")
+}
+
+func TestAutomationStaleFinalizerDoesNotCancelResumedDelegation(t *testing.T) {
+ f := newAutomationFixture(t)
+ a := f.create(t, types.AutomationPatch{})
+ r, err := f.svc.RunNow(f.ctx, a.ID)
+ require.NoError(t, err)
+ require.NoError(t, f.db.Model(r).Updates(map[string]any{
+ "status": types.AutomationRunRunning, "session_id": "session", "assistant_message_id": "answer",
+ }).Error)
+ stopped := false
+ f.svc.StopDelegation = func(context.Context, uint64, string, string, string) error { stopped = true; return nil }
+ ok, err := f.svc.finishRun(
+ f.ctx,
+ r,
+ []string{types.AutomationRunAwaitingApproval},
+ types.AutomationRunFailed,
+ "stale timeout",
+ nil,
+ )
+ require.NoError(t, err)
+ require.False(t, ok)
+ require.False(t, stopped)
+ ok, err = f.svc.finishRun(
+ f.ctx,
+ r,
+ []string{types.AutomationRunRunning},
+ types.AutomationRunNeedsAttention,
+ "executor lost",
+ nil,
+ )
+ require.NoError(t, err)
+ require.True(t, ok)
+ require.True(t, stopped)
+}
+
+func TestAutomationCancellationWinsApprovalRace(t *testing.T) {
+ f := newAutomationFixture(t)
+ a := f.create(t, types.AutomationPatch{})
+ f.suspendOnApproval(t, "op:x")
+ queued, err := f.svc.RunNow(f.ctx, a.ID)
+ require.NoError(t, err)
+ r := f.settleRun(t, queued.ID)
+ require.Equal(t, types.AutomationRunAwaitingApproval, r.Status)
+ // A remote cancellation can land after execute checked its context but
+ // before it records the approval suspension. Do not resume that run.
+ require.NoError(t, f.db.Model(r).Update("cancel_requested", true).Error)
+ ok, err := f.broker.Store().Transition(f.ctx, r.PendingPromptID, types.AgentPromptTransition{
+ From: []string{types.AgentPromptSuspended},
+ To: types.AgentPromptAnswered,
+ Response: types.JSON(`{"approved":true}`),
+ })
+ require.NoError(t, err)
+ require.True(t, ok)
+ require.NoError(t, f.svc.queueResume(f.ctx, r.ID, r.PendingPromptID))
+ require.NoError(t, f.svc.reconcileAwaiting(f.ctx, f.now))
+ require.NoError(t, f.db.First(&r, "id = ?", r.ID).Error)
+ require.Equal(t, types.AutomationRunCancelled, r.Status)
+}
diff --git a/internal/application/service/automation_runner.go b/internal/application/service/automation_runner.go
index e10c4e391..34724bace 100644
--- a/internal/application/service/automation_runner.go
+++ b/internal/application/service/automation_runner.go
@@ -36,6 +36,7 @@ type runOutcome struct {
answer string
usage *types.TokenUsage
promptID string
+ elapsed time.Duration
// pause, when set, pauses the automation with this reason.
pause string
}
@@ -56,6 +57,7 @@ func (s *AutomationService) execute(parent context.Context, run *types.Automatio
}()
go s.heartbeat(ctx, run.ID, cancel)
+ started := time.Now()
out := func() (out runOutcome) {
defer func() {
if r := recover(); r != nil {
@@ -64,11 +66,15 @@ func (s *AutomationService) execute(parent context.Context, run *types.Automatio
}()
return s.runOnce(ctx, run)
}()
- if ctx.Err() != nil && out.status != types.AutomationRunAwaitingApproval {
+ out.elapsed = time.Since(started)
+ if ctx.Err() != nil {
var row types.AutomationRun
if err := s.db.WithContext(context.WithoutCancel(ctx)).Select("cancel_requested").
Where("id = ?", run.ID).Take(&row).Error; err == nil && row.CancelRequested {
out.status, out.reason = types.AutomationRunCancelled, "cancelled by the owner"
+ if out.promptID != "" {
+ s.closePrompt(context.WithoutCancel(ctx), out.promptID, types.AgentPromptCanceled, out.reason)
+ }
}
}
s.record(ctx, run, out)
@@ -109,7 +115,10 @@ func (s *AutomationService) heartbeat(ctx context.Context, runID string, cancel
func (s *AutomationService) record(ctx context.Context, run *types.AutomationRun, out runOutcome) {
ctx = context.WithoutCancel(ctx)
run.Answer = clipRunes(out.answer, maxStoredAnswerRunes)
- updates := map[string]any{"answer": run.Answer, "updated_at": s.now()}
+ updates := map[string]any{
+ "answer": run.Answer, "updated_at": s.now(),
+ "execution_duration_ms": gorm.Expr("execution_duration_ms + ?", out.elapsed.Milliseconds()),
+ }
if out.usage != nil {
if data, err := json.Marshal(out.usage); err == nil {
updates["usage"] = types.JSON(data)
@@ -204,6 +213,7 @@ func (s *AutomationService) ownerContext(ctx context.Context, a *types.Automatio
// runOnce executes run, or continues it from its answered approval.
func (s *AutomationService) runOnce(ctx context.Context, run *types.AutomationRun) runOutcome {
+ started := time.Now()
if s.sessions == nil {
return failed("the agent executor is unavailable")
}
@@ -234,7 +244,14 @@ func (s *AutomationService) runOnce(ctx context.Context, run *types.AutomationRu
if m := a.Limits.MaxDurationMinutes; m > 0 {
limit = time.Duration(m) * time.Minute
}
- ctx, cancel := context.WithTimeout(ctx, limit)
+ remaining := limit - time.Duration(run.ExecutionDurationMS)*time.Millisecond - time.Since(started)
+ if remaining <= 0 {
+ if run.Resume && run.PendingPromptID != "" {
+ s.closePrompt(ctx, run.PendingPromptID, types.AgentPromptFailed, "execution time limit exceeded")
+ }
+ return failed(fmt.Sprintf("the run took longer than %s", limit))
+ }
+ ctx, cancel := context.WithTimeout(ctx, remaining)
defer cancel()
var resume *types.AgentResume
@@ -524,7 +541,10 @@ func (s *AutomationService) agentQA(
}
mu.Lock()
defer mu.Unlock()
- answer.IsCompleted = suspended == nil
+ answer.IsCompleted = suspended == nil || err != nil
+ if answer.IsCompleted {
+ answer.RunState = ""
+ }
answer.UpdatedAt = s.now()
if err != nil && answer.Content == "" {
answer.Content = err.Error()
diff --git a/internal/application/service/automation_scheduler.go b/internal/application/service/automation_scheduler.go
index 6c2196c70..c248c2ff2 100644
--- a/internal/application/service/automation_scheduler.go
+++ b/internal/application/service/automation_scheduler.go
@@ -163,6 +163,9 @@ func (s *AutomationService) startQueued(ctx context.Context, now time.Time) erro
}
var queued []*types.AutomationRun
if err := s.db.WithContext(ctx).Where("status = ?", types.AutomationRunQueued).
+ Where(`NOT EXISTS (SELECT 1 FROM automation_runs AS holding
+ WHERE holding.automation_id = automation_runs.automation_id AND holding.status IN ? )`,
+ []string{types.AutomationRunRunning, types.AutomationRunAwaitingApproval}).
Order("resume DESC, scheduled_at").Limit(free * 4).Find(&queued).Error; err != nil {
return err
}
@@ -186,16 +189,6 @@ func (s *AutomationService) startQueued(ctx context.Context, now time.Time) erro
}
func (s *AutomationService) startRun(ctx context.Context, run *types.AutomationRun, now time.Time) (bool, error) {
- var holding int64
- if err := s.db.WithContext(ctx).Model(&types.AutomationRun{}).
- Where("automation_id = ? AND id <> ? AND status IN ?", run.AutomationID, run.ID, []string{
- types.AutomationRunRunning, types.AutomationRunAwaitingApproval,
- }).Count(&holding).Error; err != nil {
- return false, err
- }
- if holding > 0 {
- return false, nil
- }
lease := now.Add(s.cfg.Lease)
updates := map[string]any{
"status": types.AutomationRunRunning, "owner": s.instance, "lease_until": lease, "updated_at": now,
@@ -203,10 +196,33 @@ func (s *AutomationService) startRun(ctx context.Context, run *types.AutomationR
if run.StartedAt == nil {
updates["started_at"] = now
}
- res := s.db.WithContext(ctx).Model(&types.AutomationRun{}).
- Where("id = ? AND status = ?", run.ID, types.AutomationRunQueued).Updates(updates)
- if res.Error != nil || res.RowsAffected == 0 {
- return false, res.Error
+ started := false
+ err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ // Take the automation's write lock before reading its runs. A run-row
+ // CAS alone cannot serialize two different queued runs. Writing first
+ // also avoids a deferred read-to-write transaction upgrade on SQLite.
+ locked := tx.Model(&types.Automation{}).Where("id = ?", run.AutomationID).
+ UpdateColumn("version", gorm.Expr("version + 1"))
+ if locked.Error != nil || locked.RowsAffected == 0 {
+ return locked.Error
+ }
+ var holding int64
+ if err := tx.Model(&types.AutomationRun{}).
+ Where("automation_id = ? AND id <> ? AND status IN ?", run.AutomationID, run.ID, []string{
+ types.AutomationRunRunning, types.AutomationRunAwaitingApproval,
+ }).Count(&holding).Error; err != nil {
+ return err
+ }
+ if holding > 0 {
+ return nil
+ }
+ res := tx.Model(&types.AutomationRun{}).
+ Where("id = ? AND status = ?", run.ID, types.AutomationRunQueued).Updates(updates)
+ started = res.Error == nil && res.RowsAffected > 0
+ return res.Error
+ })
+ if err != nil || !started {
+ return false, err
}
run.Status, run.Owner, run.LeaseUntil = types.AutomationRunRunning, s.instance, &lease
if run.StartedAt == nil {
@@ -266,20 +282,31 @@ func (s *AutomationService) reconcileAwaiting(ctx context.Context, now time.Time
if s.broker == nil {
return nil
}
- var waiting []*types.AutomationRun
- if err := s.db.WithContext(ctx).Where("status = ?", types.AutomationRunAwaitingApproval).
- Order("updated_at").Limit(automationClaimBatch * 2).Find(&waiting).Error; err != nil {
- return err
- }
- for _, run := range waiting {
- if err := s.reconcileRun(ctx, run, now); err != nil {
- logger.Warnf(ctx, "Reconcile automation run %s: %v", run.ID, err)
+ // Keyset pagination visits later approvals even when earlier ones stay
+ // open. Offsets would skip rows as reconciled runs leave the waiting set.
+ for cursor := ""; ; {
+ var waiting []*types.AutomationRun
+ if err := s.db.WithContext(ctx).
+ Where("status = ? AND id > ?", types.AutomationRunAwaitingApproval, cursor).
+ Order("id").Limit(automationClaimBatch * 2).Find(&waiting).Error; err != nil {
+ return err
+ }
+ for _, run := range waiting {
+ if err := s.reconcileRun(ctx, run, now); err != nil {
+ logger.Warnf(ctx, "Reconcile automation run %s: %v", run.ID, err)
+ }
}
+ if len(waiting) < automationClaimBatch*2 {
+ return nil
+ }
+ cursor = waiting[len(waiting)-1].ID
}
- return nil
}
func (s *AutomationService) reconcileRun(ctx context.Context, run *types.AutomationRun, now time.Time) error {
+ if run.CancelRequested {
+ return s.cancelRun(ctx, run, "cancelled by the owner")
+ }
waiting := []string{types.AutomationRunAwaitingApproval}
prompt, err := s.broker.Store().Get(ctx, run.PendingPromptID)
if err != nil {
@@ -324,7 +351,8 @@ func (s *AutomationService) reconcileRun(ctx context.Context, run *types.Automat
// queue, to continue from the answer.
func (s *AutomationService) queueResume(ctx context.Context, runID, promptID string) error {
res := s.db.WithContext(ctx).Model(&types.AutomationRun{}).
- Where("id = ? AND status = ? AND pending_prompt_id = ?", runID, types.AutomationRunAwaitingApproval, promptID).
+ Where("id = ? AND status = ? AND pending_prompt_id = ? AND cancel_requested = ?",
+ runID, types.AutomationRunAwaitingApproval, promptID, false).
Updates(map[string]any{"status": types.AutomationRunQueued, "resume": true, "updated_at": s.now()})
if res.Error == nil && res.RowsAffected > 0 {
s.Wake()
@@ -407,8 +435,15 @@ func (s *AutomationService) finishRun(
return false, res.Error
}
run.Status, run.Error, run.FinishedAt = status, reason, &now
+ // Only the winning finalizer may stop work. A stale approval reaper must
+ // not cancel a run that another scheduler has already resumed.
+ var stopErr error
+ if status == types.AutomationRunFailed || status == types.AutomationRunCancelled ||
+ status == types.AutomationRunNeedsAttention {
+ stopErr = s.stopRunDelegation(context.WithoutCancel(ctx), run)
+ }
s.settle(ctx, run)
- return true, nil
+ return true, stopErr
}
// settle records a finished run on its automation, pauses it after too many
diff --git a/internal/application/service/session_agent_qa.go b/internal/application/service/session_agent_qa.go
index 22db2983f..92448798d 100644
--- a/internal/application/service/session_agent_qa.go
+++ b/internal/application/service/session_agent_qa.go
@@ -431,6 +431,7 @@ func (s *sessionService) AgentQA(
// run elsewhere. Session, attachments and images are reloaded from storage.
func resumableAgentRequest(ctx context.Context, req *types.QARequest) types.ResumableAgentRequest {
r := types.ResumableAgentRequest{
+ ActingVia: types.ActingViaFromContext(ctx),
Query: req.Query,
SummaryModelID: req.SummaryModelID,
ReasoningEffort: req.ReasoningEffort,
diff --git a/internal/container/container.go b/internal/container/container.go
index a8fd408d0..8121e8d9b 100644
--- a/internal/container/container.go
+++ b/internal/container/container.go
@@ -655,6 +655,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
}
automations.Bind(sessionService, broker, runs, streams)
automations.Events = sessions.CollaborationEvents
+ automations.StopDelegation = collaboration.StopOwned
broker.SetRunHooks(runs.Hooks())
runs.StopDelegation = func(ctx context.Context, p *types.AgentPrompt) error {
return collaboration.StopPrompt(ctx, p)
diff --git a/internal/database/migration_source_test.go b/internal/database/migration_source_test.go
new file mode 100644
index 000000000..e2be2cf51
--- /dev/null
+++ b/internal/database/migration_source_test.go
@@ -0,0 +1,22 @@
+package database
+
+import (
+ "path/filepath"
+ "testing"
+
+ "github.com/golang-migrate/migrate/v4/source"
+ "github.com/stretchr/testify/require"
+)
+
+// Check both migration registries without requiring a live PostgreSQL server.
+// Independently merged migrations must not reuse a version in either dialect.
+func TestMigrationSourcesHaveUniqueVersions(t *testing.T) {
+ root := sqliteRepoRoot(t)
+ for _, dialect := range []string{"sqlite", "versioned"} {
+ t.Run(dialect, func(t *testing.T) {
+ migrations, err := source.Open("file://" + filepath.Join(root, "migrations", dialect))
+ require.NoError(t, err)
+ require.NoError(t, migrations.Close())
+ })
+ }
+}
diff --git a/internal/database/migration_sqlite_automation_duration_test.go b/internal/database/migration_sqlite_automation_duration_test.go
new file mode 100644
index 000000000..bd357aa9a
--- /dev/null
+++ b/internal/database/migration_sqlite_automation_duration_test.go
@@ -0,0 +1,36 @@
+package database
+
+import (
+ "fmt"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestSQLiteAutomationDurationUpgradePreservesExistingRuns(t *testing.T) {
+ repoRoot := sqliteRepoRoot(t)
+ for _, version := range []int{45, 47} {
+ t.Run(fmt.Sprintf("from-v%d", version), func(t *testing.T) {
+ chdirAndRestore(t, copySQLiteMigrationsThrough(t, repoRoot, version))
+ dbPath := filepath.Join(t.TempDir(), "automations.db")
+ require.NoError(t, RunMigrationsWithOptions("sqlite3://unused", MigrationOptions{SQLiteDBPath: dbPath}))
+ db := openSQLiteDB(t, dbPath)
+ _, err := db.Exec(`INSERT INTO automation_runs
+ (id, automation_id, tenant_id, scheduled_at, trigger_kind, status, answer, created_at, updated_at)
+ VALUES ('existing-run', 'automation', 1, CURRENT_TIMESTAMP, 'schedule',
+ 'awaiting_approval', 'saved answer', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
+ require.NoError(t, err)
+
+ chdirAndRestore(t, repoRoot)
+ require.NoError(t, RunMigrationsWithOptions("sqlite3://unused", MigrationOptions{SQLiteDBPath: dbPath}))
+ var duration int64
+ var answer, status string
+ require.NoError(t, db.QueryRow(`SELECT execution_duration_ms, answer, status
+ FROM automation_runs WHERE id = 'existing-run'`).Scan(&duration, &answer, &status))
+ require.Zero(t, duration)
+ require.Equal(t, "saved answer", answer)
+ require.Equal(t, "awaiting_approval", status)
+ })
+ }
+}
diff --git a/internal/database/migration_sqlite_versioned_schema_test.go b/internal/database/migration_sqlite_versioned_schema_test.go
index 043b976da..5e72bf994 100644
--- a/internal/database/migration_sqlite_versioned_schema_test.go
+++ b/internal/database/migration_sqlite_versioned_schema_test.go
@@ -52,6 +52,7 @@ var versionedSQLiteTables = []string{
// versionedSQLiteColumns maps each existing table to the columns that the
// versioned migrations add and the SQLite baseline was missing.
var versionedSQLiteColumns = map[string][]string{
+ "automation_runs": {"execution_duration_ms"}, // 000048
"model_catalog_configs": {"version", "overlay", "history", "updated_by", "updated_at"}, // 000031
"memory_subjects": {"extraction_state"}, // 000094
"memory_items": {"replaces_id"}, // 000094
@@ -86,7 +87,7 @@ var versionedSQLiteColumns = map[string][]string{
}, // 000028
}
-const expectedSQLiteMigrationVersion = 47
+const expectedSQLiteMigrationVersion = 48
func TestSQLiteMigrationsCreateVersionedSchema(t *testing.T) {
repoRoot := sqliteRepoRoot(t)
@@ -262,19 +263,20 @@ func TestSQLiteSandboxAllocationMigrationRollsBack(t *testing.T) {
require.NoError(t, err)
t.Cleanup(func() { _, _ = m.Close() })
- require.NoError(t, m.Steps(-1))
+ // Target the allocation migration even when newer migrations exist.
+ require.NoError(t, m.Migrate(46))
version, dirty = sqliteMigrationState(t, sqlDB)
- require.Equal(t, expectedSQLiteMigrationVersion-1, version)
+ require.Equal(t, 46, version)
require.False(t, dirty)
for _, table := range []string{"session_sandbox_assignments", "sandbox_user_machines"} {
require.Falsef(t, sqliteTableExists(t, sqlDB, table), "down must drop %s", table)
}
require.True(t, sqliteColumnExists(t, sqlDB, "sessions", "owner_auth_source"),
- "rolling back the allocation tables keeps the owner source of 000034")
+ "rolling back the allocation tables keeps the owner source of 000046")
- require.NoError(t, m.Steps(1))
+ require.NoError(t, m.Migrate(47))
version, dirty = sqliteMigrationState(t, sqlDB)
- require.Equal(t, expectedSQLiteMigrationVersion, version)
+ require.Equal(t, 47, version)
require.False(t, dirty)
require.True(t, sqliteTableExists(t, sqlDB, "session_sandbox_assignments"))
require.True(t, sqliteTableExists(t, sqlDB, "sandbox_user_machines"))
diff --git a/internal/types/agent_prompt.go b/internal/types/agent_prompt.go
index 1365f3b4f..8a47fbede 100644
--- a/internal/types/agent_prompt.go
+++ b/internal/types/agent_prompt.go
@@ -168,8 +168,10 @@ func (p *AgentPrompt) TimelineView() *PendingPromptView {
// ResumableAgentRequest is the part of a QARequest needed to rebuild the run
// on another replica. It holds the original query and durable request settings.
type ResumableAgentRequest struct {
- AgentID string `json:"agent_id"`
- Query string `json:"query,omitempty"`
+ // ActingVia keeps scheduler-owned runs out of ordinary chat recovery.
+ ActingVia string `json:"acting_via,omitempty"`
+ AgentID string `json:"agent_id"`
+ Query string `json:"query,omitempty"`
// AgentTenantID is the agent owner's workspace; it differs from the
// session's when the agent is shared.
AgentTenantID uint64 `json:"agent_tenant_id,omitempty"`
diff --git a/internal/types/automation.go b/internal/types/automation.go
index a88ba78ac..316762fed 100644
--- a/internal/types/automation.go
+++ b/internal/types/automation.go
@@ -170,6 +170,9 @@ type AutomationRun struct {
Resume bool `json:"-" gorm:"not null;default:false"`
// CancelRequested stops the run at its next check.
CancelRequested bool `json:"cancel_requested" gorm:"not null;default:false"`
+ // ExecutionDurationMS accumulates execution segments across approvals;
+ // queued time and time spent waiting for an answer do not consume it.
+ ExecutionDurationMS int64 `json:"-" gorm:"not null;default:0"`
// Answer is the start of the final answer, for listings.
Answer string `json:"answer" gorm:"type:text;not null;default:''" extensions:"x-agent,x-agent-response"`
Error string `json:"error" gorm:"type:text;not null;default:''" extensions:"x-agent,x-agent-response"`
diff --git a/migrations/sqlite/000048_automation_execution_duration.down.sql b/migrations/sqlite/000048_automation_execution_duration.down.sql
new file mode 100644
index 000000000..1193a7951
--- /dev/null
+++ b/migrations/sqlite/000048_automation_execution_duration.down.sql
@@ -0,0 +1 @@
+ALTER TABLE automation_runs DROP COLUMN execution_duration_ms;
diff --git a/migrations/sqlite/000048_automation_execution_duration.up.sql b/migrations/sqlite/000048_automation_execution_duration.up.sql
new file mode 100644
index 000000000..1d44f81c9
--- /dev/null
+++ b/migrations/sqlite/000048_automation_execution_duration.up.sql
@@ -0,0 +1 @@
+ALTER TABLE automation_runs ADD COLUMN execution_duration_ms BIGINT NOT NULL DEFAULT 0;
diff --git a/migrations/versioned/000131_automation_execution_duration.down.sql b/migrations/versioned/000131_automation_execution_duration.down.sql
new file mode 100644
index 000000000..1193a7951
--- /dev/null
+++ b/migrations/versioned/000131_automation_execution_duration.down.sql
@@ -0,0 +1 @@
+ALTER TABLE automation_runs DROP COLUMN execution_duration_ms;
diff --git a/migrations/versioned/000131_automation_execution_duration.up.sql b/migrations/versioned/000131_automation_execution_duration.up.sql
new file mode 100644
index 000000000..1d44f81c9
--- /dev/null
+++ b/migrations/versioned/000131_automation_execution_duration.up.sql
@@ -0,0 +1 @@
+ALTER TABLE automation_runs ADD COLUMN execution_duration_ms BIGINT NOT NULL DEFAULT 0;