Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(automation): enforce execution identity, cancellation, and scheduling limits (#4506)
lyingbuga8eb7e28fa40
diff --git a/internal/application/service/agent_collaboration.go b/internal/application/service/agent_collaboration.goindex 197814623..6e9ca5124 100644--- a/internal/application/service/agent_collaboration.go+++ b/internal/application/service/agent_collaboration.go@@ -184,12 +184,37 @@ func (s *AgentCollaboration) prune(ctx context.Context) error {type delegatedConfig struct {Agent types.CustomAgent `json:"agent"`Access types.AssistantAccessLevel `json:"access"`+ // Durable execution identity follows children across scheduler instances+ // and approval resumes; it must not turn an automation into a human turn.+ ActingVia string `json:"acting_via,omitempty"`+ Mode types.RunMode `json:"mode,omitempty"`// AssistantParent records that the delegating run was the WeKnora// assistant, whose level stays under the workspace's assistant ceiling.AssistantParent bool `json:"assistant_parent,omitempty"`Inherited *inheritedAgentConfig `json:"inherited,omitempty"`}+func (d *delegatedConfig) captureExecution(ctx context.Context) {+ d.ActingVia = types.ActingViaFromContext(ctx)+ d.Mode = types.RunModeAttended+ if mode, ok := ctx.Value(types.RunModeContextKey).(types.RunMode); ok {+ d.Mode = mode.Normalize()+ }+ if d.ActingVia != "" {+ d.Mode = types.RunModeDeferred+ }+}++func (d *delegatedConfig) runMode() types.RunMode {+ if d.ActingVia != "" {+ return types.RunModeDeferred+ }+ if d.Mode == "" {+ return types.RunModeAttended // snapshots created before mode was stored+ }+ return d.Mode.Normalize()+}+// level is the child's access level: the delegating run's, capped by the// child agent's own ceiling and, under an assistant parent, by the// workspace's current assistant ceiling.@@ -222,7 +247,9 @@ func (s *AgentCollaboration) resolveDelegated(id == types.BuiltinSkillInstallerID {return "", errors.New("only same-workspace reasoning agents without management privileges may be delegated")}- data, err := json.Marshal(delegatedConfig{Agent: *a, Access: level, AssistantParent: assistantParent})+ saved := delegatedConfig{Agent: *a, Access: level, AssistantParent: assistantParent}+ saved.captureExecution(ctx)+ data, err := json.Marshal(saved)return string(data), err}@@ -377,7 +404,11 @@ func (s *AgentCollaboration) workerContext(ctx context.Context, scope collaboratctx = context.WithValue(ctx, types.TenantRoleContextKey, membership.Role)ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant)ctx = types.WithCaller(ctx, types.Caller{TenantID: scope.TenantID, UserID: scope.UserID, Role: membership.Role})- ctx = types.WithPrincipal(ctx, types.Principal{Type: types.PrincipalWebUser, ID: scope.UserID})+ principal := types.Principal{Type: types.PrincipalWebUser, ID: scope.UserID}+ if previous, ok := types.PrincipalFromContext(ctx); ok && previous.ID == scope.UserID {+ principal.ActingVia = previous.ActingVia+ }+ ctx = types.WithPrincipal(ctx, principal)return ctx, nil}@@ -397,6 +428,10 @@ func (s *AgentCollaboration) run(ctx context.Context, w collaboration.Work) (colif err = json.Unmarshal([]byte(w.Member.ConfigJSON), &saved); err != nil {return result, err}+ ctx = types.WithPrincipal(ctx, types.Principal{+ Type: types.PrincipalWebUser, ID: w.Scope.UserID, ActingVia: saved.ActingVia,+ })+ ctx = types.WithRunMode(ctx, saved.runMode())// Both modes pin configuration, but reauthorize the source before each turn.if saved.Inherited == nil {if _, err = s.resolve(ctx, w.Member.AgentID, types.AssistantAccessReadOnly); err != nil {@@ -610,7 +645,7 @@ func (s *AgentCollaboration) run(ctx context.Context, w collaboration.Work) (colctx,&types.QARequest{Resume: resume,- Mode: types.RunModeAttended,+ Mode: saved.runMode(),GrantSessionID: w.Scope.SessionID,Suspendable: true,Session: session,diff --git a/internal/application/service/agent_collaboration_inherited.go b/internal/application/service/agent_collaboration_inherited.goindex e2d8e8064..b2b5211bf 100644--- a/internal/application/service/agent_collaboration_inherited.go+++ b/internal/application/service/agent_collaboration_inherited.go@@ -75,10 +75,12 @@ func inheritedChildSnapshot(ctx context.Context, cfg *types.AgentConfig, toolNam}}// Marshal immediately: later changes to parent config cannot change queued work.- data, err := json.Marshal(delegatedConfig{+ saved := delegatedConfig{Agent: *source.Agent, Access: cfg.AssistantAccess, AssistantParent: cfg.OperationsAgentID != "",Inherited: inherited,- })+ }+ saved.captureExecution(ctx)+ data, err := json.Marshal(saved)return string(data), err}diff --git a/internal/application/service/agent_collaboration_recovery.go b/internal/application/service/agent_collaboration_recovery.goindex 223f2c3c5..9ba5f3c71 100644--- a/internal/application/service/agent_collaboration_recovery.go+++ b/internal/application/service/agent_collaboration_recovery.go@@ -223,6 +223,11 @@ func (s *AgentCollaboration) resumeParent(ctx context.Context, b *collaboration.if err = json.Unmarshal(snap.Request, &saved); err != nil {return err}+ if saved.ActingVia != "" {+ // Automation leases, cancellation and duration belong to their own+ // scheduler. Never revive one through ordinary parent recovery.+ return errors.New("scheduler-owned parent cannot resume through collaboration recovery")+ }if saved.AgentID == "" || saved.SharedAgentReadOnly {return errors.New("invalid parent recovery identity")}diff --git a/internal/application/service/automation.go b/internal/application/service/automation.goindex f5f07d7e9..4f24789e0 100644--- a/internal/application/service/automation.go+++ b/internal/application/service/automation.go@@ -80,6 +80,8 @@ type AutomationService struct {// Events publishes a run to its message's event stream; see// AgentCollaboration.ChildEvents.Events func(context.Context, *types.Session, *types.Message, *event.EventBus) func(context.Context) error+ // StopDelegation fences children and discards recoverable parent waits.+ StopDelegation func(context.Context, uint64, string, string, string) errorcfg AutomationConfiginstance string@@ -621,16 +623,37 @@ func (s *AutomationService) cancelRun(ctx context.Context, run *types.AutomationUpdate("cancel_requested", true).Error; err != nil {return err}+ stopErr := s.stopRunDelegation(ctx, run)s.mu.Lock()cancel := s.running[run.ID]s.mu.Unlock()if cancel != nil {cancel()}+ return stopErr}return nil}+func (s *AutomationService) stopRunDelegation(ctx context.Context, run *types.AutomationRun) error {+ if s.StopDelegation == nil {+ return nil+ }+ // The caller may hold the row from before the executor created its turn.+ var current types.AutomationRun+ if err := s.db.WithContext(ctx).Where("id = ?", run.ID).Take(¤t).Error; err != nil {+ return err+ }+ if current.SessionID == "" || current.AssistantMessageID == "" {+ return nil+ }+ a, err := s.automationOf(ctx, ¤t)+ if err != nil {+ return err+ }+ return s.StopDelegation(ctx, a.TenantID, a.OwnerUserID, current.SessionID, current.AssistantMessageID)+}+// closePrompt ends the open prompt id a run waits on and completes the// run's message, as the prompt reaper would.func (s *AutomationService) closePrompt(ctx context.Context, id, status, reason string) {diff --git a/internal/application/service/automation_regression_test.go b/internal/application/service/automation_regression_test.gonew file mode 100644index 000000000..aa22ad3c2--- /dev/null+++ b/internal/application/service/automation_regression_test.go@@ -0,0 +1,462 @@+package service++import (+ "context"+ "encoding/json"+ "fmt"+ "strings"+ "sync"+ "testing"+ "time"++ "github.com/Tencent/WeKnora/internal/agent"+ "github.com/Tencent/WeKnora/internal/agent/collaboration"+ "github.com/Tencent/WeKnora/internal/event"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/stretchr/testify/require"+ "gorm.io/driver/sqlite"+ "gorm.io/gorm"+ "gorm.io/gorm/logger"+)++func TestAutomationDelegatedExecutionKeepsIdentity(t *testing.T) {+ for _, inherited := range []bool{false, true} {+ t.Run(fmt.Sprintf("inherited=%t", inherited), func(t *testing.T) {+ svc, agents, _, ctx := collaborationService(t)+ via := types.AutomationActingVia("scheduled-task")+ parent := types.WithPrincipal(ctx, types.Principal{Type: types.PrincipalWebUser, ID: "u", ActingVia: via})+ parent = types.WithRunMode(parent, types.RunModeDeferred)+ var snapshot string+ var err error+ if inherited {+ parent = context.WithValue(parent, childSourceKey{}, childSource{Agent: agents.agent, ModelID: "model"})+ snapshot, err = inheritedChildSnapshot(+ parent,+ &types.AgentConfig{AssistantAccess: types.AssistantAccessAutoWrite},+ nil,+ )+ } else {+ snapshot, err = svc.resolve(parent, "a", types.AssistantAccessAutoWrite)+ }+ require.NoError(t, err)+ scope := collaboration.Scope{TenantID: automationTenant, UserID: "u", SessionID: "parent", ID: "answer"}+ require.NoError(t, svc.Runtime.Store.Ensure(ctx, scope))+ _, err = svc.Runtime.Spawn(ctx, scope, "spawn", "child", "a", snapshot, "work")+ require.NoError(t, err)+ require.NoError(t, svc.Runtime.Store.Change(ctx, scope, func(g *collaboration.Group) error {+ r := g.Members[0].Runs[0]+ r.State = "running"+ r.Owner = "worker"+ r.Lease = time.Now().Add(time.Minute)+ return nil+ }))+ called := false+ svc.Bind(+ &collaborationSessions{+ run: func(child context.Context, req *types.QARequest, bus *event.EventBus) error {+ called = true+ require.Equal(t, via, types.ActingViaFromContext(child))+ require.Equal(t, types.RunModeDeferred, req.Mode)+ require.False(t, req.Mode.Attended(), "ask_user must stay unavailable")+ _, _, err := automationOwner(child)+ require.Error(t, err, "delegation must not bypass the automation management guard")+ return complete(child, bus, event.AgentCompleteData{FinalAnswer: "done"})+ },+ },+ )+ g, err := svc.Runtime.Store.Read(ctx, scope)+ require.NoError(t, err)+ // A scheduler starts from its own context, not the parent's request context.+ _, err = svc.run(+ context.Background(),+ collaboration.Work{Scope: scope, Member: *g.Members[0], Run: *g.Members[0].Runs[0]},+ )+ require.NoError(t, err)+ require.True(t, called)+ })+ }+}++func TestAutomationCancelStopsDelegationAndRecovery(t *testing.T) {+ f := newAutomationFixture(t)+ require.NoError(t, f.db.AutoMigrate(&collaboration.Record{}))+ delegated := NewAgentCollaboration(f.db, f.agents, f.messages, f.users, f.members, f.svc.tenants)+ f.svc.StopDelegation = delegated.StopOwned+ a := f.create(t, types.AutomationPatch{})+ queued, err := f.svc.RunNow(f.ctx, a.ID)+ require.NoError(t, err)+ entered := make(chan collaboration.Scope, 1)+ f.sessions.run = func(ctx context.Context, req *types.QARequest, _ *event.EventBus) error {+ scope := collaboration.Scope{+ TenantID: automationTenant,+ UserID: "u",+ SessionID: req.Session.ID,+ ID: req.AssistantMessageID,+ }+ b, err := delegated.Runtime.AcquireParent(ctx, scope)+ if err != nil {+ return err+ }+ _, err = delegated.Runtime.Spawn(b.Context(ctx), scope, "spawn", "child", "a", "{}", "work")+ if err != nil {+ return err+ }+ _, err = delegated.Runtime.Spawn(b.Context(ctx), scope, "spawn-running", "running_child", "a", "{}", "work")+ if err != nil {+ return err+ }+ if err = delegated.Runtime.Store.Change(b.Context(ctx), scope, func(g *collaboration.Group) error {+ r := g.Members[1].Runs[0]+ r.State, r.Owner, r.Lease = "running", "remote-worker", time.Now().Add(time.Minute)+ return nil+ }); err != nil {+ return err+ }+ saved := resumableAgentRequest(ctx, req)+ data, err := json.Marshal(+ agent.RunSnapshot{Version: agent.RunSnapshotVersion, Request: mustJSON(t, saved), CollaborationWait: true},+ )+ if err != nil {+ return err+ }+ if err = b.SaveWait(ctx, data, time.Now().Add(time.Minute)); err != nil {+ return err+ }+ entered <- scope+ <-ctx.Done()+ // Exercise the same cleanup as an engine cancelled while in wait_agent.+ b.End(true)+ return ctx.Err()+ }+ require.NoError(t, f.svc.startQueued(f.ctx, f.now))+ var scope collaboration.Scope+ select {+ case scope = <-entered:+ case <-time.After(5 * time.Second):+ t.Fatal("run did not start")+ }+ _, err = f.svc.CancelRun(f.ctx, queued.ID)+ require.NoError(t, err)+ done := f.settleRun(t, queued.ID)+ require.Equal(t, types.AutomationRunCancelled, done.Status)+ g, err := delegated.Runtime.Store.Read(f.ctx, scope)+ require.NoError(t, err)+ require.NotEqual(t, "active", g.Status)+ require.Nil(t, g.ParentWait, "cancelled checkpoints must not be recovered")+ require.True(t, g.Members[0].Runs[0].CancelRequested)+ require.Equal(t, "cancelled", g.Members[0].Runs[0].State)+ require.True(t, g.Members[1].Runs[0].CancelRequested)+ require.ErrorIs(t, delegated.Runtime.CheckOwner(f.ctx, collaboration.Work{+ Scope: scope, Member: *g.Members[1], Run: *g.Members[1].Runs[0],+ }), collaboration.ErrConflict, "a remote child must lose permission to execute tools immediately")+}++func mustJSON(t *testing.T, v any) json.RawMessage {+ t.Helper()+ data, err := json.Marshal(v)+ require.NoError(t, err)+ return data+}++func TestAutomationParentRecoveryDoesNotEscapeScheduler(t *testing.T) {+ svc, _, messages, ctx := collaborationService(t)+ scope := collaboration.Scope{TenantID: automationTenant, UserID: "u", SessionID: "parent", ID: "answer"}+ b, err := svc.Runtime.AcquireParent(ctx, scope)+ require.NoError(t, err)+ t.Cleanup(func() { b.End(false) })+ require.NoError(t, messages.UpdateMessage(ctx, &types.Message{ID: scope.ID, SessionID: scope.SessionID}))+ svc.streams = &collaborationStreams{}+ svc.ParentEvents = func(+ context.Context, *types.Session, *types.Message, *event.EventBus,+ ) func(context.Context) error {+ return func(context.Context) error { return nil }+ }+ called := false+ svc.Bind(+ &collaborationSessions{+ run: func(context.Context, *types.QARequest, *event.EventBus) error { called = true; return nil },+ },+ )+ ctx = types.WithPrincipal(+ ctx,+ types.Principal{Type: types.PrincipalWebUser, ID: "u", ActingVia: types.AutomationActingVia("a")},+ )+ req := resumableAgentRequest(+ ctx,+ &types.QARequest{CustomAgent: &types.CustomAgent{ID: "a"}, Mode: types.RunModeDeferred},+ )+ snapshot := mustJSON(+ t,+ agent.RunSnapshot{Version: agent.RunSnapshotVersion, Request: mustJSON(t, req), CollaborationWait: true},+ )+ err = svc.resumeParent(context.Background(), b, snapshot)+ require.ErrorContains(t, err, "scheduler-owned parent")+ require.False(t, called)+}++func TestAutomationStartSerializesDifferentRunsAcrossInstances(t *testing.T) {+ f := newAutomationFixture(t)+ a := f.create(t, types.AutomationPatch{OverlapPolicy: ptr(types.AutomationPolicyQueue)})+ rows := []*types.AutomationRun{+ {+ AutomationID: a.ID,+ TenantID: automationTenant,+ ScheduledAt: f.now.Add(-time.Hour),+ TriggerKind: types.AutomationRunSchedule,+ Status: types.AutomationRunQueued,+ Resume: true,+ },+ {+ AutomationID: a.ID,+ TenantID: automationTenant,+ ScheduledAt: f.now,+ TriggerKind: types.AutomationRunSchedule,+ Status: types.AutomationRunQueued,+ },+ }+ for _, r := range rows {+ require.NoError(t, f.db.Create(r).Error)+ }+ // A separate connection pool represents another scheduler process.+ db, err := gorm.Open(+ sqlite.Open(f.db.Dialector.(*sqlite.Dialector).DSN),+ &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)},+ )+ require.NoError(t, err)+ raw, err := db.DB()+ require.NoError(t, err)+ t.Cleanup(func() { _ = raw.Close() })+ other := &AutomationService{db: db, instance: "second-instance"}+ arrived := make(chan struct{}, 2)+ release := make(chan struct{})+ for _, db := range []*gorm.DB{f.db, db} {+ require.NoError(t, db.Callback().Query().After("gorm:query").Register("admission_barrier", func(db *gorm.DB) {+ if n, ok := db.Statement.Dest.(*int64); ok && *n == 0 && db.Statement.Table == "automation_runs" &&+ strings.Contains(db.Statement.SQL.String(), "count(*)") {+ arrived <- struct{}{}+ <-release+ }+ }))+ }+ var wg sync.WaitGroup+ wg.Add(2)+ started := make([]bool, 2)+ errs := make([]error, 2)+ go func() { defer wg.Done(); started[0], errs[0] = f.svc.startRun(f.ctx, rows[0], f.now) }()+ select {+ case <-arrived:+ case <-time.After(5 * time.Second):+ close(release)+ t.Fatal("first admission did not reach check")+ }+ go func() { defer wg.Done(); started[1], errs[1] = other.startRun(f.ctx, rows[1], f.now) }()+ // The old implementation lets both count queries observe zero. With the+ // automation lock the second query cannot run until the first commits.+ select {+ case <-arrived:+ case <-time.After(100 * time.Millisecond):+ }+ close(release)+ wg.Wait()+ require.NoError(t, errs[0])+ require.NoError(t, errs[1])+ require.True(t, started[0])+ require.False(t, started[1])+ var active int64+ require.NoError(+ t,+ f.db.Model(&types.AutomationRun{}).+ Where("automation_id = ? AND status = ?", a.ID, types.AutomationRunRunning).+ Count(&active).+ Error,+ )+ require.EqualValues(t, 1, active)+}++func TestAutomationQueuedRunsSkipBlockedHead(t *testing.T) {+ f := newAutomationFixture(t)+ for i := 0; i < 16; i++ {+ a := f.create(+ t,+ types.AutomationPatch{+ Name: ptr(fmt.Sprintf("blocked%d", i)),+ OverlapPolicy: ptr(types.AutomationPolicyQueue),+ },+ )+ for j, status := range []string{types.AutomationRunAwaitingApproval, types.AutomationRunQueued} {+ require.NoError(+ t,+ f.db.Create(+ &types.AutomationRun{+ AutomationID: a.ID,+ TenantID: automationTenant,+ ScheduledAt: f.now.Add(time.Duration(j-2) * time.Hour),+ TriggerKind: types.AutomationRunSchedule,+ Status: status,+ },+ ).Error,+ )+ }+ }+ a := f.create(t, types.AutomationPatch{Name: ptr("runnable")})+ r, err := f.svc.RunNow(f.ctx, a.ID)+ require.NoError(t, err)+ require.NoError(t, f.svc.startQueued(context.Background(), f.now))+ require.Eventually(t, func() bool {+ var row types.AutomationRun+ return f.db.First(&row, "id = ?", r.ID).Error == nil && row.Status == types.AutomationRunSucceeded &&+ len(f.svc.slots) == 0+ }, 5*time.Second, 10*time.Millisecond)+}++func TestAutomationApprovalReconciliationVisitsEveryBatch(t *testing.T) {+ f := newAutomationFixture(t)+ f.svc.cfg.MaxPerTenant = 200+ var expired, answered *types.AutomationRun+ for i := 0; i < 102; i++ {+ timeout := 10080+ updated := f.now.Add(-time.Hour)+ if i == 100 {+ timeout = 5+ updated = f.now.Add(-10 * time.Minute)+ }+ a := f.create(+ t,+ types.AutomationPatch{Name: ptr(fmt.Sprintf("approval%d", i)), ApprovalTimeoutMinutes: ptr(timeout)},+ )+ status := types.AgentPromptSuspended+ if i == 0 || i == 101 {+ status = types.AgentPromptAnswered+ }+ p := &types.AgentPrompt{ID: fmt.Sprintf("p%d", i), Status: status}+ require.NoError(t, f.broker.Store().Create(f.ctx, p))+ r := &types.AutomationRun{+ ID: fmt.Sprintf("run-%03d", i),+ AutomationID: a.ID,+ TenantID: automationTenant,+ ScheduledAt: updated,+ TriggerKind: types.AutomationRunSchedule,+ Status: types.AutomationRunAwaitingApproval,+ PendingPromptID: p.ID,+ UpdatedAt: updated,+ }+ require.NoError(t, f.db.Create(r).Error)+ if i == 100 {+ expired = r+ }+ if i == 101 {+ answered = r+ }+ }+ require.NoError(t, f.svc.reconcileAwaiting(f.ctx, f.now))+ require.NoError(t, f.db.First(expired, "id = ?", expired.ID).Error)+ require.Equal(t, types.AutomationRunFailed, expired.Status)+ require.NoError(t, f.db.First(answered, "id = ?", answered.ID).Error)+ require.Equal(t, types.AutomationRunQueued, answered.Status)+ require.True(t, answered.Resume)+}++func TestAutomationApprovalResumeUsesRemainingExecutionBudget(t *testing.T) {+ f := newAutomationFixture(t)+ f.svc.cfg.DefaultMaxDuration = time.Second+ a := f.create(t, types.AutomationPatch{})+ f.suspendOnApproval(t, "op:x")+ original := f.sessions.run+ f.sessions.run = func(ctx context.Context, req *types.QARequest, bus *event.EventBus) error {+ select {+ case <-ctx.Done():+ return ctx.Err()+ case <-time.After(600 * time.Millisecond):+ }+ return original(ctx, req, bus)+ }+ queued, err := f.svc.RunNow(f.ctx, a.ID)+ require.NoError(t, err)+ suspended := f.settleRun(t, queued.ID)+ require.Equal(t, types.AutomationRunAwaitingApproval, suspended.Status)+ // Approval waiting is free, even across service restarts.+ f.now = f.now.Add(12 * time.Hour)+ restarted := NewAutomationService(+ f.db,+ f.agents,+ f.users,+ f.members,+ f.svc.tenants,+ f.messages,+ f.svc.grants,+ f.svc.notices,+ )+ restarted.Configure(AutomationConfig{DefaultMaxDuration: time.Second})+ restarted.now = func() time.Time { return f.now }+ restarted.Bind(f.sessions, f.broker, nil, nil)+ f.svc = restarted+ ok, err := f.broker.Store().Transition(f.ctx, suspended.PendingPromptID, types.AgentPromptTransition{+ From: []string{types.AgentPromptSuspended},+ To: types.AgentPromptAnswered,+ Response: types.JSON(`{"approved":true}`),+ })+ require.NoError(t, err)+ require.True(t, ok)+ resumed := f.settleRun(t, queued.ID)+ require.Equal(t, types.AutomationRunFailed, resumed.Status)+ require.Contains(t, resumed.Error, "longer than 1s")+ require.NotNil(t, f.sessions.last().req.Resume, "waiting time must not exhaust the execution budget")+}++func TestAutomationStaleFinalizerDoesNotCancelResumedDelegation(t *testing.T) {+ f := newAutomationFixture(t)+ a := f.create(t, types.AutomationPatch{})+ r, err := f.svc.RunNow(f.ctx, a.ID)+ require.NoError(t, err)+ require.NoError(t, f.db.Model(r).Updates(map[string]any{+ "status": types.AutomationRunRunning, "session_id": "session", "assistant_message_id": "answer",+ }).Error)+ stopped := false+ f.svc.StopDelegation = func(context.Context, uint64, string, string, string) error { stopped = true; return nil }+ ok, err := f.svc.finishRun(+ f.ctx,+ r,+ []string{types.AutomationRunAwaitingApproval},+ types.AutomationRunFailed,+ "stale timeout",+ nil,+ )+ require.NoError(t, err)+ require.False(t, ok)+ require.False(t, stopped)+ ok, err = f.svc.finishRun(+ f.ctx,+ r,+ []string{types.AutomationRunRunning},+ types.AutomationRunNeedsAttention,+ "executor lost",+ nil,+ )+ require.NoError(t, err)+ require.True(t, ok)+ require.True(t, stopped)+}++func TestAutomationCancellationWinsApprovalRace(t *testing.T) {+ f := newAutomationFixture(t)+ a := f.create(t, types.AutomationPatch{})+ f.suspendOnApproval(t, "op:x")+ queued, err := f.svc.RunNow(f.ctx, a.ID)+ require.NoError(t, err)+ r := f.settleRun(t, queued.ID)+ require.Equal(t, types.AutomationRunAwaitingApproval, r.Status)+ // A remote cancellation can land after execute checked its context but+ // before it records the approval suspension. Do not resume that run.+ require.NoError(t, f.db.Model(r).Update("cancel_requested", true).Error)+ ok, err := f.broker.Store().Transition(f.ctx, r.PendingPromptID, types.AgentPromptTransition{+ From: []string{types.AgentPromptSuspended},+ To: types.AgentPromptAnswered,+ Response: types.JSON(`{"approved":true}`),+ })+ require.NoError(t, err)+ require.True(t, ok)+ require.NoError(t, f.svc.queueResume(f.ctx, r.ID, r.PendingPromptID))+ require.NoError(t, f.svc.reconcileAwaiting(f.ctx, f.now))+ require.NoError(t, f.db.First(&r, "id = ?", r.ID).Error)+ require.Equal(t, types.AutomationRunCancelled, r.Status)+}diff --git a/internal/application/service/automation_runner.go b/internal/application/service/automation_runner.goindex e10c4e391..34724bace 100644--- a/internal/application/service/automation_runner.go+++ b/internal/application/service/automation_runner.go@@ -36,6 +36,7 @@ type runOutcome struct {answer stringusage *types.TokenUsagepromptID string+ elapsed time.Duration// pause, when set, pauses the automation with this reason.pause string}@@ -56,6 +57,7 @@ func (s *AutomationService) execute(parent context.Context, run *types.Automatio}()go s.heartbeat(ctx, run.ID, cancel)+ started := time.Now()out := func() (out runOutcome) {defer func() {if r := recover(); r != nil {@@ -64,11 +66,15 @@ func (s *AutomationService) execute(parent context.Context, run *types.Automatio}()return s.runOnce(ctx, run)}()- if ctx.Err() != nil && out.status != types.AutomationRunAwaitingApproval {+ out.elapsed = time.Since(started)+ if ctx.Err() != nil {var row types.AutomationRunif err := s.db.WithContext(context.WithoutCancel(ctx)).Select("cancel_requested").Where("id = ?", run.ID).Take(&row).Error; err == nil && row.CancelRequested {out.status, out.reason = types.AutomationRunCancelled, "cancelled by the owner"+ if out.promptID != "" {+ s.closePrompt(context.WithoutCancel(ctx), out.promptID, types.AgentPromptCanceled, out.reason)+ }}}s.record(ctx, run, out)@@ -109,7 +115,10 @@ func (s *AutomationService) heartbeat(ctx context.Context, runID string, cancelfunc (s *AutomationService) record(ctx context.Context, run *types.AutomationRun, out runOutcome) {ctx = context.WithoutCancel(ctx)run.Answer = clipRunes(out.answer, maxStoredAnswerRunes)- updates := map[string]any{"answer": run.Answer, "updated_at": s.now()}+ updates := map[string]any{+ "answer": run.Answer, "updated_at": s.now(),+ "execution_duration_ms": gorm.Expr("execution_duration_ms + ?", out.elapsed.Milliseconds()),+ }if out.usage != nil {if data, err := json.Marshal(out.usage); err == nil {updates["usage"] = types.JSON(data)@@ -204,6 +213,7 @@ func (s *AutomationService) ownerContext(ctx context.Context, a *types.Automatio// runOnce executes run, or continues it from its answered approval.func (s *AutomationService) runOnce(ctx context.Context, run *types.AutomationRun) runOutcome {+ started := time.Now()if s.sessions == nil {return failed("the agent executor is unavailable")}@@ -234,7 +244,14 @@ func (s *AutomationService) runOnce(ctx context.Context, run *types.AutomationRuif m := a.Limits.MaxDurationMinutes; m > 0 {limit = time.Duration(m) * time.Minute}- ctx, cancel := context.WithTimeout(ctx, limit)+ remaining := limit - time.Duration(run.ExecutionDurationMS)*time.Millisecond - time.Since(started)+ if remaining <= 0 {+ if run.Resume && run.PendingPromptID != "" {+ s.closePrompt(ctx, run.PendingPromptID, types.AgentPromptFailed, "execution time limit exceeded")+ }+ return failed(fmt.Sprintf("the run took longer than %s", limit))+ }+ ctx, cancel := context.WithTimeout(ctx, remaining)defer cancel()var resume *types.AgentResume@@ -524,7 +541,10 @@ func (s *AutomationService) agentQA(}mu.Lock()defer mu.Unlock()- answer.IsCompleted = suspended == nil+ answer.IsCompleted = suspended == nil || err != nil+ if answer.IsCompleted {+ answer.RunState = ""+ }answer.UpdatedAt = s.now()if err != nil && answer.Content == "" {answer.Content = err.Error()diff --git a/internal/application/service/automation_scheduler.go b/internal/application/service/automation_scheduler.goindex 6c2196c70..c248c2ff2 100644--- a/internal/application/service/automation_scheduler.go+++ b/internal/application/service/automation_scheduler.go@@ -163,6 +163,9 @@ func (s *AutomationService) startQueued(ctx context.Context, now time.Time) erro}var queued []*types.AutomationRunif err := s.db.WithContext(ctx).Where("status = ?", types.AutomationRunQueued).+ Where(`NOT EXISTS (SELECT 1 FROM automation_runs AS holding+ WHERE holding.automation_id = automation_runs.automation_id AND holding.status IN ? )`,+ []string{types.AutomationRunRunning, types.AutomationRunAwaitingApproval}).Order("resume DESC, scheduled_at").Limit(free * 4).Find(&queued).Error; err != nil {return err}@@ -186,16 +189,6 @@ func (s *AutomationService) startQueued(ctx context.Context, now time.Time) erro}func (s *AutomationService) startRun(ctx context.Context, run *types.AutomationRun, now time.Time) (bool, error) {- var holding int64- if err := s.db.WithContext(ctx).Model(&types.AutomationRun{}).- Where("automation_id = ? AND id <> ? AND status IN ?", run.AutomationID, run.ID, []string{- types.AutomationRunRunning, types.AutomationRunAwaitingApproval,- }).Count(&holding).Error; err != nil {- return false, err- }- if holding > 0 {- return false, nil- }lease := now.Add(s.cfg.Lease)updates := map[string]any{"status": types.AutomationRunRunning, "owner": s.instance, "lease_until": lease, "updated_at": now,@@ -203,10 +196,33 @@ func (s *AutomationService) startRun(ctx context.Context, run *types.AutomationRif run.StartedAt == nil {updates["started_at"] = now}- res := s.db.WithContext(ctx).Model(&types.AutomationRun{}).- Where("id = ? AND status = ?", run.ID, types.AutomationRunQueued).Updates(updates)- if res.Error != nil || res.RowsAffected == 0 {- return false, res.Error+ started := false+ err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ // Take the automation's write lock before reading its runs. A run-row+ // CAS alone cannot serialize two different queued runs. Writing first+ // also avoids a deferred read-to-write transaction upgrade on SQLite.+ locked := tx.Model(&types.Automation{}).Where("id = ?", run.AutomationID).+ UpdateColumn("version", gorm.Expr("version + 1"))+ if locked.Error != nil || locked.RowsAffected == 0 {+ return locked.Error+ }+ var holding int64+ if err := tx.Model(&types.AutomationRun{}).+ Where("automation_id = ? AND id <> ? AND status IN ?", run.AutomationID, run.ID, []string{+ types.AutomationRunRunning, types.AutomationRunAwaitingApproval,+ }).Count(&holding).Error; err != nil {+ return err+ }+ if holding > 0 {+ return nil+ }+ res := tx.Model(&types.AutomationRun{}).+ Where("id = ? AND status = ?", run.ID, types.AutomationRunQueued).Updates(updates)+ started = res.Error == nil && res.RowsAffected > 0+ return res.Error+ })+ if err != nil || !started {+ return false, err}run.Status, run.Owner, run.LeaseUntil = types.AutomationRunRunning, s.instance, &leaseif run.StartedAt == nil {@@ -266,20 +282,31 @@ func (s *AutomationService) reconcileAwaiting(ctx context.Context, now time.Timeif s.broker == nil {return nil}- var waiting []*types.AutomationRun- if err := s.db.WithContext(ctx).Where("status = ?", types.AutomationRunAwaitingApproval).- Order("updated_at").Limit(automationClaimBatch * 2).Find(&waiting).Error; err != nil {- return err- }- for _, run := range waiting {- if err := s.reconcileRun(ctx, run, now); err != nil {- logger.Warnf(ctx, "Reconcile automation run %s: %v", run.ID, err)+ // Keyset pagination visits later approvals even when earlier ones stay+ // open. Offsets would skip rows as reconciled runs leave the waiting set.+ for cursor := ""; ; {+ var waiting []*types.AutomationRun+ if err := s.db.WithContext(ctx).+ Where("status = ? AND id > ?", types.AutomationRunAwaitingApproval, cursor).+ Order("id").Limit(automationClaimBatch * 2).Find(&waiting).Error; err != nil {+ return err+ }+ for _, run := range waiting {+ if err := s.reconcileRun(ctx, run, now); err != nil {+ logger.Warnf(ctx, "Reconcile automation run %s: %v", run.ID, err)+ }}+ if len(waiting) < automationClaimBatch*2 {+ return nil+ }+ cursor = waiting[len(waiting)-1].ID}- return nil}func (s *AutomationService) reconcileRun(ctx context.Context, run *types.AutomationRun, now time.Time) error {+ if run.CancelRequested {+ return s.cancelRun(ctx, run, "cancelled by the owner")+ }waiting := []string{types.AutomationRunAwaitingApproval}prompt, err := s.broker.Store().Get(ctx, run.PendingPromptID)if err != nil {@@ -324,7 +351,8 @@ func (s *AutomationService) reconcileRun(ctx context.Context, run *types.Automat// queue, to continue from the answer.func (s *AutomationService) queueResume(ctx context.Context, runID, promptID string) error {res := s.db.WithContext(ctx).Model(&types.AutomationRun{}).- Where("id = ? AND status = ? AND pending_prompt_id = ?", runID, types.AutomationRunAwaitingApproval, promptID).+ Where("id = ? AND status = ? AND pending_prompt_id = ? AND cancel_requested = ?",+ runID, types.AutomationRunAwaitingApproval, promptID, false).Updates(map[string]any{"status": types.AutomationRunQueued, "resume": true, "updated_at": s.now()})if res.Error == nil && res.RowsAffected > 0 {s.Wake()@@ -407,8 +435,15 @@ func (s *AutomationService) finishRun(return false, res.Error}run.Status, run.Error, run.FinishedAt = status, reason, &now+ // Only the winning finalizer may stop work. A stale approval reaper must+ // not cancel a run that another scheduler has already resumed.+ var stopErr error+ if status == types.AutomationRunFailed || status == types.AutomationRunCancelled ||+ status == types.AutomationRunNeedsAttention {+ stopErr = s.stopRunDelegation(context.WithoutCancel(ctx), run)+ }s.settle(ctx, run)- return true, nil+ return true, stopErr}// settle records a finished run on its automation, pauses it after too manydiff --git a/internal/application/service/session_agent_qa.go b/internal/application/service/session_agent_qa.goindex 22db2983f..92448798d 100644--- a/internal/application/service/session_agent_qa.go+++ b/internal/application/service/session_agent_qa.go@@ -431,6 +431,7 @@ func (s *sessionService) AgentQA(// run elsewhere. Session, attachments and images are reloaded from storage.func resumableAgentRequest(ctx context.Context, req *types.QARequest) types.ResumableAgentRequest {r := types.ResumableAgentRequest{+ ActingVia: types.ActingViaFromContext(ctx),Query: req.Query,SummaryModelID: req.SummaryModelID,ReasoningEffort: req.ReasoningEffort,diff --git a/internal/container/container.go b/internal/container/container.goindex a8fd408d0..8121e8d9b 100644--- a/internal/container/container.go+++ b/internal/container/container.go@@ -655,6 +655,7 @@ func BuildContainer(container *dig.Container) *dig.Container {}automations.Bind(sessionService, broker, runs, streams)automations.Events = sessions.CollaborationEvents+ automations.StopDelegation = collaboration.StopOwnedbroker.SetRunHooks(runs.Hooks())runs.StopDelegation = func(ctx context.Context, p *types.AgentPrompt) error {return collaboration.StopPrompt(ctx, p)diff --git a/internal/database/migration_source_test.go b/internal/database/migration_source_test.gonew file mode 100644index 000000000..e2be2cf51--- /dev/null+++ b/internal/database/migration_source_test.go@@ -0,0 +1,22 @@+package database++import (+ "path/filepath"+ "testing"++ "github.com/golang-migrate/migrate/v4/source"+ "github.com/stretchr/testify/require"+)++// Check both migration registries without requiring a live PostgreSQL server.+// Independently merged migrations must not reuse a version in either dialect.+func TestMigrationSourcesHaveUniqueVersions(t *testing.T) {+ root := sqliteRepoRoot(t)+ for _, dialect := range []string{"sqlite", "versioned"} {+ t.Run(dialect, func(t *testing.T) {+ migrations, err := source.Open("file://" + filepath.Join(root, "migrations", dialect))+ require.NoError(t, err)+ require.NoError(t, migrations.Close())+ })+ }+}diff --git a/internal/database/migration_sqlite_automation_duration_test.go b/internal/database/migration_sqlite_automation_duration_test.gonew file mode 100644index 000000000..bd357aa9a--- /dev/null+++ b/internal/database/migration_sqlite_automation_duration_test.go@@ -0,0 +1,36 @@+package database++import (+ "fmt"+ "path/filepath"+ "testing"++ "github.com/stretchr/testify/require"+)++func TestSQLiteAutomationDurationUpgradePreservesExistingRuns(t *testing.T) {+ repoRoot := sqliteRepoRoot(t)+ for _, version := range []int{45, 47} {+ t.Run(fmt.Sprintf("from-v%d", version), func(t *testing.T) {+ chdirAndRestore(t, copySQLiteMigrationsThrough(t, repoRoot, version))+ dbPath := filepath.Join(t.TempDir(), "automations.db")+ require.NoError(t, RunMigrationsWithOptions("sqlite3://unused", MigrationOptions{SQLiteDBPath: dbPath}))+ db := openSQLiteDB(t, dbPath)+ _, err := db.Exec(`INSERT INTO automation_runs+ (id, automation_id, tenant_id, scheduled_at, trigger_kind, status, answer, created_at, updated_at)+ VALUES ('existing-run', 'automation', 1, CURRENT_TIMESTAMP, 'schedule',+ 'awaiting_approval', 'saved answer', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)+ require.NoError(t, err)++ chdirAndRestore(t, repoRoot)+ require.NoError(t, RunMigrationsWithOptions("sqlite3://unused", MigrationOptions{SQLiteDBPath: dbPath}))+ var duration int64+ var answer, status string+ require.NoError(t, db.QueryRow(`SELECT execution_duration_ms, answer, status+ FROM automation_runs WHERE id = 'existing-run'`).Scan(&duration, &answer, &status))+ require.Zero(t, duration)+ require.Equal(t, "saved answer", answer)+ require.Equal(t, "awaiting_approval", status)+ })+ }+}diff --git a/internal/database/migration_sqlite_versioned_schema_test.go b/internal/database/migration_sqlite_versioned_schema_test.goindex 043b976da..5e72bf994 100644--- a/internal/database/migration_sqlite_versioned_schema_test.go+++ b/internal/database/migration_sqlite_versioned_schema_test.go@@ -52,6 +52,7 @@ var versionedSQLiteTables = []string{// versionedSQLiteColumns maps each existing table to the columns that the// versioned migrations add and the SQLite baseline was missing.var versionedSQLiteColumns = map[string][]string{+ "automation_runs": {"execution_duration_ms"}, // 000048"model_catalog_configs": {"version", "overlay", "history", "updated_by", "updated_at"}, // 000031"memory_subjects": {"extraction_state"}, // 000094"memory_items": {"replaces_id"}, // 000094@@ -86,7 +87,7 @@ var versionedSQLiteColumns = map[string][]string{}, // 000028}-const expectedSQLiteMigrationVersion = 47+const expectedSQLiteMigrationVersion = 48func TestSQLiteMigrationsCreateVersionedSchema(t *testing.T) {repoRoot := sqliteRepoRoot(t)@@ -262,19 +263,20 @@ func TestSQLiteSandboxAllocationMigrationRollsBack(t *testing.T) {require.NoError(t, err)t.Cleanup(func() { _, _ = m.Close() })- require.NoError(t, m.Steps(-1))+ // Target the allocation migration even when newer migrations exist.+ require.NoError(t, m.Migrate(46))version, dirty = sqliteMigrationState(t, sqlDB)- require.Equal(t, expectedSQLiteMigrationVersion-1, version)+ require.Equal(t, 46, version)require.False(t, dirty)for _, table := range []string{"session_sandbox_assignments", "sandbox_user_machines"} {require.Falsef(t, sqliteTableExists(t, sqlDB, table), "down must drop %s", table)}require.True(t, sqliteColumnExists(t, sqlDB, "sessions", "owner_auth_source"),- "rolling back the allocation tables keeps the owner source of 000034")+ "rolling back the allocation tables keeps the owner source of 000046")- require.NoError(t, m.Steps(1))+ require.NoError(t, m.Migrate(47))version, dirty = sqliteMigrationState(t, sqlDB)- require.Equal(t, expectedSQLiteMigrationVersion, version)+ require.Equal(t, 47, version)require.False(t, dirty)require.True(t, sqliteTableExists(t, sqlDB, "session_sandbox_assignments"))require.True(t, sqliteTableExists(t, sqlDB, "sandbox_user_machines"))diff --git a/internal/types/agent_prompt.go b/internal/types/agent_prompt.goindex 1365f3b4f..8a47fbede 100644--- a/internal/types/agent_prompt.go+++ b/internal/types/agent_prompt.go@@ -168,8 +168,10 @@ func (p *AgentPrompt) TimelineView() *PendingPromptView {// ResumableAgentRequest is the part of a QARequest needed to rebuild the run// on another replica. It holds the original query and durable request settings.type ResumableAgentRequest struct {- AgentID string `json:"agent_id"`- Query string `json:"query,omitempty"`+ // ActingVia keeps scheduler-owned runs out of ordinary chat recovery.+ ActingVia string `json:"acting_via,omitempty"`+ AgentID string `json:"agent_id"`+ Query string `json:"query,omitempty"`// AgentTenantID is the agent owner's workspace; it differs from the// session's when the agent is shared.AgentTenantID uint64 `json:"agent_tenant_id,omitempty"`diff --git a/internal/types/automation.go b/internal/types/automation.goindex a88ba78ac..316762fed 100644--- a/internal/types/automation.go+++ b/internal/types/automation.go@@ -170,6 +170,9 @@ type AutomationRun struct {Resume bool `json:"-" gorm:"not null;default:false"`// CancelRequested stops the run at its next check.CancelRequested bool `json:"cancel_requested" gorm:"not null;default:false"`+ // ExecutionDurationMS accumulates execution segments across approvals;+ // queued time and time spent waiting for an answer do not consume it.+ ExecutionDurationMS int64 `json:"-" gorm:"not null;default:0"`// Answer is the start of the final answer, for listings.Answer string `json:"answer" gorm:"type:text;not null;default:''" extensions:"x-agent,x-agent-response"`Error string `json:"error" gorm:"type:text;not null;default:''" extensions:"x-agent,x-agent-response"`diff --git a/migrations/sqlite/000048_automation_execution_duration.down.sql b/migrations/sqlite/000048_automation_execution_duration.down.sqlnew file mode 100644index 000000000..1193a7951--- /dev/null+++ b/migrations/sqlite/000048_automation_execution_duration.down.sql@@ -0,0 +1 @@+ALTER TABLE automation_runs DROP COLUMN execution_duration_ms;diff --git a/migrations/sqlite/000048_automation_execution_duration.up.sql b/migrations/sqlite/000048_automation_execution_duration.up.sqlnew file mode 100644index 000000000..1d44f81c9--- /dev/null+++ b/migrations/sqlite/000048_automation_execution_duration.up.sql@@ -0,0 +1 @@+ALTER TABLE automation_runs ADD COLUMN execution_duration_ms BIGINT NOT NULL DEFAULT 0;diff --git a/migrations/versioned/000131_automation_execution_duration.down.sql b/migrations/versioned/000131_automation_execution_duration.down.sqlnew file mode 100644index 000000000..1193a7951--- /dev/null+++ b/migrations/versioned/000131_automation_execution_duration.down.sql@@ -0,0 +1 @@+ALTER TABLE automation_runs DROP COLUMN execution_duration_ms;diff --git a/migrations/versioned/000131_automation_execution_duration.up.sql b/migrations/versioned/000131_automation_execution_duration.up.sqlnew file mode 100644index 000000000..1d44f81c9--- /dev/null+++ b/migrations/versioned/000131_automation_execution_duration.up.sql@@ -0,0 +1 @@+ALTER TABLE automation_runs ADD COLUMN execution_duration_ms BIGINT NOT NULL DEFAULT 0;