Fully autonomous AI Agents system capable of performing complex penetration testing tasks
Go ai-agentsai-security-toolanthropicautonomous-agentsgolanggptgraphqlmulti-agent-systemoffensive-securityopen-sourceopenaipenetration-testingpenetration-testing-toolsreactsecurity-automationsecurity-testingsecurity-toolsself-hosted
refactor(logging): standardize error logging across controllers and providers
Dmitry Ng21baa20a0b65
diff --git a/backend/pkg/controller/assistant.go b/backend/pkg/controller/assistant.goindex bb4811c9..f8ac1f5c 100644--- a/backend/pkg/controller/assistant.go+++ b/backend/pkg/controller/assistant.go@@ -446,12 +446,12 @@ func (aw *assistantWorker) worker() {}if err := aw.SetStatus(ctx, database.AssistantStatusRunning); err != nil {- aw.logger.WithError(err).Error("failed to set assistant status to waiting")+ obs.LogErrorOrCancel(aw.logger, err, "failed to set assistant status to waiting")}defer func() {if err := aw.SetStatus(ctx, database.AssistantStatusWaiting); err != nil {- aw.logger.WithError(err).Error("failed to set assistant status to waiting")+ obs.LogErrorOrCancel(aw.logger, err, "failed to set assistant status to waiting")}}()@@ -489,7 +489,7 @@ func (aw *assistantWorker) worker() {case ain := <-aw.input:err := perform(aw.ctx, ain.input, ain.useAgents)if err != nil {- aw.logger.WithError(err).Error("failed to perform assistant chain")+ obs.LogErrorOrCancel(aw.logger, err, "failed to perform assistant chain")}ain.done <- err}diff --git a/backend/pkg/controller/flow.go b/backend/pkg/controller/flow.goindex 735ed350..b5b83545 100644--- a/backend/pkg/controller/flow.go+++ b/backend/pkg/controller/flow.go@@ -50,6 +50,7 @@ type FlowWorker interface {Stop(ctx context.Context) errorRename(ctx context.Context, title string) errorWaitTaskCompletion(ctx context.Context) error+ InvalidateTaskSubtasks(ctx context.Context, taskID int64, subtaskIDs []int64)}type flowWorker struct {@@ -140,7 +141,7 @@ func NewFlowWorker(UserID: fwc.userID,})if err != nil {- logrus.WithError(err).Error("failed to create flow in DB")+ obs.LogErrorOrCancel(logrus.WithContext(ctx), err, "failed to create flow in DB")return nil, fmt.Errorf("failed to create flow in DB: %w", err)}@@ -548,6 +549,17 @@ func (fw *flowWorker) SetStatus(ctx context.Context, status database.FlowStatus)return nil}+// InvalidateTaskSubtasks drops stale workers after direct DB deletion,+// preventing delayed ErrNoRows failures.+func (fw *flowWorker) InvalidateTaskSubtasks(ctx context.Context, taskID int64, subtaskIDs []int64) {+ task, err := fw.tc.GetTask(ctx, taskID)+ if err != nil {+ return+ }++ task.InvalidateSubtasks(subtaskIDs)+}+func (fw *flowWorker) AddAssistant(ctx context.Context, aw AssistantWorker) error {fw.awsMX.Lock()defer fw.awsMX.Unlock()diff --git a/backend/pkg/controller/subtask.go b/backend/pkg/controller/subtask.goindex ba002299..10ce85ec 100644--- a/backend/pkg/controller/subtask.go+++ b/backend/pkg/controller/subtask.go@@ -2,6 +2,7 @@ package controllerimport ("context"+ "database/sql""errors""fmt""sync"@@ -189,6 +190,22 @@ func (stw *subtaskWorker) SetStatus(ctx context.Context, status database.SubtaskID: stw.subtaskCtx.SubtaskID,})if err != nil {+ if errors.Is(err, sql.ErrNoRows) {+ // Replacement can delete the row before this stale worker finishes.+ // Treat it as complete to keep task shutdown idempotent.+ logrus.WithContext(ctx).WithFields(logrus.Fields{+ "subtask_id": stw.subtaskCtx.SubtaskID,+ "requested_status": status,+ }).Warn("subtask no longer exists in the database, treating as already finished")++ stw.mx.Lock()+ stw.completed = true+ stw.waiting = false+ stw.mx.Unlock()++ return nil+ }+return fmt.Errorf("failed to set subtask %d status: %w", stw.subtaskCtx.SubtaskID, err)}diff --git a/backend/pkg/controller/subtasks.go b/backend/pkg/controller/subtasks.goindex fcdc7c79..4dd05f0a 100644--- a/backend/pkg/controller/subtasks.go+++ b/backend/pkg/controller/subtasks.go@@ -22,6 +22,7 @@ type SubtaskController interface {PopSubtask(ctx context.Context, updater TaskUpdater) (SubtaskWorker, error)ListSubtasks(ctx context.Context) []SubtaskWorkerGetSubtask(ctx context.Context, subtaskID int64) (SubtaskWorker, error)+ InvalidateSubtasks(subtaskIDs []int64)}type subtaskController struct {@@ -119,6 +120,7 @@ func (stc *subtaskController) RefineSubtasks(ctx context.Context) error {if err != nil {return fmt.Errorf("failed to delete subtasks for task %d: %w", stc.taskCtx.TaskID, err)}+ stc.InvalidateSubtasks(subtaskIDs)// TODO: change it to insert subtasks in transaction and union it with delete onesfor _, info := range plan {@@ -191,3 +193,18 @@ func (stc *subtaskController) GetSubtask(ctx context.Context, subtaskID int64) (return subtask, nil}++// InvalidateSubtasks drops workers deleted outside this controller,+// preventing stale cache lookups.+func (stc *subtaskController) InvalidateSubtasks(subtaskIDs []int64) {+ if len(subtaskIDs) == 0 {+ return+ }++ stc.mx.Lock()+ defer stc.mx.Unlock()++ for _, id := range subtaskIDs {+ delete(stc.subtasks, id)+ }+}diff --git a/backend/pkg/controller/task.go b/backend/pkg/controller/task.goindex 4de17399..d2c5f742 100644--- a/backend/pkg/controller/task.go+++ b/backend/pkg/controller/task.go@@ -2,6 +2,7 @@ package controllerimport ("context"+ "database/sql""errors""fmt""sync"@@ -33,6 +34,7 @@ type TaskWorker interface {PutInput(ctx context.Context, input string) errorRun(ctx context.Context) errorFinish(ctx context.Context) error+ InvalidateSubtasks(subtaskIDs []int64)}type taskWorker struct {@@ -204,6 +206,22 @@ func (tw *taskWorker) SetStatus(ctx context.Context, status database.TaskStatus)ID: tw.taskCtx.TaskID,})if err != nil {+ if errors.Is(err, sql.ErrNoRows) {+ // Replacement can leave this worker behind after deleting its row.+ // Treat it as complete so flow shutdown remains idempotent.+ logrus.WithContext(ctx).WithFields(logrus.Fields{+ "task_id": tw.taskCtx.TaskID,+ "requested_status": status,+ }).Warn("task no longer exists in the database, treating as already finished")++ tw.mx.Lock()+ tw.completed = true+ tw.waiting = false+ tw.mx.Unlock()++ return nil+ }+return fmt.Errorf("failed to set task %d status: %w", tw.taskCtx.TaskID, err)}@@ -397,3 +415,9 @@ func (tw *taskWorker) Finish(ctx context.Context) error {return nil}++// InvalidateSubtasks evicts subtaskIDs from this task's in-memory subtask+// controller. See flowWorker.InvalidateTaskSubtasks for why this is needed.+func (tw *taskWorker) InvalidateSubtasks(subtaskIDs []int64) {+ tw.stc.InvalidateSubtasks(subtaskIDs)+}diff --git a/backend/pkg/observability/obs.go b/backend/pkg/observability/obs.goindex 7c0a857b..8bcd6cab 100644--- a/backend/pkg/observability/obs.go+++ b/backend/pkg/observability/obs.go@@ -786,3 +786,21 @@ func getPackageName(f string) string {return f}++// LogErrorOrCancel logs err on entry at Error level, unless err is (or wraps)+// context.Canceled, in which case it logs at Warn level instead.+//+// In this codebase context.Canceled almost always propagates from an explicit+// user action (stopping/pausing a flow or task, closing an assistant chat),+// not from a genuine failure. Logging it at Error level pollutes error-rate+// monitoring and alerting with expected, benign events. This helper keeps+// that single decision in one place so every call site treats cancellation+// consistently.+func LogErrorOrCancel(entry *logrus.Entry, err error, msg string) {+ if errors.Is(err, context.Canceled) {+ entry.WithError(err).Warn(msg)+ return+ }++ entry.WithError(err).Error(msg)+}diff --git a/backend/pkg/providers/assistant.go b/backend/pkg/providers/assistant.goindex c77040c9..e84f6fbb 100644--- a/backend/pkg/providers/assistant.go+++ b/backend/pkg/providers/assistant.go@@ -50,6 +50,7 @@ type FlowWorker interface {Stop(ctx context.Context) errorRename(ctx context.Context, title string) errorWaitTaskCompletion(ctx context.Context) error+ InvalidateTaskSubtasks(ctx context.Context, taskID int64, subtaskIDs []int64)}type assistantProvider struct {@@ -650,6 +651,10 @@ func (ap *assistantProvider) patchAssistantFlowSubtasks(if err := db.DeleteSubtasks(ctx, idsToDelete); err != nil {return fmt.Errorf("failed to delete subtasks for task %d: %w", taskID, err)}+ // The task worker for taskID (if currently loaded in memory) still holds+ // SubtaskWorker objects for idsToDelete; without this it would later try+ // to operate on rows that no longer exist (e.g. on flow Finish/Stop).+ ap.flowWorker.InvalidateTaskSubtasks(ctx, taskID, idsToDelete)}// result contains only entries that were in patchableSubs (minus removed ones)diff --git a/backend/pkg/providers/handlers.go b/backend/pkg/providers/handlers.goindex 47112911..e071fc40 100644--- a/backend/pkg/providers/handlers.go+++ b/backend/pkg/providers/handlers.go@@ -4,6 +4,7 @@ import ("context""crypto/sha256""encoding/json"+ "errors""fmt""strings"@@ -22,26 +23,36 @@ import ()func wrapError(ctx context.Context, msg string, err error) error {- logrus.WithContext(ctx).WithError(err).Error(msg)+ obs.LogErrorOrCancel(logrus.WithContext(ctx), err, msg)return fmt.Errorf("%s: %w", msg, err)}func wrapErrorEndAgentSpan(ctx context.Context, span langfuse.Agent, msg string, err error) error {- logrus.WithContext(ctx).WithError(err).Error(msg)+ obs.LogErrorOrCancel(logrus.WithContext(ctx), err, msg)err = fmt.Errorf("%s: %w", msg, err)++ level := langfuse.ObservationLevelError+ if errors.Is(err, context.Canceled) {+ level = langfuse.ObservationLevelWarning+ }span.End(langfuse.WithAgentStatus(err.Error()),- langfuse.WithAgentLevel(langfuse.ObservationLevelError),+ langfuse.WithAgentLevel(level),)return err}func wrapErrorEndEvaluatorSpan(ctx context.Context, span langfuse.Evaluator, msg string, err error) error {- logrus.WithContext(ctx).WithError(err).Error(msg)+ obs.LogErrorOrCancel(logrus.WithContext(ctx), err, msg)err = fmt.Errorf("%s: %w", msg, err)++ level := langfuse.ObservationLevelError+ if errors.Is(err, context.Canceled) {+ level = langfuse.ObservationLevelWarning+ }span.End(langfuse.WithEvaluatorStatus(err.Error()),- langfuse.WithEvaluatorLevel(langfuse.ObservationLevelError),+ langfuse.WithEvaluatorLevel(level),)return err}diff --git a/backend/pkg/providers/helpers.go b/backend/pkg/providers/helpers.goindex 3e126a16..7b3bb02d 100644--- a/backend/pkg/providers/helpers.go+++ b/backend/pkg/providers/helpers.go@@ -618,19 +618,19 @@ func (fp *flowProvider) processChain() error {msgChain, err := fp.db.GetMsgChain(ctx, msgChainID)if err != nil {- logger.WithError(err).Error("failed to get message chain")+ obs.LogErrorOrCancel(logger, err, "failed to get message chain")return fmt.Errorf("failed to get message chain %d: %w", msgChainID, err)}var chain []llms.MessageContentif err := json.Unmarshal(msgChain.Chain, &chain); err != nil {- logger.WithError(err).Error("failed to unmarshal message chain")+ obs.LogErrorOrCancel(logger, err, "failed to unmarshal message chain")return fmt.Errorf("failed to unmarshal message chain %d: %w", msgChainID, err)}updatedChain, err := transform(chain)if err != nil {- logger.WithError(err).Error("failed to transform chain")+ obs.LogErrorOrCancel(logger, err, "failed to transform chain")return fmt.Errorf("failed to transform chain: %w", err)}@@ -680,7 +680,7 @@ func (fp *flowProvider) processChain(chainBlob, err := json.Marshal(updatedChain)if err != nil {- logger.WithError(err).Error("failed to marshal updated chain")+ obs.LogErrorOrCancel(logger, err, "failed to marshal updated chain")return fmt.Errorf("failed to marshal updated chain %d: %w", msgChainID, err)}@@ -692,7 +692,7 @@ func (fp *flowProvider) processChain(ID: msgChainID,})if err != nil {- logger.WithError(err).Error("failed to update message chain")+ obs.LogErrorOrCancel(logger, err, "failed to update message chain")return fmt.Errorf("failed to update message chain %d: %w", msgChainID, err)}diff --git a/backend/pkg/providers/performer.go b/backend/pkg/providers/performer.goindex c1c990aa..c4b9db14 100644--- a/backend/pkg/providers/performer.go+++ b/backend/pkg/providers/performer.go@@ -127,12 +127,12 @@ func (fp *flowProvider) performAgentChain(} else {result, err = fp.callWithRetries(ctx, optAgentType, chainID, taskID, subtaskID, chain, executor, executionContext)if err != nil {- logger.WithError(err).Error("failed to call agent chain")+ obs.LogErrorOrCancel(logger, err, "failed to call agent chain")return err}if err := fp.updateMsgChainUsage(ctx, chainID, optAgentType, result.info, rollLastUpdateTime()); err != nil {- logger.WithError(err).Error("failed to update msg chain usage")+ obs.LogErrorOrCancel(logger, err, "failed to update msg chain usage")return err}}@@ -160,7 +160,7 @@ func (fp *flowProvider) performAgentChain(}fields["execution"] = executionContext[:min(1000, len(executionContext))]}- logger.WithError(err).WithFields(fields).Error("failed to perform reflector")+ obs.LogErrorOrCancel(logger.WithFields(fields), err, "failed to perform reflector")return err}}@@ -179,7 +179,7 @@ func (fp *flowProvider) performAgentChain(chain = append(chain, msg)if err := fp.updateMsgChain(ctx, optAgentType, chainID, chain, rollLastUpdateTime()); err != nil {- logger.WithError(err).Error("failed to update msg chain")+ obs.LogErrorOrCancel(logger, err, "failed to update msg chain")return err}@@ -200,10 +200,10 @@ func (fp *flowProvider) performAgentChain(}if err != nil {- logger.WithError(err).WithFields(logrus.Fields{+ obs.LogErrorOrCancel(logger.WithFields(logrus.Fields{"func_name": funcName,"func_args": toolCall.FunctionCall.Arguments,- }).Error("failed to exec tool call")+ }), err, "failed to exec tool call")return err}@@ -218,7 +218,7 @@ func (fp *flowProvider) performAgentChain(},})if err := fp.updateMsgChain(ctx, optAgentType, chainID, chain, rollLastUpdateTime()); err != nil {- logger.WithError(err).Error("failed to update msg chain")+ obs.LogErrorOrCancel(logger, err, "failed to update msg chain")return err}@@ -250,7 +250,7 @@ func (fp *flowProvider) performAgentChain()logger.WithError(err).Warn("failed to summarize chain")} else if err := fp.updateMsgChain(ctx, optAgentType, chainID, chain, rollLastUpdateTime()); err != nil {- logger.WithError(err).Error("failed to update msg chain")+ obs.LogErrorOrCancel(logger, err, "failed to update msg chain")return err}}@@ -333,7 +333,7 @@ func (fp *flowProvider) execToolCall(for idx := 0; idx <= maxRetriesToCallFunction; idx++ {if idx == maxRetriesToCallFunction {err = fmt.Errorf("reached max retries to call function: %w", err)- logger.WithError(err).Error("failed to exec function")+ obs.LogErrorOrCancel(logger, err, "failed to exec function")return "", fmt.Errorf("failed to exec function '%s': %w", funcName, err)}@@ -343,6 +343,15 @@ func (fp *flowProvider) execToolCall(return "", err}+ if errors.Is(err, tools.ErrFlowStateGuard) {+ // Not an argument-format problem: the flow/task/subtask is in a state+ // that makes this call impossible right now (task running, no plan+ // yet, stale subtask, ...). Retrying fixToolCallArgs can't change+ // that, so surface it to the agent immediately instead of burning+ // retries on "corrected" arguments that won't help.+ return "", err+ }+logger.WithError(err).Warn("failed to exec function")funcExecErr := err@@ -354,7 +363,7 @@ func (fp *flowProvider) execToolCall(funcArgs, err = fp.fixToolCallArgs(ctx, funcName, funcArgs, funcSchema, funcExecErr)if err != nil {- logger.WithError(err).Error("failed to fix tool call args")+ obs.LogErrorOrCancel(logger, err, "failed to fix tool call args")return "", fmt.Errorf("failed to fix tool call args: %w", err)}} else {@@ -674,10 +683,14 @@ func (fp *flowProvider) performReflector(chain = append(chain, llms.TextParts(llms.ChatMessageTypeHuman, advice))result, err := fp.callWithRetries(ctx, optOriginType, chainID, taskID, subtaskID, chain, executor, executionContext)if err != nil {- logger.WithError(err).Error("failed to call agent chain by reflector")+ obs.LogErrorOrCancel(logger, err, "failed to call agent chain by reflector")+ level := langfuse.ObservationLevelError+ if errors.Is(err, context.Canceled) {+ level = langfuse.ObservationLevelWarning+ }opts = append(opts,langfuse.WithAgentStatus(err.Error()),- langfuse.WithAgentLevel(langfuse.ObservationLevelError),+ langfuse.WithAgentLevel(level),)return nil, err}diff --git a/backend/pkg/providers/performers.go b/backend/pkg/providers/performers.goindex 1f35d34a..a05a3a39 100644--- a/backend/pkg/providers/performers.go+++ b/backend/pkg/providers/performers.go@@ -319,7 +319,7 @@ func (fp *flowProvider) performSubtasksRefiner(DurationSeconds: time.Since(startTime).Seconds(),})if err != nil {- logger.WithError(err).Error("failed to create msg chain")+ obs.LogErrorOrCancel(logger, err, "failed to create msg chain")return nil, fmt.Errorf("failed to create msg chain: %w", err)}@@ -327,7 +327,7 @@ func (fp *flowProvider) performSubtasksRefiner(err = fp.performAgentChain(ctx, optAgentType, msgChain.ID, &taskID, nil, chain, executor, fp.summarizer)if err != nil {- logger.WithError(err).Error("failed to perform subtasks refiner agent chain")+ obs.LogErrorOrCancel(logger, err, "failed to perform subtasks refiner agent chain")return nil, fmt.Errorf("failed to get subtasks refiner result: %w", err)}diff --git a/backend/pkg/server/auth/auth_middleware.go b/backend/pkg/server/auth/auth_middleware.goindex ab175614..5fc7358f 100644--- a/backend/pkg/server/auth/auth_middleware.go+++ b/backend/pkg/server/auth/auth_middleware.go@@ -74,10 +74,12 @@ func (p *AuthMiddleware) tryAuth(}if withFail && result != authResultOk {- if errors.Is(authErr, errCookieClaimInvalid) {- // An expired/absent session cookie hitting a protected endpoint is- // routine (e.g. a stale browser tab polling after logout/expiry),- // not an application error - log it quietly instead of at Error.+ if isRoutineAuthFailure(authErr) {+ // An expired/absent/invalidated session or token hitting a protected+ // endpoint is routine (e.g. a stale browser tab polling after+ // logout/expiry, or a password change/DB reseed invalidating the+ // stored hash), not an application error - log it quietly instead+ // of at Error.response.ErrorWithLevel(c, response.ErrAuthRequired, authErr, logrus.WarnLevel)} else {response.Error(c, response.ErrAuthRequired, authErr)@@ -87,10 +89,28 @@ func (p *AuthMiddleware) tryAuth(c.Next()}+// isRoutineAuthFailure reports whether authErr represents an expected,+// non-malicious session/token invalidation rather than a genuine application+// error, so callers can log it at a quieter level.+func isRoutineAuthFailure(authErr error) bool {+ return errors.Is(authErr, errCookieClaimInvalid) ||+ errors.Is(authErr, errSessionExpired) ||+ errors.Is(authErr, errUserHashMismatch)+}+// errCookieClaimInvalid is returned by tryUserCookieAuthentication when the// session cookie is present but missing one or more required claims (expired// or otherwise invalid session) - a routine, expected condition.-var errCookieClaimInvalid = errors.New("cookie claim invalid")+//+// errSessionExpired and errUserHashMismatch mark the same category of routine+// session/token invalidation, just detected a bit later during validation: a+// session past its TTL, or a stored hash that no longer matches the user+// record (e.g. after a password change or a test database reseed).+var (+ errCookieClaimInvalid = errors.New("cookie claim invalid")+ errSessionExpired = errors.New("session expired")+ errUserHashMismatch = errors.New("user hash mismatch")+)func (p *AuthMiddleware) tryUserCookieAuthentication(c *gin.Context) (authResult, error) {sessionObject, exists := c.Get(sessions.DefaultKey)@@ -128,7 +148,7 @@ func (p *AuthMiddleware) tryUserCookieAuthentication(c *gin.Context) (authResultreturn authResultFail, errors.New("token claim invalid")}if time.Now().Unix() > expVal {- return authResultFail, errors.New("session expired")+ return authResultFail, errSessionExpired}// Verify user hash matches database@@ -152,7 +172,7 @@ func (p *AuthMiddleware) tryUserCookieAuthentication(c *gin.Context) (authResult}if dbHash != sessionHash {- return authResultFail, errors.New("user hash mismatch - session invalid for this installation")+ return authResultFail, fmt.Errorf("%w - session invalid for this installation", errUserHashMismatch)}c.Set("prm", prms)@@ -224,7 +244,7 @@ func (p *AuthMiddleware) tryProtoTokenAuthentication(c *gin.Context) (authResult}if dbHash != apiClaims.UHASH {- return authResultFail, errors.New("user hash mismatch - token invalid for this installation")+ return authResultFail, fmt.Errorf("%w - token invalid for this installation", errUserHashMismatch)}// generate UUID from user hash (fallback to empty string if hash is invalid)diff --git a/backend/pkg/tools/browser.go b/backend/pkg/tools/browser.goindex a3f2932e..af5b2348 100644--- a/backend/pkg/tools/browser.go+++ b/backend/pkg/tools/browser.go@@ -147,6 +147,13 @@ func (b *browser) Handle(ctx context.Context, name string, args json.RawMessage)return "", fmt.Errorf("failed to unmarshal browser action: %w", err)}+ // LLMs occasionally wrap the URL with an incidental leading/trailing+ // newline, carriage return, or space (e.g. copy-pasted from command+ // output); net/url.Parse rejects control characters outright, failing the+ // whole call over whitespace that carries no meaning. Strip it here so it+ // never reaches resolveUrl.+ action.Url = strings.TrimSpace(action.Url)+if action.Action == "" {// The LLM occasionally omits the required 'action' field even though the// tool schema marks it required. 'markdown' is the safest default: it isdiff --git a/backend/pkg/tools/flow_manager.go b/backend/pkg/tools/flow_manager.goindex e8d02102..3a3bbf75 100644--- a/backend/pkg/tools/flow_manager.go+++ b/backend/pkg/tools/flow_manager.go@@ -11,6 +11,28 @@ import ("pentagi/pkg/database")+// ErrFlowStateGuard is a sentinel wrapped into flow_manager tool errors that stem+// from the flow/task/subtask being in a state that makes the requested operation+// currently impossible (a task already running, no plan yet, an unknown task ID,+// a subtask that no longer exists after a database-level change, etc.) rather+// than from malformed call arguments. Callers can check errors.Is(err,+// ErrFlowStateGuard) to skip argument-fixing retries for these: no amount of+// "corrected" arguments changes the underlying flow state, so retrying only+// wastes LLM round-trips before the same guard fires again.+var ErrFlowStateGuard = errors.New("flow state guard")++// stateGuardError marks err so errors.Is(_, ErrFlowStateGuard) reports true,+// without altering err's displayed message - callers, and the LLM, only ever+// see the original err.Error() text.+type stateGuardError struct{ error }++func (e stateGuardError) Is(target error) bool { return target == ErrFlowStateGuard }+func (e stateGuardError) Unwrap() error { return e.error }++// stateGuard wraps a flow_manager guard-rail error so it can be recognized via+// errors.Is(err, ErrFlowStateGuard) further up the call stack.+func stateGuard(err error) error { return stateGuardError{err} }+// flowStatusTool implements get_flow_status — reads flow/task/subtask state from DB.type flowStatusTool struct {flowID int64@@ -735,7 +757,7 @@ func (t *submitFlowInputTool) Handle(ctx context.Context, name string, args json// (most likely after a system restart or database cleanup while the subtask was at an ask checkpoint).// The subtask can no longer be resumed; the flow must be stopped and restarted.if strings.Contains(err.Error(), "no rows in result set") {- return "", fmt.Errorf(+ return "", stateGuard(fmt.Errorf("the waiting subtask's execution context is no longer available in the database — "+"this typically happens after a system restart when the subtask was paused at an ask checkpoint. "+"Recovery: (1) call %s to cancel the stale task; "+@@ -743,7 +765,7 @@ func (t *submitFlowInputTool) Handle(ctx context.Context, name string, args json"if the flow already has planned subtasks that should still run, "+"use %s to inspect the remaining plan and %s to patch it before resuming",StopFlowToolName, SubmitFlowInputToolName,- GetFlowStatusToolName, PatchFlowSubtasksToolName)+ GetFlowStatusToolName, PatchFlowSubtasksToolName))}return "", fmt.Errorf("failed to submit flow input: %w", err)}@@ -830,11 +852,11 @@ func (t *patchFlowSubtasksTool) Handle(ctx context.Context, name string, args jsfor _, task := range tasks {if task.Status == database.TaskStatusRunning {- return "", fmt.Errorf(+ return "", stateGuard(fmt.Errorf("task %q (ID: %d) is currently running; "+"patching is not allowed while a task is executing. "+"Call %s first, then retry %s",- task.Title, task.ID, StopFlowToolName, PatchFlowSubtasksToolName)+ task.Title, task.ID, StopFlowToolName, PatchFlowSubtasksToolName))}}@@ -846,10 +868,10 @@ func (t *patchFlowSubtasksTool) Handle(ctx context.Context, name string, args js}}if !taskBelongsToFlow {- return "", fmt.Errorf(+ return "", stateGuard(fmt.Errorf("task ID %d was not found in this flow; "+"obtain a valid task ID from %s with detail='tasks'",- action.TaskID, GetFlowStatusToolName)+ action.TaskID, GetFlowStatusToolName))}planned, err := t.db.GetTaskPlannedSubtasks(ctx, action.TaskID)@@ -869,24 +891,24 @@ func (t *patchFlowSubtasksTool) Handle(ctx context.Context, name string, args js}switch st.Status {case database.SubtaskStatusWaiting:- return "", fmt.Errorf(+ return "", stateGuard(fmt.Errorf("task %d has a subtask (ID: %d, %q) waiting for user input. "+"Only 'created' subtasks can be patched, but you can include the waiting subtask's ID in your operations to modify or remove it. "+"Alternatively, answer it via %s first",- action.TaskID, st.ID, st.Title, SubmitFlowInputToolName)+ action.TaskID, st.ID, st.Title, SubmitFlowInputToolName))case database.SubtaskStatusRunning:- return "", fmt.Errorf(+ return "", stateGuard(fmt.Errorf("task %d has a subtask (ID: %d, %q) currently running. "+"Call %s first, then retry",- action.TaskID, st.ID, st.Title, StopFlowToolName)+ action.TaskID, st.ID, st.Title, StopFlowToolName))}}- return "", fmt.Errorf(+ return "", stateGuard(fmt.Errorf("no 'created' subtasks found for task %d; "+"all subtasks have been executed or the task has no plan yet. "+"Use %s to create a new task instead",- action.TaskID, SubmitFlowInputToolName)+ action.TaskID, SubmitFlowInputToolName))}patch := SubtaskPatch{diff --git a/backend/pkg/tools/guide.go b/backend/pkg/tools/guide.goindex 04b6f4a7..da29cd61 100644--- a/backend/pkg/tools/guide.go+++ b/backend/pkg/tools/guide.go@@ -138,7 +138,7 @@ func (g *guide) Handle(ctx context.Context, name string, args json.RawMessage) (vectorstores.WithFilters(filters),)if err != nil {- queryLogger.WithError(err).Error("failed to search for similar documents")+ obs.LogErrorOrCancel(queryLogger, err, "failed to search for similar documents")continue // Continue with other queries even if one fails}diff --git a/backend/pkg/tools/memory.go b/backend/pkg/tools/memory.goindex 2b779b23..90e83767 100644--- a/backend/pkg/tools/memory.go+++ b/backend/pkg/tools/memory.go@@ -126,7 +126,7 @@ func (m *memory) Handle(ctx context.Context, name string, args json.RawMessage)vectorstores.WithFilters(filters),)if err != nil {- queryLogger.WithError(err).Error("failed to search for similar documents")+ obs.LogErrorOrCancel(queryLogger, err, "failed to search for similar documents")continue // Continue with other queries even if one fails}@@ -140,7 +140,7 @@ func (m *memory) Handle(ctx context.Context, name string, args json.RawMessage)vectorstores.WithFilters(globalFilters),)if err != nil {- queryLogger.WithError(err).Error("failed to search with global filters")+ obs.LogErrorOrCancel(queryLogger, err, "failed to search with global filters")continue}if len(docs) > 0 {diff --git a/backend/pkg/tools/search.go b/backend/pkg/tools/search.goindex 6af9a482..cac8dda0 100644--- a/backend/pkg/tools/search.go+++ b/backend/pkg/tools/search.go@@ -136,7 +136,7 @@ func (s *search) Handle(ctx context.Context, name string, args json.RawMessage)vectorstores.WithFilters(filters),)if err != nil {- queryLogger.WithError(err).Error("failed to search answer for query")+ obs.LogErrorOrCancel(queryLogger, err, "failed to search answer for query")continue // Continue with other queries even if one fails}diff --git a/backend/pkg/tools/searchers/duckduckgo.go b/backend/pkg/tools/searchers/duckduckgo.goindex 6147da2e..b2ff13e4 100644--- a/backend/pkg/tools/searchers/duckduckgo.go+++ b/backend/pkg/tools/searchers/duckduckgo.go@@ -119,7 +119,7 @@ func (d *duckduckgo) Handle(ctx context.Context, req Request) (string, error) {}),)- logger.WithError(err).Error("failed to search in DuckDuckGo")+ obs.LogErrorOrCancel(logger, err, "failed to search in DuckDuckGo")// DuckDuckGo already retries transient failures internally (see search);// by the time an error surfaces here, moving to the next engine is the// right call rather than burning another round-trip on the same one.diff --git a/backend/pkg/tools/searchers/firecrawl.go b/backend/pkg/tools/searchers/firecrawl.goindex 7670bdb0..540cd83b 100644--- a/backend/pkg/tools/searchers/firecrawl.go+++ b/backend/pkg/tools/searchers/firecrawl.go@@ -124,7 +124,7 @@ func (f *firecrawl) Handle(ctx context.Context, req Request) (string, error) {}),)- logger.WithError(err).Error("failed to search in firecrawl")+ obs.LogErrorOrCancel(logger, err, "failed to search in firecrawl")return "", err}diff --git a/backend/pkg/tools/searchers/google.go b/backend/pkg/tools/searchers/google.goindex e1b563f4..9735cc97 100644--- a/backend/pkg/tools/searchers/google.go+++ b/backend/pkg/tools/searchers/google.go@@ -72,7 +72,7 @@ func (g *google) Handle(ctx context.Context, req Request) (string, error) {}),)- logger.WithError(err).Error("failed to search in google")+ obs.LogErrorOrCancel(logger, err, "failed to search in google")return "", classifyGoogleError(err)}diff --git a/backend/pkg/tools/searchers/internal.go b/backend/pkg/tools/searchers/internal.goindex 82608ecf..de67b63c 100644--- a/backend/pkg/tools/searchers/internal.go+++ b/backend/pkg/tools/searchers/internal.go@@ -103,7 +103,7 @@ func (e *internalEngine) Handle(ctx context.Context, req Request) (string, error"error": err.Error(),}),)- logger.WithError(err).Error("internal analytics engine failed")+ obs.LogErrorOrCancel(logger, err, "internal analytics engine failed")return "", err}diff --git a/backend/pkg/tools/searchers/perplexity.go b/backend/pkg/tools/searchers/perplexity.goindex a06e45bc..1752aea2 100644--- a/backend/pkg/tools/searchers/perplexity.go+++ b/backend/pkg/tools/searchers/perplexity.go@@ -125,7 +125,7 @@ func (p *perplexity) Handle(ctx context.Context, req Request) (string, error) {}),)- logger.WithError(err).Error("failed to search in perplexity")+ obs.LogErrorOrCancel(logger, err, "failed to search in perplexity")return "", err}diff --git a/backend/pkg/tools/searchers/searxng.go b/backend/pkg/tools/searchers/searxng.goindex 3969ebd0..ff83f277 100644--- a/backend/pkg/tools/searchers/searxng.go+++ b/backend/pkg/tools/searchers/searxng.go@@ -70,7 +70,7 @@ func (s *searxng) Handle(ctx context.Context, req Request) (string, error) {}),)- logger.WithError(err).Error("failed to search in searxng")+ obs.LogErrorOrCancel(logger, err, "failed to search in searxng")return "", err}diff --git a/backend/pkg/tools/searchers/sploitus.go b/backend/pkg/tools/searchers/sploitus.goindex 8f71ec2e..bcb79517 100644--- a/backend/pkg/tools/searchers/sploitus.go+++ b/backend/pkg/tools/searchers/sploitus.go@@ -98,7 +98,7 @@ func (s *sploitus) Handle(ctx context.Context, req Request) (string, error) {}),)- logger.WithError(err).Error("failed to search in Sploitus")+ obs.LogErrorOrCancel(logger, err, "failed to search in Sploitus")return "", err}diff --git a/backend/pkg/tools/searchers/tavily.go b/backend/pkg/tools/searchers/tavily.goindex 98f09e5b..2bea2550 100644--- a/backend/pkg/tools/searchers/tavily.go+++ b/backend/pkg/tools/searchers/tavily.go@@ -93,7 +93,7 @@ func (t *tavily) Handle(ctx context.Context, req Request) (string, error) {}),)- logger.WithError(err).Error("failed to search in tavily")+ obs.LogErrorOrCancel(logger, err, "failed to search in tavily")return "", err}diff --git a/backend/pkg/tools/searchers/traversaal.go b/backend/pkg/tools/searchers/traversaal.goindex 58864f3a..5ef3103e 100644--- a/backend/pkg/tools/searchers/traversaal.go+++ b/backend/pkg/tools/searchers/traversaal.go@@ -61,7 +61,7 @@ func (t *traversaal) Handle(ctx context.Context, req Request) (string, error) {}),)- logger.WithError(err).Error("failed to search in traversaal")+ obs.LogErrorOrCancel(logger, err, "failed to search in traversaal")return "", err}