A platform for community discussion. Free, open, simple.
FEATURE: Time out workflow runs that never finish
Régis Hanol4fd6529a61d4
diff --git a/plugins/discourse-workflows/app/jobs/scheduled/discourse_workflows/expire_stale_executions.rb b/plugins/discourse-workflows/app/jobs/scheduled/discourse_workflows/expire_stale_executions.rbnew file mode 100644index 00000000000..e4e1b76106b--- /dev/null+++ b/plugins/discourse-workflows/app/jobs/scheduled/discourse_workflows/expire_stale_executions.rb@@ -0,0 +1,13 @@+# frozen_string_literal: true++module Jobs+ module DiscourseWorkflows+ class ExpireStaleExecutions < ::Jobs::Scheduled+ every 5.minutes++ def execute(_args = nil)+ ::DiscourseWorkflows::Execution::ExpireStale.call+ end+ end+ end+enddiff --git a/plugins/discourse-workflows/app/models/discourse_workflows/execution.rb b/plugins/discourse-workflows/app/models/discourse_workflows/execution.rbindex 5e395fd7b06..a799f49dc5a 100644--- a/plugins/discourse-workflows/app/models/discourse_workflows/execution.rb+++ b/plugins/discourse-workflows/app/models/discourse_workflows/execution.rb@@ -29,6 +29,14 @@ module DiscourseWorkflows-> { where.not(started_at: nil, finished_at: nil).where(status: %i[success error]) }scope :expired_waiting, -> { where(status: :waiting).where("waiting_until < ?", Time.current) }+ scope :stale, ->(cutoff) { where(status: %i[pending running]).where("updated_at < ?", cutoff) }++ CLEARED_WAIT_STATE = {+ resume_token: nil,+ waiting_node_id: nil,+ waiting_until: nil,+ timeout_action: nil,+ }.freezescope :with_snapshot_summary,-> do@@ -192,25 +200,9 @@ module DiscourseWorkflowsnode_id = waiting_node_idclaimed =transaction do- run_time_ms = execution_data && self.class.compute_run_time_ms(execution_data.steps_array)-- affected =- self- .class- .where(id:, status: :waiting, resume_token:, waiting_until:, timeout_action:)- .update_all(- status: self.class.statuses[:error],- error: message,- finished_at: Time.current,- run_time_ms:,- waiting_node_id: nil,- waiting_until: nil,- resume_token: nil,- timeout_action: nil,- updated_at: Time.current,- )-- next false if affected.zero?+ guards = { status: :waiting, resume_token:, waiting_until:, timeout_action: }+ next false unless fail_if_unchanged!(message, guards)+update_step_status_in_data!(node_id,Executor::Step::WAITING,@@ -221,11 +213,14 @@ module DiscourseWorkflowsendreturn false unless claimed- reload- trigger_error_workflow(StandardError.new(message))- DiscourseWorkflows::ExecutionProgressPublisher.publish(self, refresh: true)- DiscourseWorkflows::WorkflowCallContinuation.child_failed!(self)- true+ notify_failure!(StandardError.new(message))+ end++ def fail_as_stale!+ error = Executor::ExecutionTimedOut.new(pending? ? :queue_timed_out : :run_timed_out)+ return false unless fail_if_unchanged!(error.message, { status:, updated_at: })++ notify_failure!(error)enddef trigger_error_workflow(error, steps: execution_data&.steps_array || [])@@ -296,6 +291,31 @@ module DiscourseWorkflowsendexecution_data.update!(data: full_data)end++ def fail_if_unchanged!(message, guards)+ now = Time.current++ self+ .class+ .where(id:, **guards)+ .update_all(+ status: self.class.statuses[:error],+ error: message,+ finished_at: now,+ run_time_ms: execution_data && self.class.compute_run_time_ms(execution_data.steps_array),+ updated_at: now,+ **CLEARED_WAIT_STATE,+ )+ .positive?+ end++ def notify_failure!(error)+ reload+ trigger_error_workflow(error)+ DiscourseWorkflows::ExecutionProgressPublisher.publish(self, refresh: true)+ DiscourseWorkflows::WorkflowCallContinuation.child_failed!(self)+ true+ endendenddiff --git a/plugins/discourse-workflows/app/services/discourse_workflows/execution/expire_stale.rb b/plugins/discourse-workflows/app/services/discourse_workflows/execution/expire_stale.rbnew file mode 100644index 00000000000..7e6a3a792c9--- /dev/null+++ b/plugins/discourse-workflows/app/services/discourse_workflows/execution/expire_stale.rb@@ -0,0 +1,30 @@+# frozen_string_literal: true++module DiscourseWorkflows+ class Execution::ExpireStale+ include Service::Base++ GRACE_PERIOD = 10.minutes++ policy :workflows_enabled, class_name: DiscourseWorkflows::Policy::WorkflowsEnabled+ model :stale_executions, optional: true++ each :stale_executions do+ step :expire_execution+ end++ private++ def fetch_stale_executions+ timeout = SiteSetting.discourse_workflows_run_timeout_minutes.minutes+ DiscourseWorkflows::Execution+ .stale((timeout + GRACE_PERIOD).ago)+ .includes(:execution_data)+ .limit(500)+ end++ def expire_execution(stale_execution:)+ stale_execution.fail_as_stale!+ end+ end+enddiff --git a/plugins/discourse-workflows/config/locales/server.en.yml b/plugins/discourse-workflows/config/locales/server.en.ymlindex dd3e709db9b..8af78369ed2 100644--- a/plugins/discourse-workflows/config/locales/server.en.yml+++ b/plugins/discourse-workflows/config/locales/server.en.yml@@ -76,6 +76,12 @@ en:invalid_value: "is not valid for this field's type"incompatible_value_for_new_type: "has a current value that is not valid for the new type. Update or clear the value first."approval_timed_out: "Approval timed out"+ run_timed_out:+ one: "Execution ran for more than %{count} minute without pausing"+ other: "Execution ran for more than %{count} minutes without pausing"+ queue_timed_out:+ one: "Execution was queued for more than %{count} minute without starting"+ other: "Execution was queued for more than %{count} minutes without starting"rate_limited:global:one: "This workflow wasn't run because %{count} workflow execution already started in the last minute across all workflows, which is the most allowed. Runs will go through again once fewer workflow executions have started in the last minute."diff --git a/plugins/discourse-workflows/config/settings.yml b/plugins/discourse-workflows/config/settings.ymlindex cab21a70040..3c5b5696fcf 100644--- a/plugins/discourse-workflows/config/settings.yml+++ b/plugins/discourse-workflows/config/settings.yml@@ -30,6 +30,11 @@ plugins:discourse_workflows_max_executions_per_minute_per_workflow:default: 10hidden: true+ discourse_workflows_run_timeout_minutes:+ default: 60+ min: 1+ max: 10080+ hidden: trueworkflow_executions_retention_days:default: 30min: 0diff --git a/plugins/discourse-workflows/lib/discourse_workflows/executor.rb b/plugins/discourse-workflows/lib/discourse_workflows/executor.rbindex 9b75eb965fe..f31de1a1c1d 100644--- a/plugins/discourse-workflows/lib/discourse_workflows/executor.rb+++ b/plugins/discourse-workflows/lib/discourse_workflows/executor.rb@@ -17,6 +17,17 @@ module DiscourseWorkflowsendend+ class ExecutionTimedOut < StandardError+ def initialize(reason = :run_timed_out)+ super(+ I18n.t(+ "discourse_workflows.errors.#{reason}",+ count: SiteSetting.discourse_workflows_run_timeout_minutes,+ ),+ )+ end+ end+delegate :execution, to: :@storedef initialize(workflow, trigger_node_id, trigger_data, options = ExecutionOptions.new)@@ -242,6 +253,8 @@ module DiscourseWorkflowsenddef execute_flow(setup_method, *setup_args)+ @deadline = SiteSetting.discourse_workflows_run_timeout_minutes.minutes.from_now+beginreturn execution if send(setup_method, *setup_args) == falseyield@@ -303,6 +316,7 @@ module DiscourseWorkflowsnode, input_groups, input_sources = @queue[@queue_index]@queue_index += 1execute_node(node, input_groups, input_sources || {})+ check_deadline!endbreak unless flush_partial_waiting_inputs@@ -361,6 +375,7 @@ module DiscourseWorkflowsraise StandardError, I18n.t("discourse_workflows.errors.step_execution.wait_requested")end+ check_deadline!step.mark_waiting!@waiting_node = node@waiting_step = step@@ -586,9 +601,14 @@ module DiscourseWorkflowsworkflow_call_stack: @workflow_call_stack,runtime_state:,static_data_state: @context.static_data_state,+ deadline: @deadline,)end+ def check_deadline!+ raise ExecutionTimedOut if @deadline.past?+ end+def normalize_result(result, node, ports, input_groups)source = "#{node.name} (#{node.type})"ItemContract.validate_output_arrays!(result, source:, ports:)@@ -700,6 +720,8 @@ module DiscourseWorkflowsenddef continued_error_outputs(node, input_groups, error)+ return if error.is_a?(ExecutionTimedOut)+case node_error_mode(node)when "continueRegularOutput"[primary_input_items(input_groups)]@@ -964,6 +986,7 @@ module DiscourseWorkflowstimeout_action:,steps: @steps,)+ return @store.execution unless executionJobs.enqueue_in([resolved - now, 0].max,@@ -976,6 +999,7 @@ module DiscourseWorkflowsdef begin_workflow_call_wait!(wait_request)execution = @store.pause_waiting_execution!(node: @waiting_node, steps: @steps)+ return @store.execution unless executionDiscourseWorkflows::WorkflowCallContinuation.begin_child_call!(execution:,diff --git a/plugins/discourse-workflows/lib/discourse_workflows/executor/execution_store.rb b/plugins/discourse-workflows/lib/discourse_workflows/executor/execution_store.rbindex f19ab12a495..8a010ef7664 100644--- a/plugins/discourse-workflows/lib/discourse_workflows/executor/execution_store.rb+++ b/plugins/discourse-workflows/lib/discourse_workflows/executor/execution_store.rb@@ -65,16 +65,10 @@ module DiscourseWorkflowsenddef finish!(steps:)- save!(steps)- execution.update!(- status: :success,- finished_at: Time.current,- run_time_ms: Execution.compute_run_time_ms(steps),- resume_token: nil,- waiting_node_id: nil,- waiting_until: nil,- timeout_action: nil,- )+ unless persist_if_active!(steps, status: :success, **finished_attributes(steps))+ return execution+ end+publish_execution_run_datapublish_progress(refresh: true)if @options.workflow_call_child?@@ -84,17 +78,13 @@ module DiscourseWorkflowsenddef fail!(error:, steps: [])- save!(steps)- execution.update!(+ attributes = {status: :error,error: error.message.to_s.truncate(1000),- finished_at: Time.current,- run_time_ms: Execution.compute_run_time_ms(steps),- resume_token: nil,- waiting_node_id: nil,- waiting_until: nil,- timeout_action: nil,- )+ **finished_attributes(steps),+ }+ return execution unless persist_if_active!(steps, **attributes)+execution.trigger_error_workflow(error, steps:)publish_execution_run_data(force: @options.draft_execution || @options.workflow_snapshot.present?,@@ -121,16 +111,14 @@ module DiscourseWorkflowsenddef pause_waiting_execution!(node:, waiting_until: nil, timeout_action: nil, steps: [])- execution.transaction do- execution.update!(- status: :waiting,- waiting_node_id: node.id,- waiting_until:,- resume_token: @execution_context.resume_token,- timeout_action:,- )- save!(steps)- end+ attributes = {+ status: :waiting,+ waiting_node_id: node.id,+ waiting_until:,+ resume_token: @execution_context.resume_token,+ timeout_action:,+ }+ return unless persist_if_active!(steps, **attributes)publish_waiting_form_notification(node)publish_progress(refresh: true)@@ -138,13 +126,7 @@ module DiscourseWorkflowsenddef clear_waiting_execution!- execution.update!(- status: :running,- waiting_node_id: nil,- waiting_until: nil,- resume_token: nil,- timeout_action: nil,- )+ execution.update!(status: :running, **Execution::CLEARED_WAIT_STATE)enddef publish_progress(step: nil, refresh: false)@@ -153,6 +135,30 @@ module DiscourseWorkflowsprivate+ def persist_if_active!(steps, **attributes)+ execution.transaction do+ if Execution.where(id: execution.id, status: %i[running waiting]).lock.exists?+ save!(steps)+ execution.update!(attributes)+ true+ else+ Rails.logger.warn(+ "discourse-workflows: execution #{execution.id} is no longer active, " \+ "discarding its #{attributes[:status]} update",+ )+ false+ end+ end+ end++ def finished_attributes(steps)+ {+ finished_at: Time.current,+ run_time_ms: Execution.compute_run_time_ms(steps),+ **Execution::CLEARED_WAIT_STATE,+ }+ end+def persist_execution!(status:, trigger_data:, finished_at: nil, error: nil)@execution = @options.existing_execution || Execution.newcreated = @execution.new_record?diff --git a/plugins/discourse-workflows/lib/discourse_workflows/executor/node_execution_context.rb b/plugins/discourse-workflows/lib/discourse_workflows/executor/node_execution_context.rbindex 48fa0f31fde..b5dcade43f6 100644--- a/plugins/discourse-workflows/lib/discourse_workflows/executor/node_execution_context.rb+++ b/plugins/discourse-workflows/lib/discourse_workflows/executor/node_execution_context.rb@@ -117,7 +117,13 @@ module DiscourseWorkflowsendend- attr_reader :user, :vars, :execution_id, :node_id, :webhook_ctx, :workflow_call_stack+ attr_reader :user,+ :vars,+ :execution_id,+ :node_id,+ :webhook_ctx,+ :workflow_call_stack,+ :deadlinedef initialize(input_items:,@@ -148,7 +154,8 @@ module DiscourseWorkflowswebhook_context: nil,workflow_call_stack: [],runtime_state: RuntimeState.new,- static_data_state: nil+ static_data_state: nil,+ deadline: nil)@input_groups = normalize_input_groups(input_groups, input_items)@input_items = @input_groups[0] || []@@ -185,6 +192,7 @@ module DiscourseWorkflows@webhook_ctx = webhook_context@workflow_call_stack = Array(workflow_call_stack).map(&:to_s)@static_data_state = static_data_state+ @deadline = deadlineend# Static data access.diff --git a/plugins/discourse-workflows/spec/lib/discourse_workflows/executor_timeout_spec.rb b/plugins/discourse-workflows/spec/lib/discourse_workflows/executor_timeout_spec.rbnew file mode 100644index 00000000000..c0c5fdd51f1--- /dev/null+++ b/plugins/discourse-workflows/spec/lib/discourse_workflows/executor_timeout_spec.rb@@ -0,0 +1,125 @@+# frozen_string_literal: true++RSpec.describe DiscourseWorkflows::Executor do+ fab!(:user)++ let(:log_configuration) { {} }+ let(:workflow) do+ graph =+ build_workflow_graph do |g|+ g.node "trigger-1", "trigger:manual"+ g.node "log-1", "action:log", configuration: log_configuration+ g.node "log-2", "action:log"+ g.chain "trigger-1", "log-1", "log-2"+ end+ Fabricate(:discourse_workflows_workflow, created_by: user, published: true, **graph)+ end++ before do+ SiteSetting.discourse_workflows_run_timeout_minutes = 10+ freeze_time+ end++ def run_workflow+ described_class.new(workflow, "trigger-1", {}).run+ end++ def stub_node(node_class, node_id = nil)+ allow_any_instance_of(node_class).to receive(:execute).and_wrap_original do |original, exec_ctx|+ yield exec_ctx if node_id.nil? || exec_ctx.node_id == node_id+ original.call(exec_ctx)+ end+ end++ it "fails the execution instead of running the next node once a node overruns the deadline",+ :aggregate_failures do+ stub_node(DiscourseWorkflows::Nodes::Log::V1, "log-1") { freeze_time(11.minutes.from_now) }++ execution = run_workflow++ expect(execution.reload).to have_attributes(+ status: "error",+ error: "Execution ran for more than 10 minutes without pausing",+ )+ expect(execution.execution_data.find_step(node_id: "log-1")["status"]).to eq("success")+ expect(execution.execution_data.find_step(node_id: "log-2")).to be_nil+ end++ it "fails the execution instead of pausing once a node overruns the deadline" do+ graph =+ build_workflow_graph do |g|+ g.node "trigger-1", "trigger:manual"+ g.node "wait-1", "flow:wait", configuration: { "resume" => "webhook" }+ g.chain "trigger-1", "wait-1"+ end+ wait_workflow =+ Fabricate(:discourse_workflows_workflow, created_by: user, published: true, **graph)+ stub_node(DiscourseWorkflows::Nodes::Wait::V1) { freeze_time(11.minutes.from_now) }++ execution = described_class.new(wait_workflow, "trigger-1", {}).run++ expect(execution.reload).to have_attributes(status: "error", waiting_node_id: nil)+ expect(execution.execution_data.find_step(node_id: "wait-1")["status"]).to eq("error")+ end++ it "does not schedule a resume for an execution that was already failed as stale" do+ graph =+ build_workflow_graph do |g|+ g.node "trigger-1", "trigger:manual"+ g.node "wait-1", "flow:wait", configuration: { "resume" => "webhook" }+ g.chain "trigger-1", "wait-1"+ end+ wait_workflow =+ Fabricate(:discourse_workflows_workflow, created_by: user, published: true, **graph)+ stub_node(DiscourseWorkflows::Nodes::Wait::V1) do |exec_ctx|+ DiscourseWorkflows::Execution.find(exec_ctx.execution_id).update!(status: :error)+ end++ described_class.new(wait_workflow, "trigger-1", {}).run++ expect(Jobs::DiscourseWorkflows::ResumeWaitingExecution.jobs).to be_empty+ end++ context "when the node continues on error" do+ let(:log_configuration) { { "onError" => "continueRegularOutput" } }++ it "does not swallow a timeout raised by the node" do+ stub_node(DiscourseWorkflows::Nodes::Log::V1, "log-1") do+ raise DiscourseWorkflows::Executor::ExecutionTimedOut+ end++ execution = run_workflow++ expect(execution.execution_data.find_step(node_id: "log-1")["status"]).to eq("error")+ end+ end++ it "does not overwrite an execution that was already failed as stale" do+ stub_node(DiscourseWorkflows::Nodes::Log::V1, "log-1") do |exec_ctx|+ DiscourseWorkflows::Execution.find(exec_ctx.execution_id).update!(+ status: :error,+ error: "stale",+ )+ end++ expect(run_workflow.reload).to have_attributes(status: "error", error: "stale")+ end++ it "triggers the error workflow once when the sweep fails a run still stuck in a node" do+ error_graph = build_workflow_graph { |g| g.node "error-1", "trigger:error" }+ error_workflow = Fabricate(:discourse_workflows_workflow, published: true, **error_graph)+ workflow.update!(error_workflow_id: error_workflow.id)+ stub_node(DiscourseWorkflows::Nodes::Log::V1, "log-1") do+ freeze_time(21.minutes.from_now)+ DiscourseWorkflows::Execution::ExpireStale.call+ end++ run_workflow++ error_jobs =+ Jobs::DiscourseWorkflows::ExecuteWorkflow.jobs.select do |job|+ job["args"].first["execution_mode"] == "error_mode"+ end+ expect(error_jobs.size).to eq(1)+ end+enddiff --git a/plugins/discourse-workflows/spec/lib/discourse_workflows/executor_wait_spec.rb b/plugins/discourse-workflows/spec/lib/discourse_workflows/executor_wait_spec.rbindex 3730bb2c531..eba31d7f915 100644--- a/plugins/discourse-workflows/spec/lib/discourse_workflows/executor_wait_spec.rb+++ b/plugins/discourse-workflows/spec/lib/discourse_workflows/executor_wait_spec.rb@@ -218,6 +218,26 @@ RSpec.describe DiscourseWorkflows::Executor doendendend++ it "fails the execution when scheduling the resume fails after the wait was saved" do+ graph =+ build_workflow_graph do |g|+ g.node "trigger-1", "trigger:manual"+ g.node "wait-1", "flow:wait", configuration: { "resume" => "webhook" }+ g.chain "trigger-1", "wait-1"+ end+ workflow =+ Fabricate(:discourse_workflows_workflow, created_by: user, published: true, **graph)+ allow(Jobs).to receive(:enqueue_in).and_raise("boom")++ execution = described_class.new(workflow, "trigger-1", {}).run++ expect(execution.reload).to have_attributes(+ status: "error",+ error: "boom",+ waiting_until: nil,+ )+ endenddescribe ".resume" dodiff --git a/plugins/discourse-workflows/spec/models/discourse_workflows/execution_spec.rb b/plugins/discourse-workflows/spec/models/discourse_workflows/execution_spec.rbindex 3a766f50f72..ded20a6e1c9 100644--- a/plugins/discourse-workflows/spec/models/discourse_workflows/execution_spec.rb+++ b/plugins/discourse-workflows/spec/models/discourse_workflows/execution_spec.rb@@ -128,6 +128,38 @@ RSpec.describe DiscourseWorkflows::Execution doendend+ describe "#fail_as_stale!" do+ fab!(:execution) { Fabricate(:discourse_workflows_execution, status: :running) }++ before { SiteSetting.discourse_workflows_run_timeout_minutes = 10 }++ it "fails a running execution for running too long" do+ expect(execution.fail_as_stale!).to be(true)+ expect(execution.reload).to have_attributes(+ status: "error",+ error: "Execution ran for more than 10 minutes without pausing",+ )+ end++ it "fails a pending execution for waiting too long in the queue" do+ execution.update!(status: :pending)++ execution.fail_as_stale!++ expect(execution.reload).to have_attributes(+ status: "error",+ error: "Execution was queued for more than 10 minutes without starting",+ )+ end++ it "leaves an execution that moved on since being loaded alone" do+ described_class.find(execution.id).update!(status: :success)++ expect(execution.fail_as_stale!).to be(false)+ expect(execution.reload.status).to eq("success")+ end+ end+describe ".compute_run_time_ms" dodef build_step(node_type: "action:code", started_at: nil, finished_at: nil)node = Struct.new(:id, :name, :type).new(SecureRandom.uuid, "Node", node_type)diff --git a/plugins/discourse-workflows/spec/services/discourse_workflows/execution/expire_stale_spec.rb b/plugins/discourse-workflows/spec/services/discourse_workflows/execution/expire_stale_spec.rbnew file mode 100644index 00000000000..a2be03b5b71--- /dev/null+++ b/plugins/discourse-workflows/spec/services/discourse_workflows/execution/expire_stale_spec.rb@@ -0,0 +1,47 @@+# frozen_string_literal: true++RSpec.describe DiscourseWorkflows::Execution::ExpireStale do+ describe ".call" do+ subject(:result) { described_class.call }++ before do+ SiteSetting.discourse_workflows_run_timeout_minutes = 10+ freeze_time+ end++ context "when plugin is disabled" do+ before { SiteSetting.enable_discourse_workflows = false }++ it { is_expected.to fail_a_policy(:workflows_enabled) }+ end++ context "when there are no stale executions" do+ it { is_expected.to run_successfully }+ end++ context "with executions of various ages and statuses" do+ let!(:stale_running) do+ Fabricate(:discourse_workflows_execution, status: :running, updated_at: 21.minutes.ago)+ end+ let!(:stale_pending) do+ Fabricate(:discourse_workflows_execution, status: :pending, updated_at: 21.minutes.ago)+ end+ let!(:stale_waiting) do+ Fabricate(:discourse_workflows_execution, status: :waiting, updated_at: 21.minutes.ago)+ end+ let!(:recent_running) do+ Fabricate(:discourse_workflows_execution, status: :running, updated_at: 19.minutes.ago)+ end++ it "fails only pending and running executions past the timeout and grace period" do+ result++ expect(+ [stale_running, stale_pending, stale_waiting, recent_running].map do |execution|+ execution.reload.status+ end,+ ).to eq(%w[error error waiting running])+ end+ end+ end+enddiff --git a/plugins/discourse-workflows/spec/support/node_execution_helpers.rb b/plugins/discourse-workflows/spec/support/node_execution_helpers.rbindex d8fe308e60f..34fd5d4b113 100644--- a/plugins/discourse-workflows/spec/support/node_execution_helpers.rb+++ b/plugins/discourse-workflows/spec/support/node_execution_helpers.rb@@ -11,6 +11,7 @@ module NodeExecutionHelpersworkflow: nil,node_settings: {},node: nil,+ deadline: nil,&block)item = { "json" => {} } if item.nil? && input_items.nil?@@ -34,6 +35,7 @@ module NodeExecutionHelpersresolver_context: resolver_context,user: user,node_settings: node_settings,+ deadline: deadline,}kwargs[:node_context] = node_context if node_contextkwargs[:workflow] = workflow if workflow