Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: Time out workflow runs that never finish

Régis Hanol4fd6529a61d4
diff --git a/plugins/discourse-workflows/app/jobs/scheduled/discourse_workflows/expire_stale_executions.rb b/plugins/discourse-workflows/app/jobs/scheduled/discourse_workflows/expire_stale_executions.rb
new file mode 100644
index 00000000000..e4e1b76106b
--- /dev/null
+++ b/plugins/discourse-workflows/app/jobs/scheduled/discourse_workflows/expire_stale_executions.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Jobs
+ module DiscourseWorkflows
+ class ExpireStaleExecutions < ::Jobs::Scheduled
+ every 5.minutes
+
+ def execute(_args = nil)
+ ::DiscourseWorkflows::Execution::ExpireStale.call
+ end
+ end
+ end
+end
diff --git a/plugins/discourse-workflows/app/models/discourse_workflows/execution.rb b/plugins/discourse-workflows/app/models/discourse_workflows/execution.rb
index 5e395fd7b06..a799f49dc5a 100644
--- a/plugins/discourse-workflows/app/models/discourse_workflows/execution.rb
+++ b/plugins/discourse-workflows/app/models/discourse_workflows/execution.rb
@@ -29,6 +29,14 @@ module DiscourseWorkflows
-> { where.not(started_at: nil, finished_at: nil).where(status: %i[success error]) }
scope :expired_waiting, -> { where(status: :waiting).where("waiting_until < ?", Time.current) }
+ scope :stale, ->(cutoff) { where(status: %i[pending running]).where("updated_at < ?", cutoff) }
+
+ CLEARED_WAIT_STATE = {
+ resume_token: nil,
+ waiting_node_id: nil,
+ waiting_until: nil,
+ timeout_action: nil,
+ }.freeze
scope :with_snapshot_summary,
-> do
@@ -192,25 +200,9 @@ module DiscourseWorkflows
node_id = waiting_node_id
claimed =
transaction do
- run_time_ms = execution_data && self.class.compute_run_time_ms(execution_data.steps_array)
-
- affected =
- self
- .class
- .where(id:, status: :waiting, resume_token:, waiting_until:, timeout_action:)
- .update_all(
- status: self.class.statuses[:error],
- error: message,
- finished_at: Time.current,
- run_time_ms:,
- waiting_node_id: nil,
- waiting_until: nil,
- resume_token: nil,
- timeout_action: nil,
- updated_at: Time.current,
- )
-
- next false if affected.zero?
+ guards = { status: :waiting, resume_token:, waiting_until:, timeout_action: }
+ next false unless fail_if_unchanged!(message, guards)
+
update_step_status_in_data!(
node_id,
Executor::Step::WAITING,
@@ -221,11 +213,14 @@ module DiscourseWorkflows
end
return false unless claimed
- reload
- trigger_error_workflow(StandardError.new(message))
- DiscourseWorkflows::ExecutionProgressPublisher.publish(self, refresh: true)
- DiscourseWorkflows::WorkflowCallContinuation.child_failed!(self)
- true
+ notify_failure!(StandardError.new(message))
+ end
+
+ def fail_as_stale!
+ error = Executor::ExecutionTimedOut.new(pending? ? :queue_timed_out : :run_timed_out)
+ return false unless fail_if_unchanged!(error.message, { status:, updated_at: })
+
+ notify_failure!(error)
end
def trigger_error_workflow(error, steps: execution_data&.steps_array || [])
@@ -296,6 +291,31 @@ module DiscourseWorkflows
end
execution_data.update!(data: full_data)
end
+
+ def fail_if_unchanged!(message, guards)
+ now = Time.current
+
+ self
+ .class
+ .where(id:, **guards)
+ .update_all(
+ status: self.class.statuses[:error],
+ error: message,
+ finished_at: now,
+ run_time_ms: execution_data && self.class.compute_run_time_ms(execution_data.steps_array),
+ updated_at: now,
+ **CLEARED_WAIT_STATE,
+ )
+ .positive?
+ end
+
+ def notify_failure!(error)
+ reload
+ trigger_error_workflow(error)
+ DiscourseWorkflows::ExecutionProgressPublisher.publish(self, refresh: true)
+ DiscourseWorkflows::WorkflowCallContinuation.child_failed!(self)
+ true
+ end
end
end
diff --git a/plugins/discourse-workflows/app/services/discourse_workflows/execution/expire_stale.rb b/plugins/discourse-workflows/app/services/discourse_workflows/execution/expire_stale.rb
new file mode 100644
index 00000000000..7e6a3a792c9
--- /dev/null
+++ b/plugins/discourse-workflows/app/services/discourse_workflows/execution/expire_stale.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module DiscourseWorkflows
+ class Execution::ExpireStale
+ include Service::Base
+
+ GRACE_PERIOD = 10.minutes
+
+ policy :workflows_enabled, class_name: DiscourseWorkflows::Policy::WorkflowsEnabled
+ model :stale_executions, optional: true
+
+ each :stale_executions do
+ step :expire_execution
+ end
+
+ private
+
+ def fetch_stale_executions
+ timeout = SiteSetting.discourse_workflows_run_timeout_minutes.minutes
+ DiscourseWorkflows::Execution
+ .stale((timeout + GRACE_PERIOD).ago)
+ .includes(:execution_data)
+ .limit(500)
+ end
+
+ def expire_execution(stale_execution:)
+ stale_execution.fail_as_stale!
+ end
+ end
+end
diff --git a/plugins/discourse-workflows/config/locales/server.en.yml b/plugins/discourse-workflows/config/locales/server.en.yml
index dd3e709db9b..8af78369ed2 100644
--- a/plugins/discourse-workflows/config/locales/server.en.yml
+++ b/plugins/discourse-workflows/config/locales/server.en.yml
@@ -76,6 +76,12 @@ en:
invalid_value: "is not valid for this field's type"
incompatible_value_for_new_type: "has a current value that is not valid for the new type. Update or clear the value first."
approval_timed_out: "Approval timed out"
+ run_timed_out:
+ one: "Execution ran for more than %{count} minute without pausing"
+ other: "Execution ran for more than %{count} minutes without pausing"
+ queue_timed_out:
+ one: "Execution was queued for more than %{count} minute without starting"
+ other: "Execution was queued for more than %{count} minutes without starting"
rate_limited:
global:
one: "This workflow wasn't run because %{count} workflow execution already started in the last minute across all workflows, which is the most allowed. Runs will go through again once fewer workflow executions have started in the last minute."
diff --git a/plugins/discourse-workflows/config/settings.yml b/plugins/discourse-workflows/config/settings.yml
index cab21a70040..3c5b5696fcf 100644
--- a/plugins/discourse-workflows/config/settings.yml
+++ b/plugins/discourse-workflows/config/settings.yml
@@ -30,6 +30,11 @@ plugins:
discourse_workflows_max_executions_per_minute_per_workflow:
default: 10
hidden: true
+ discourse_workflows_run_timeout_minutes:
+ default: 60
+ min: 1
+ max: 10080
+ hidden: true
workflow_executions_retention_days:
default: 30
min: 0
diff --git a/plugins/discourse-workflows/lib/discourse_workflows/executor.rb b/plugins/discourse-workflows/lib/discourse_workflows/executor.rb
index 9b75eb965fe..f31de1a1c1d 100644
--- a/plugins/discourse-workflows/lib/discourse_workflows/executor.rb
+++ b/plugins/discourse-workflows/lib/discourse_workflows/executor.rb
@@ -17,6 +17,17 @@ module DiscourseWorkflows
end
end
+ class ExecutionTimedOut < StandardError
+ def initialize(reason = :run_timed_out)
+ super(
+ I18n.t(
+ "discourse_workflows.errors.#{reason}",
+ count: SiteSetting.discourse_workflows_run_timeout_minutes,
+ ),
+ )
+ end
+ end
+
delegate :execution, to: :@store
def initialize(workflow, trigger_node_id, trigger_data, options = ExecutionOptions.new)
@@ -242,6 +253,8 @@ module DiscourseWorkflows
end
def execute_flow(setup_method, *setup_args)
+ @deadline = SiteSetting.discourse_workflows_run_timeout_minutes.minutes.from_now
+
begin
return execution if send(setup_method, *setup_args) == false
yield
@@ -303,6 +316,7 @@ module DiscourseWorkflows
node, input_groups, input_sources = @queue[@queue_index]
@queue_index += 1
execute_node(node, input_groups, input_sources || {})
+ check_deadline!
end
break unless flush_partial_waiting_inputs
@@ -361,6 +375,7 @@ module DiscourseWorkflows
raise StandardError, I18n.t("discourse_workflows.errors.step_execution.wait_requested")
end
+ check_deadline!
step.mark_waiting!
@waiting_node = node
@waiting_step = step
@@ -586,9 +601,14 @@ module DiscourseWorkflows
workflow_call_stack: @workflow_call_stack,
runtime_state:,
static_data_state: @context.static_data_state,
+ deadline: @deadline,
)
end
+ def check_deadline!
+ raise ExecutionTimedOut if @deadline.past?
+ end
+
def normalize_result(result, node, ports, input_groups)
source = "#{node.name} (#{node.type})"
ItemContract.validate_output_arrays!(result, source:, ports:)
@@ -700,6 +720,8 @@ module DiscourseWorkflows
end
def continued_error_outputs(node, input_groups, error)
+ return if error.is_a?(ExecutionTimedOut)
+
case node_error_mode(node)
when "continueRegularOutput"
[primary_input_items(input_groups)]
@@ -964,6 +986,7 @@ module DiscourseWorkflows
timeout_action:,
steps: @steps,
)
+ return @store.execution unless execution
Jobs.enqueue_in(
[resolved - now, 0].max,
@@ -976,6 +999,7 @@ module DiscourseWorkflows
def begin_workflow_call_wait!(wait_request)
execution = @store.pause_waiting_execution!(node: @waiting_node, steps: @steps)
+ return @store.execution unless execution
DiscourseWorkflows::WorkflowCallContinuation.begin_child_call!(
execution:,
diff --git a/plugins/discourse-workflows/lib/discourse_workflows/executor/execution_store.rb b/plugins/discourse-workflows/lib/discourse_workflows/executor/execution_store.rb
index f19ab12a495..8a010ef7664 100644
--- a/plugins/discourse-workflows/lib/discourse_workflows/executor/execution_store.rb
+++ b/plugins/discourse-workflows/lib/discourse_workflows/executor/execution_store.rb
@@ -65,16 +65,10 @@ module DiscourseWorkflows
end
def finish!(steps:)
- save!(steps)
- execution.update!(
- status: :success,
- finished_at: Time.current,
- run_time_ms: Execution.compute_run_time_ms(steps),
- resume_token: nil,
- waiting_node_id: nil,
- waiting_until: nil,
- timeout_action: nil,
- )
+ unless persist_if_active!(steps, status: :success, **finished_attributes(steps))
+ return execution
+ end
+
publish_execution_run_data
publish_progress(refresh: true)
if @options.workflow_call_child?
@@ -84,17 +78,13 @@ module DiscourseWorkflows
end
def fail!(error:, steps: [])
- save!(steps)
- execution.update!(
+ attributes = {
status: :error,
error: error.message.to_s.truncate(1000),
- finished_at: Time.current,
- run_time_ms: Execution.compute_run_time_ms(steps),
- resume_token: nil,
- waiting_node_id: nil,
- waiting_until: nil,
- timeout_action: nil,
- )
+ **finished_attributes(steps),
+ }
+ return execution unless persist_if_active!(steps, **attributes)
+
execution.trigger_error_workflow(error, steps:)
publish_execution_run_data(
force: @options.draft_execution || @options.workflow_snapshot.present?,
@@ -121,16 +111,14 @@ module DiscourseWorkflows
end
def pause_waiting_execution!(node:, waiting_until: nil, timeout_action: nil, steps: [])
- execution.transaction do
- execution.update!(
- status: :waiting,
- waiting_node_id: node.id,
- waiting_until:,
- resume_token: @execution_context.resume_token,
- timeout_action:,
- )
- save!(steps)
- end
+ attributes = {
+ status: :waiting,
+ waiting_node_id: node.id,
+ waiting_until:,
+ resume_token: @execution_context.resume_token,
+ timeout_action:,
+ }
+ return unless persist_if_active!(steps, **attributes)
publish_waiting_form_notification(node)
publish_progress(refresh: true)
@@ -138,13 +126,7 @@ module DiscourseWorkflows
end
def clear_waiting_execution!
- execution.update!(
- status: :running,
- waiting_node_id: nil,
- waiting_until: nil,
- resume_token: nil,
- timeout_action: nil,
- )
+ execution.update!(status: :running, **Execution::CLEARED_WAIT_STATE)
end
def publish_progress(step: nil, refresh: false)
@@ -153,6 +135,30 @@ module DiscourseWorkflows
private
+ def persist_if_active!(steps, **attributes)
+ execution.transaction do
+ if Execution.where(id: execution.id, status: %i[running waiting]).lock.exists?
+ save!(steps)
+ execution.update!(attributes)
+ true
+ else
+ Rails.logger.warn(
+ "discourse-workflows: execution #{execution.id} is no longer active, " \
+ "discarding its #{attributes[:status]} update",
+ )
+ false
+ end
+ end
+ end
+
+ def finished_attributes(steps)
+ {
+ finished_at: Time.current,
+ run_time_ms: Execution.compute_run_time_ms(steps),
+ **Execution::CLEARED_WAIT_STATE,
+ }
+ end
+
def persist_execution!(status:, trigger_data:, finished_at: nil, error: nil)
@execution = @options.existing_execution || Execution.new
created = @execution.new_record?
diff --git a/plugins/discourse-workflows/lib/discourse_workflows/executor/node_execution_context.rb b/plugins/discourse-workflows/lib/discourse_workflows/executor/node_execution_context.rb
index 48fa0f31fde..b5dcade43f6 100644
--- a/plugins/discourse-workflows/lib/discourse_workflows/executor/node_execution_context.rb
+++ b/plugins/discourse-workflows/lib/discourse_workflows/executor/node_execution_context.rb
@@ -117,7 +117,13 @@ module DiscourseWorkflows
end
end
- attr_reader :user, :vars, :execution_id, :node_id, :webhook_ctx, :workflow_call_stack
+ attr_reader :user,
+ :vars,
+ :execution_id,
+ :node_id,
+ :webhook_ctx,
+ :workflow_call_stack,
+ :deadline
def initialize(
input_items:,
@@ -148,7 +154,8 @@ module DiscourseWorkflows
webhook_context: nil,
workflow_call_stack: [],
runtime_state: RuntimeState.new,
- static_data_state: nil
+ static_data_state: nil,
+ deadline: nil
)
@input_groups = normalize_input_groups(input_groups, input_items)
@input_items = @input_groups[0] || []
@@ -185,6 +192,7 @@ module DiscourseWorkflows
@webhook_ctx = webhook_context
@workflow_call_stack = Array(workflow_call_stack).map(&:to_s)
@static_data_state = static_data_state
+ @deadline = deadline
end
# Static data access.
diff --git a/plugins/discourse-workflows/spec/lib/discourse_workflows/executor_timeout_spec.rb b/plugins/discourse-workflows/spec/lib/discourse_workflows/executor_timeout_spec.rb
new file mode 100644
index 00000000000..c0c5fdd51f1
--- /dev/null
+++ b/plugins/discourse-workflows/spec/lib/discourse_workflows/executor_timeout_spec.rb
@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseWorkflows::Executor do
+ fab!(:user)
+
+ let(:log_configuration) { {} }
+ let(:workflow) do
+ graph =
+ build_workflow_graph do |g|
+ g.node "trigger-1", "trigger:manual"
+ g.node "log-1", "action:log", configuration: log_configuration
+ g.node "log-2", "action:log"
+ g.chain "trigger-1", "log-1", "log-2"
+ end
+ Fabricate(:discourse_workflows_workflow, created_by: user, published: true, **graph)
+ end
+
+ before do
+ SiteSetting.discourse_workflows_run_timeout_minutes = 10
+ freeze_time
+ end
+
+ def run_workflow
+ described_class.new(workflow, "trigger-1", {}).run
+ end
+
+ def stub_node(node_class, node_id = nil)
+ allow_any_instance_of(node_class).to receive(:execute).and_wrap_original do |original, exec_ctx|
+ yield exec_ctx if node_id.nil? || exec_ctx.node_id == node_id
+ original.call(exec_ctx)
+ end
+ end
+
+ it "fails the execution instead of running the next node once a node overruns the deadline",
+ :aggregate_failures do
+ stub_node(DiscourseWorkflows::Nodes::Log::V1, "log-1") { freeze_time(11.minutes.from_now) }
+
+ execution = run_workflow
+
+ expect(execution.reload).to have_attributes(
+ status: "error",
+ error: "Execution ran for more than 10 minutes without pausing",
+ )
+ expect(execution.execution_data.find_step(node_id: "log-1")["status"]).to eq("success")
+ expect(execution.execution_data.find_step(node_id: "log-2")).to be_nil
+ end
+
+ it "fails the execution instead of pausing once a node overruns the deadline" do
+ graph =
+ build_workflow_graph do |g|
+ g.node "trigger-1", "trigger:manual"
+ g.node "wait-1", "flow:wait", configuration: { "resume" => "webhook" }
+ g.chain "trigger-1", "wait-1"
+ end
+ wait_workflow =
+ Fabricate(:discourse_workflows_workflow, created_by: user, published: true, **graph)
+ stub_node(DiscourseWorkflows::Nodes::Wait::V1) { freeze_time(11.minutes.from_now) }
+
+ execution = described_class.new(wait_workflow, "trigger-1", {}).run
+
+ expect(execution.reload).to have_attributes(status: "error", waiting_node_id: nil)
+ expect(execution.execution_data.find_step(node_id: "wait-1")["status"]).to eq("error")
+ end
+
+ it "does not schedule a resume for an execution that was already failed as stale" do
+ graph =
+ build_workflow_graph do |g|
+ g.node "trigger-1", "trigger:manual"
+ g.node "wait-1", "flow:wait", configuration: { "resume" => "webhook" }
+ g.chain "trigger-1", "wait-1"
+ end
+ wait_workflow =
+ Fabricate(:discourse_workflows_workflow, created_by: user, published: true, **graph)
+ stub_node(DiscourseWorkflows::Nodes::Wait::V1) do |exec_ctx|
+ DiscourseWorkflows::Execution.find(exec_ctx.execution_id).update!(status: :error)
+ end
+
+ described_class.new(wait_workflow, "trigger-1", {}).run
+
+ expect(Jobs::DiscourseWorkflows::ResumeWaitingExecution.jobs).to be_empty
+ end
+
+ context "when the node continues on error" do
+ let(:log_configuration) { { "onError" => "continueRegularOutput" } }
+
+ it "does not swallow a timeout raised by the node" do
+ stub_node(DiscourseWorkflows::Nodes::Log::V1, "log-1") do
+ raise DiscourseWorkflows::Executor::ExecutionTimedOut
+ end
+
+ execution = run_workflow
+
+ expect(execution.execution_data.find_step(node_id: "log-1")["status"]).to eq("error")
+ end
+ end
+
+ it "does not overwrite an execution that was already failed as stale" do
+ stub_node(DiscourseWorkflows::Nodes::Log::V1, "log-1") do |exec_ctx|
+ DiscourseWorkflows::Execution.find(exec_ctx.execution_id).update!(
+ status: :error,
+ error: "stale",
+ )
+ end
+
+ expect(run_workflow.reload).to have_attributes(status: "error", error: "stale")
+ end
+
+ it "triggers the error workflow once when the sweep fails a run still stuck in a node" do
+ error_graph = build_workflow_graph { |g| g.node "error-1", "trigger:error" }
+ error_workflow = Fabricate(:discourse_workflows_workflow, published: true, **error_graph)
+ workflow.update!(error_workflow_id: error_workflow.id)
+ stub_node(DiscourseWorkflows::Nodes::Log::V1, "log-1") do
+ freeze_time(21.minutes.from_now)
+ DiscourseWorkflows::Execution::ExpireStale.call
+ end
+
+ run_workflow
+
+ error_jobs =
+ Jobs::DiscourseWorkflows::ExecuteWorkflow.jobs.select do |job|
+ job["args"].first["execution_mode"] == "error_mode"
+ end
+ expect(error_jobs.size).to eq(1)
+ end
+end
diff --git a/plugins/discourse-workflows/spec/lib/discourse_workflows/executor_wait_spec.rb b/plugins/discourse-workflows/spec/lib/discourse_workflows/executor_wait_spec.rb
index 3730bb2c531..eba31d7f915 100644
--- a/plugins/discourse-workflows/spec/lib/discourse_workflows/executor_wait_spec.rb
+++ b/plugins/discourse-workflows/spec/lib/discourse_workflows/executor_wait_spec.rb
@@ -218,6 +218,26 @@ RSpec.describe DiscourseWorkflows::Executor do
end
end
end
+
+ it "fails the execution when scheduling the resume fails after the wait was saved" do
+ graph =
+ build_workflow_graph do |g|
+ g.node "trigger-1", "trigger:manual"
+ g.node "wait-1", "flow:wait", configuration: { "resume" => "webhook" }
+ g.chain "trigger-1", "wait-1"
+ end
+ workflow =
+ Fabricate(:discourse_workflows_workflow, created_by: user, published: true, **graph)
+ allow(Jobs).to receive(:enqueue_in).and_raise("boom")
+
+ execution = described_class.new(workflow, "trigger-1", {}).run
+
+ expect(execution.reload).to have_attributes(
+ status: "error",
+ error: "boom",
+ waiting_until: nil,
+ )
+ end
end
describe ".resume" do
diff --git a/plugins/discourse-workflows/spec/models/discourse_workflows/execution_spec.rb b/plugins/discourse-workflows/spec/models/discourse_workflows/execution_spec.rb
index 3a766f50f72..ded20a6e1c9 100644
--- a/plugins/discourse-workflows/spec/models/discourse_workflows/execution_spec.rb
+++ b/plugins/discourse-workflows/spec/models/discourse_workflows/execution_spec.rb
@@ -128,6 +128,38 @@ RSpec.describe DiscourseWorkflows::Execution do
end
end
+ describe "#fail_as_stale!" do
+ fab!(:execution) { Fabricate(:discourse_workflows_execution, status: :running) }
+
+ before { SiteSetting.discourse_workflows_run_timeout_minutes = 10 }
+
+ it "fails a running execution for running too long" do
+ expect(execution.fail_as_stale!).to be(true)
+ expect(execution.reload).to have_attributes(
+ status: "error",
+ error: "Execution ran for more than 10 minutes without pausing",
+ )
+ end
+
+ it "fails a pending execution for waiting too long in the queue" do
+ execution.update!(status: :pending)
+
+ execution.fail_as_stale!
+
+ expect(execution.reload).to have_attributes(
+ status: "error",
+ error: "Execution was queued for more than 10 minutes without starting",
+ )
+ end
+
+ it "leaves an execution that moved on since being loaded alone" do
+ described_class.find(execution.id).update!(status: :success)
+
+ expect(execution.fail_as_stale!).to be(false)
+ expect(execution.reload.status).to eq("success")
+ end
+ end
+
describe ".compute_run_time_ms" do
def build_step(node_type: "action:code", started_at: nil, finished_at: nil)
node = Struct.new(:id, :name, :type).new(SecureRandom.uuid, "Node", node_type)
diff --git a/plugins/discourse-workflows/spec/services/discourse_workflows/execution/expire_stale_spec.rb b/plugins/discourse-workflows/spec/services/discourse_workflows/execution/expire_stale_spec.rb
new file mode 100644
index 00000000000..a2be03b5b71
--- /dev/null
+++ b/plugins/discourse-workflows/spec/services/discourse_workflows/execution/expire_stale_spec.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseWorkflows::Execution::ExpireStale do
+ describe ".call" do
+ subject(:result) { described_class.call }
+
+ before do
+ SiteSetting.discourse_workflows_run_timeout_minutes = 10
+ freeze_time
+ end
+
+ context "when plugin is disabled" do
+ before { SiteSetting.enable_discourse_workflows = false }
+
+ it { is_expected.to fail_a_policy(:workflows_enabled) }
+ end
+
+ context "when there are no stale executions" do
+ it { is_expected.to run_successfully }
+ end
+
+ context "with executions of various ages and statuses" do
+ let!(:stale_running) do
+ Fabricate(:discourse_workflows_execution, status: :running, updated_at: 21.minutes.ago)
+ end
+ let!(:stale_pending) do
+ Fabricate(:discourse_workflows_execution, status: :pending, updated_at: 21.minutes.ago)
+ end
+ let!(:stale_waiting) do
+ Fabricate(:discourse_workflows_execution, status: :waiting, updated_at: 21.minutes.ago)
+ end
+ let!(:recent_running) do
+ Fabricate(:discourse_workflows_execution, status: :running, updated_at: 19.minutes.ago)
+ end
+
+ it "fails only pending and running executions past the timeout and grace period" do
+ result
+
+ expect(
+ [stale_running, stale_pending, stale_waiting, recent_running].map do |execution|
+ execution.reload.status
+ end,
+ ).to eq(%w[error error waiting running])
+ end
+ end
+ end
+end
diff --git a/plugins/discourse-workflows/spec/support/node_execution_helpers.rb b/plugins/discourse-workflows/spec/support/node_execution_helpers.rb
index d8fe308e60f..34fd5d4b113 100644
--- a/plugins/discourse-workflows/spec/support/node_execution_helpers.rb
+++ b/plugins/discourse-workflows/spec/support/node_execution_helpers.rb
@@ -11,6 +11,7 @@ module NodeExecutionHelpers
workflow: nil,
node_settings: {},
node: nil,
+ deadline: nil,
&block
)
item = { "json" => {} } if item.nil? && input_items.nil?
@@ -34,6 +35,7 @@ module NodeExecutionHelpers
resolver_context: resolver_context,
user: user,
node_settings: node_settings,
+ deadline: deadline,
}
kwargs[:node_context] = node_context if node_context
kwargs[:workflow] = workflow if workflow