Skip to content

A platform for community discussion. Free, open, simple.

FIX: Mark interrupted workflows as failed when workers stop (#43725)

Régis Hanola9c7727da867
diff --git a/plugins/discourse-workflows/lib/discourse_workflows/executor.rb b/plugins/discourse-workflows/lib/discourse_workflows/executor.rb
index 3873b1f9e43..43c0745732c 100644
--- a/plugins/discourse-workflows/lib/discourse_workflows/executor.rb
+++ b/plugins/discourse-workflows/lib/discourse_workflows/executor.rb
@@ -247,12 +247,17 @@ module DiscourseWorkflows
end
def execute_flow(setup_method, *setup_args, &block)
- return execution if send(setup_method, *setup_args) == false
- yield
- process_queue
- @store.finish!(steps: @steps)
- rescue ExecutionPaused => e
- begin_wait!(e.wait_request)
+ begin
+ return execution if send(setup_method, *setup_args) == false
+ yield
+ process_queue
+ @store.finish!(steps: @steps)
+ rescue ExecutionPaused => e
+ begin_wait!(e.wait_request)
+ end
+ rescue Sidekiq::Shutdown => e
+ @store.fail!(error: e, steps: @steps) if @store.owns_execution?
+ raise
rescue => e
raise unless @store.owns_execution?
@@ -402,6 +407,9 @@ module DiscourseWorkflows
route_downstream(node, output_arrays)
rescue ExecutionPaused
raise
+ rescue Sidekiq::Shutdown => e
+ step.fail!(e.message)
+ raise
rescue => e
if (handled_outputs = continued_error_outputs(node, input_groups, e))
step_log = collect_step_log(exec_ctx, resolver)
diff --git a/plugins/discourse-workflows/spec/integration/execution_lifecycle_spec.rb b/plugins/discourse-workflows/spec/integration/execution_lifecycle_spec.rb
index 7876b32cbcb..aaeea3529e3 100644
--- a/plugins/discourse-workflows/spec/integration/execution_lifecycle_spec.rb
+++ b/plugins/discourse-workflows/spec/integration/execution_lifecycle_spec.rb
@@ -41,6 +41,134 @@ RSpec.describe "Workflow execution lifecycle" do
stub_request(:get, checkpoint_url).to_return(http_success)
end
+ describe "job interruption and redelivery" do
+ it "records an interrupted manual execution after redelivery", :aggregate_failures do
+ stub_request(:get, checkpoint_url).to_raise(Sidekiq::Shutdown).then.to_return(http_success)
+ execution =
+ DiscourseWorkflows::Execution.create_pending_manual!(
+ workflow: workflow,
+ trigger_node_id: "trigger-1",
+ trigger_data: {
+ },
+ )
+ job = Jobs::DiscourseWorkflows::ExecuteManualWorkflow.new
+ args = {
+ execution_id: execution.id,
+ current_site_id: RailsMultisite::ConnectionManagement.current_db,
+ }
+
+ expect { job.perform(args) }.to raise_error(Sidekiq::Shutdown)
+ job.perform(args)
+
+ expect(execution.reload).to have_attributes(status: "error", finished_at: be_present)
+ expect(workflow.executions.count).to eq(1)
+ expect(a_request(:post, effect_url)).to have_been_made.once
+ expect(execution.execution_data.steps_array.last["status"]).to eq("error")
+ end
+
+ it "preserves one execution and one external effect when a regular job is redelivered",
+ :aggregate_failures do
+ error_graph = build_workflow_graph { |builder| builder.node "error-1", "trigger:error" }
+ error_workflow = Fabricate(:discourse_workflows_workflow, published: true, **error_graph)
+ workflow.update!(error_workflow_id: error_workflow.id)
+ stub_request(:get, checkpoint_url).to_raise(Sidekiq::Shutdown).then.to_return(http_success)
+ job = Jobs::DiscourseWorkflows::ExecuteWorkflow.new
+ job.jid = "workflow-redelivery"
+ args = {
+ workflow_id: workflow.id,
+ trigger_node_id: "trigger-1",
+ current_site_id: RailsMultisite::ConnectionManagement.current_db,
+ }
+
+ expect { job.perform(args) }.to raise_error(Sidekiq::Shutdown)
+ job.perform(args)
+
+ expect(workflow.executions.count).to eq(1)
+ expect(workflow.executions.where(status: :running)).to be_empty
+ expect(a_request(:post, effect_url)).to have_been_made.once
+ expect(
+ Jobs::DiscourseWorkflows::ExecuteWorkflow.jobs.sole["args"].first["workflow_id"],
+ ).to eq(error_workflow.id)
+ end
+
+ it "retries when shutdown interrupts execution creation" do
+ job = Jobs::DiscourseWorkflows::ExecuteWorkflow.new
+ job.jid = "workflow-interrupted-creation"
+ args = {
+ workflow_id: workflow.id,
+ trigger_node_id: "trigger-1",
+ current_site_id: RailsMultisite::ConnectionManagement.current_db,
+ }
+ creation_attempts = 0
+ interrupt_creation = -> { raise Sidekiq::Shutdown if (creation_attempts += 1) == 1 }
+ DiscourseWorkflows::Execution.set_callback(:create, :before, interrupt_creation)
+
+ expect { job.perform(args) }.to raise_error(Sidekiq::Shutdown)
+ expect(workflow.executions).to be_empty
+ job.perform(args)
+
+ expect(workflow.executions.sole).to be_success
+ expect(a_request(:post, effect_url)).to have_been_made.once
+ ensure
+ DiscourseWorkflows::Execution.skip_callback(:create, :before, interrupt_creation)
+ end
+
+ it "records an interrupted resumed execution after redelivery", :aggregate_failures do
+ freeze_time
+
+ waiting_workflow =
+ build_workflow do |builder|
+ builder.node "wait-1",
+ "flow:wait",
+ parameters: {
+ resume: "time_interval",
+ wait_amount: 1,
+ wait_unit: "minutes",
+ }
+ end
+ execution = DiscourseWorkflows::Executor.new(waiting_workflow, "trigger-1", {}).run
+ freeze_time(2.minutes.from_now)
+ stub_request(:get, checkpoint_url).to_raise(Sidekiq::Shutdown).then.to_return(http_success)
+ job = Jobs::DiscourseWorkflows::ResumeWaitingExecution.new
+ args = {
+ execution_id: execution.id,
+ resume_token: execution.resume_token,
+ current_site_id: RailsMultisite::ConnectionManagement.current_db,
+ }
+
+ expect { job.perform(args) }.to raise_error(Sidekiq::Shutdown)
+ job.perform(args)
+
+ expect(execution.reload).to have_attributes(status: "error", finished_at: be_present)
+ expect(a_request(:post, effect_url)).to have_been_made.once
+ end
+
+ it "records shutdown while saving a waiting checkpoint" do
+ waiting_workflow =
+ build_workflow do |builder|
+ builder.node "wait-1", "flow:wait", parameters: { resume: "webhook" }
+ end
+ checkpoint_saves = 0
+ interrupt_checkpoint = -> { raise Sidekiq::Shutdown if (checkpoint_saves += 1) == 1 }
+ DiscourseWorkflows::ExecutionData.set_callback(:save, :before, interrupt_checkpoint)
+
+ expect do
+ Jobs::DiscourseWorkflows::ExecuteWorkflow.new.perform(
+ workflow_id: waiting_workflow.id,
+ trigger_node_id: "trigger-1",
+ current_site_id: RailsMultisite::ConnectionManagement.current_db,
+ )
+ end.to raise_error(Sidekiq::Shutdown)
+
+ expect(waiting_workflow.executions.sole).to have_attributes(
+ status: "error",
+ finished_at: be_present,
+ )
+ ensure
+ DiscourseWorkflows::ExecutionData.skip_callback(:save, :before, interrupt_checkpoint)
+ end
+ end
+
describe "wait job ownership" do
it "keeps a second webhook wait pending when the first callback is redelivered",
:aggregate_failures do