A platform for community discussion. Free, open, simple.
FIX: Mark interrupted workflows as failed when workers stop (#43725)
Régis Hanola9c7727da867
diff --git a/plugins/discourse-workflows/lib/discourse_workflows/executor.rb b/plugins/discourse-workflows/lib/discourse_workflows/executor.rbindex 3873b1f9e43..43c0745732c 100644--- a/plugins/discourse-workflows/lib/discourse_workflows/executor.rb+++ b/plugins/discourse-workflows/lib/discourse_workflows/executor.rb@@ -247,12 +247,17 @@ module DiscourseWorkflowsenddef execute_flow(setup_method, *setup_args, &block)- return execution if send(setup_method, *setup_args) == false- yield- process_queue- @store.finish!(steps: @steps)- rescue ExecutionPaused => e- begin_wait!(e.wait_request)+ begin+ return execution if send(setup_method, *setup_args) == false+ yield+ process_queue+ @store.finish!(steps: @steps)+ rescue ExecutionPaused => e+ begin_wait!(e.wait_request)+ end+ rescue Sidekiq::Shutdown => e+ @store.fail!(error: e, steps: @steps) if @store.owns_execution?+ raiserescue => eraise unless @store.owns_execution?@@ -402,6 +407,9 @@ module DiscourseWorkflowsroute_downstream(node, output_arrays)rescue ExecutionPausedraise+ rescue Sidekiq::Shutdown => e+ step.fail!(e.message)+ raiserescue => eif (handled_outputs = continued_error_outputs(node, input_groups, e))step_log = collect_step_log(exec_ctx, resolver)diff --git a/plugins/discourse-workflows/spec/integration/execution_lifecycle_spec.rb b/plugins/discourse-workflows/spec/integration/execution_lifecycle_spec.rbindex 7876b32cbcb..aaeea3529e3 100644--- a/plugins/discourse-workflows/spec/integration/execution_lifecycle_spec.rb+++ b/plugins/discourse-workflows/spec/integration/execution_lifecycle_spec.rb@@ -41,6 +41,134 @@ RSpec.describe "Workflow execution lifecycle" dostub_request(:get, checkpoint_url).to_return(http_success)end+ describe "job interruption and redelivery" do+ it "records an interrupted manual execution after redelivery", :aggregate_failures do+ stub_request(:get, checkpoint_url).to_raise(Sidekiq::Shutdown).then.to_return(http_success)+ execution =+ DiscourseWorkflows::Execution.create_pending_manual!(+ workflow: workflow,+ trigger_node_id: "trigger-1",+ trigger_data: {+ },+ )+ job = Jobs::DiscourseWorkflows::ExecuteManualWorkflow.new+ args = {+ execution_id: execution.id,+ current_site_id: RailsMultisite::ConnectionManagement.current_db,+ }++ expect { job.perform(args) }.to raise_error(Sidekiq::Shutdown)+ job.perform(args)++ expect(execution.reload).to have_attributes(status: "error", finished_at: be_present)+ expect(workflow.executions.count).to eq(1)+ expect(a_request(:post, effect_url)).to have_been_made.once+ expect(execution.execution_data.steps_array.last["status"]).to eq("error")+ end++ it "preserves one execution and one external effect when a regular job is redelivered",+ :aggregate_failures do+ error_graph = build_workflow_graph { |builder| builder.node "error-1", "trigger:error" }+ error_workflow = Fabricate(:discourse_workflows_workflow, published: true, **error_graph)+ workflow.update!(error_workflow_id: error_workflow.id)+ stub_request(:get, checkpoint_url).to_raise(Sidekiq::Shutdown).then.to_return(http_success)+ job = Jobs::DiscourseWorkflows::ExecuteWorkflow.new+ job.jid = "workflow-redelivery"+ args = {+ workflow_id: workflow.id,+ trigger_node_id: "trigger-1",+ current_site_id: RailsMultisite::ConnectionManagement.current_db,+ }++ expect { job.perform(args) }.to raise_error(Sidekiq::Shutdown)+ job.perform(args)++ expect(workflow.executions.count).to eq(1)+ expect(workflow.executions.where(status: :running)).to be_empty+ expect(a_request(:post, effect_url)).to have_been_made.once+ expect(+ Jobs::DiscourseWorkflows::ExecuteWorkflow.jobs.sole["args"].first["workflow_id"],+ ).to eq(error_workflow.id)+ end++ it "retries when shutdown interrupts execution creation" do+ job = Jobs::DiscourseWorkflows::ExecuteWorkflow.new+ job.jid = "workflow-interrupted-creation"+ args = {+ workflow_id: workflow.id,+ trigger_node_id: "trigger-1",+ current_site_id: RailsMultisite::ConnectionManagement.current_db,+ }+ creation_attempts = 0+ interrupt_creation = -> { raise Sidekiq::Shutdown if (creation_attempts += 1) == 1 }+ DiscourseWorkflows::Execution.set_callback(:create, :before, interrupt_creation)++ expect { job.perform(args) }.to raise_error(Sidekiq::Shutdown)+ expect(workflow.executions).to be_empty+ job.perform(args)++ expect(workflow.executions.sole).to be_success+ expect(a_request(:post, effect_url)).to have_been_made.once+ ensure+ DiscourseWorkflows::Execution.skip_callback(:create, :before, interrupt_creation)+ end++ it "records an interrupted resumed execution after redelivery", :aggregate_failures do+ freeze_time++ waiting_workflow =+ build_workflow do |builder|+ builder.node "wait-1",+ "flow:wait",+ parameters: {+ resume: "time_interval",+ wait_amount: 1,+ wait_unit: "minutes",+ }+ end+ execution = DiscourseWorkflows::Executor.new(waiting_workflow, "trigger-1", {}).run+ freeze_time(2.minutes.from_now)+ stub_request(:get, checkpoint_url).to_raise(Sidekiq::Shutdown).then.to_return(http_success)+ job = Jobs::DiscourseWorkflows::ResumeWaitingExecution.new+ args = {+ execution_id: execution.id,+ resume_token: execution.resume_token,+ current_site_id: RailsMultisite::ConnectionManagement.current_db,+ }++ expect { job.perform(args) }.to raise_error(Sidekiq::Shutdown)+ job.perform(args)++ expect(execution.reload).to have_attributes(status: "error", finished_at: be_present)+ expect(a_request(:post, effect_url)).to have_been_made.once+ end++ it "records shutdown while saving a waiting checkpoint" do+ waiting_workflow =+ build_workflow do |builder|+ builder.node "wait-1", "flow:wait", parameters: { resume: "webhook" }+ end+ checkpoint_saves = 0+ interrupt_checkpoint = -> { raise Sidekiq::Shutdown if (checkpoint_saves += 1) == 1 }+ DiscourseWorkflows::ExecutionData.set_callback(:save, :before, interrupt_checkpoint)++ expect do+ Jobs::DiscourseWorkflows::ExecuteWorkflow.new.perform(+ workflow_id: waiting_workflow.id,+ trigger_node_id: "trigger-1",+ current_site_id: RailsMultisite::ConnectionManagement.current_db,+ )+ end.to raise_error(Sidekiq::Shutdown)++ expect(waiting_workflow.executions.sole).to have_attributes(+ status: "error",+ finished_at: be_present,+ )+ ensure+ DiscourseWorkflows::ExecutionData.skip_callback(:save, :before, interrupt_checkpoint)+ end+ end+describe "wait job ownership" doit "keeps a second webhook wait pending when the first callback is redelivered",:aggregate_failures do