A platform for community discussion. Free, open, simple.
# frozen_string_literal: true
module DiscourseWorkflows
class Executor
MAX_ITERATIONS = 1000
MAX_WAIT_DURATION_SECONDS = 30.days.to_i
MAX_NODE_OUTPUT_BYTES = 50.megabytes
class ExecutionPaused < StandardError
attr_reader :wait_request
delegate :waiting_until, to: :@wait_request
def initialize(wait_request)
@wait_request = wait_request
super("Execution paused")
end
end
class ExecutionTimedOut < StandardError
def initialize(reason = :run_timed_out)
super(
I18n.t(
"discourse_workflows.errors.#{reason}",
count: SiteSetting.discourse_workflows_run_timeout_minutes,
),
)
end
end
delegate :execution, to: :@store
def initialize(workflow, trigger_node_id, trigger_data, options = ExecutionOptions.new)
@workflow = workflow
@trigger_node_id = trigger_node_id.to_s
@trigger_data =
if trigger_data.is_a?(Array)
trigger_data.map(&:deep_stringify_keys)
else
trigger_data.deep_stringify_keys
end
@options = options
workflow_version = @options.workflow_version
workflow_nodes =
if @options.workflow_snapshot
@options.workflow_snapshot.to_h["nodes"]
elsif workflow_version
workflow_version.nodes
elsif @options.draft_execution
@workflow.nodes
else
@workflow.published_nodes
end
@context =
ExecutionContext.new(
workflow: @workflow,
trigger_data: @trigger_data,
user: @options.user,
workflow_nodes:,
workflow_name: workflow_version&.name,
workflow_call_caller: @options.workflow_call_caller,
)
@store =
ExecutionStore.new(
trigger_node_id: @trigger_node_id,
execution_context: @context,
execution_mode: @options.execution_mode,
options: @options,
)
@steps = []
reset_queue!
@sandbox = nil
@waiting_node = nil
@waiting_step = nil
@pin_data_by_node_name = resolved_pin_data
@workflow_call_stack = normalized_workflow_call_stack
end
def self.resume(execution, response_items, user: nil, webhook_context: nil)
build_resume_executor(execution, user:, webhook_context:).resume_from(
execution,
response_items,
)
end
def self.resume_with_error(execution, error, user: nil)
build_resume_executor(execution, user:).resume_from_error(execution, error)
end
def self.build_resume_executor(execution, user:, webhook_context: nil)
workflow_call_caller = WorkflowCallContinuation.caller_metadata_for(execution)
options =
ExecutionOptions.new(
user:,
execution_mode: execution.execution_mode.to_sym,
workflow_snapshot: build_resume_snapshot!(execution),
webhook_context:,
workflow_call_stack: WorkflowCallContinuation.workflow_call_stack_for(execution),
workflow_call_child: workflow_call_caller.present?,
workflow_call_caller:,
)
new(execution.workflow, execution.trigger_node_id, execution.trigger_data, options)
end
private_class_method :build_resume_executor
def self.build_resume_snapshot!(execution)
unless execution.running?
raise ArgumentError,
"Cannot resume execution #{execution.id} with status '#{execution.status}' " \
"(callers must claim via Execution.claim_for_resume first)"
end
snapshot =
if execution.execution_data&.workflow_data.present?
WorkflowSnapshot.new(execution.execution_data.workflow_data)
else
WorkflowSnapshot.from_workflow(execution.workflow, published: true)
end
unless snapshot.find_node(execution.trigger_node_id)
raise "Trigger node #{execution.trigger_node_id} not found in workflow #{execution.workflow.id}"
end
snapshot
end
private_class_method :build_resume_snapshot!
def run
return execution if @store.find_job_execution
execute_flow(:start_execution!) do
next prepare_step_flow! if step_mode?
trigger_node = @snapshot.find_node(@trigger_node_id)
raise "Trigger node #{@trigger_node_id} not found in workflow snapshot" if trigger_node.nil?
seed_trigger_node!(trigger_node)
end
end
def resume_from(execution, response_items)
execute_flow(:resume_execution!, execution) do
waiting_node_id = execution.waiting_node_id
waiting_node = @snapshot.find_node(waiting_node_id)
raise "Waiting node #{waiting_node_id} not found in workflow snapshot" if waiting_node.nil?
update_waiting_step(waiting_node, response_items)
@context.store_node_output(waiting_node, response_items)
@context.store_node_run(
waiting_node,
inputs: [],
outputs: [response_items],
input_sources: @context.consume_waiting_input_sources,
)
clear_waiting!
ItemContract.validate_items!(response_items, source: "resume:#{waiting_node.type}")
enqueue_downstream(waiting_node, 0, response_items)
end
end
def resume_from_error(execution, error)
execute_flow(:resume_execution!, execution) do
waiting_node_id = execution.waiting_node_id
waiting_node = @snapshot.find_node(waiting_node_id)
raise "Waiting node #{waiting_node_id} not found in workflow snapshot" if waiting_node.nil?
step = @steps.find { |entry| entry.node_id == waiting_node.id.to_s && entry.waiting? }
input_items = execution.waiting_step_input_items
input_groups = { 0 => input_items }
if (handled_outputs = continued_error_outputs(waiting_node, input_groups, error))
handled_outputs = enforce_node_output_budget(handled_outputs, nil)
all_items = handled_outputs.flatten(1)
step&.add_metadata("handled_error", Item.error_metadata(error))
step&.succeed!(output: all_items)
step&.apply_updates!("error" => nil)
@store.publish_progress(step:) if step
@context.store_node_output(waiting_node, all_items)
@context.store_node_run(
waiting_node,
inputs: [input_items],
outputs: handled_outputs,
input_sources: @context.consume_waiting_input_sources,
)
clear_waiting!
route_downstream(waiting_node, handled_outputs)
else
step&.fail!(error.message, error_html: error.is_a?(NodeError) ? error.html : nil)
@store.publish_progress(step:) if step
raise error
end
end
end
private
def step_mode?
@options.step_node_id.present?
end
def prepare_step_flow!
step_node = @snapshot.find_node(@options.step_node_id)
raise "Step node #{@options.step_node_id} not found in workflow snapshot" if step_node.nil?
@pin_data_by_node_name.delete(step_node.name.to_s)
@step_plan =
StepExecutionPlan.new(
snapshot: @snapshot,
target: step_node,
run_data: @store.existing_run_data,
)
if @step_plan.standalone_target?
@queue << [step_node, { 0 => [Item.wrap({})] }, {}]
return
end
@step_plan.cached_frontier.each { |node| emit_cached_node!(node) }
@step_plan.trigger_roots_to_run.each { |trigger_node| seed_trigger_node!(trigger_node) }
end
def emit_cached_node!(node)
output_groups = @step_plan.cached_outputs(node)
step = record_step(node, [])
step.succeed!(output: output_groups.flatten(1))
step.add_metadata("cached", true)
@store.publish_progress(step:)
route_downstream(node, output_groups)
end
def seed_trigger_node!(trigger_node)
trigger_items =
pinned_items_for(trigger_node) ||
(
if @trigger_data.is_a?(Array)
@trigger_data.map { |data| Item.wrap(data) }
else
[Item.wrap(@trigger_data)]
end
)
ItemContract.validate_items!(trigger_items, source: "trigger:#{trigger_node.type}")
record_step(trigger_node, [], output: trigger_items, status: Step::SUCCESS)
@context.store_node_output(trigger_node, trigger_items)
@context.store_node_run(trigger_node, inputs: [], outputs: [trigger_items])
enqueue_downstream(trigger_node, 0, trigger_items)
end
def normalized_workflow_call_stack
stack = Array(@options.workflow_call_stack).map(&:to_s)
workflow_id = @workflow.id.to_s
stack.last == workflow_id ? stack : stack + [workflow_id]
end
def execute_flow(setup_method, *setup_args)
@deadline = SiteSetting.discourse_workflows_run_timeout_minutes.minutes.from_now
begin
return execution if send(setup_method, *setup_args) == false
yield
process_queue
@store.finish!(steps: @steps)
rescue ExecutionPaused => e
begin_wait!(e.wait_request)
end
rescue Sidekiq::Shutdown => e
@store.fail!(error: e, steps: @steps) if @store.owns_execution?
raise
rescue => e
raise unless @store.owns_execution?
@store.fail!(error: e, steps: @steps)
ensure
commit_static_data!
@sandbox&.dispose
end
# Persists any static_data mutations made by action nodes during the run.
# Skipped when no node ever called `get_workflow_static_data` (the state
# tracks a dirty flag). Reloads under a row lock so we don't clobber
# concurrent executions that wrote to other nodes' flat `node:<name>` slots.
def commit_static_data!
state = @context&.static_data_state
return unless state&.dirty?
@workflow.with_lock do
@workflow.reload
existing_node_data = @workflow.node_static_data_entries
changed_node_data =
state
.node
.each_with_object({}) do |(node_name, node_data), result|
next if node_data.blank? && !existing_node_data.key?(node_name)
result[node_name] = node_data
end
merged_node = existing_node_data.merge(changed_node_data)
@workflow.commit_static_data!(global: state.global, node: merged_node)
end
rescue => e
Rails.logger.warn(
"discourse-workflows: failed to commit static_data for workflow " \
"#{@workflow.id}: #{e.class}: #{e.message}",
)
end
def process_queue
iterations = 0
@queue_index = 0
loop do
while @queue_index < @queue.length
iterations += 1
raise "Max iterations (#{MAX_ITERATIONS}) exceeded" if iterations > MAX_ITERATIONS
node, input_groups, input_sources = @queue[@queue_index]
@queue_index += 1
execute_node(node, input_groups, input_sources || {})
check_deadline!
end
break unless flush_partial_waiting_inputs
end
end
def execute_node(node, input_groups, input_sources = {})
input_items = primary_input_items(input_groups)
node_type_class =
DiscourseWorkflows::Registry.find_node_type(node.type, version: node.type_version)
return handle_unknown_node(node, input_items) unless node_type_class
unless node_type_class.available?
return handle_unavailable_node(node, node_type_class, input_items)
end
if (pinned = pinned_items_for(node, node_type_class:))
return(handle_pinned_node(node, input_items, input_groups, input_sources, pinned))
end
issues = NodeIssues.for_node(node, node_type_class)
return handle_node_issues(node, input_items, issues) if issues.any?
step = record_step(node, input_items)
capture_operation_metadata(step, node, node_type_class)
js_elapsed_before = sandbox_budget_tracker.current_elapsed_ms
node_context = @context.node_context_for(node)
resolver_ctx = build_resolver_context(node, input_groups, node_context, input_sources)
resolver = build_resolver(resolver_ctx)
runtime_state = NodeExecutionContext::RuntimeState.new
exec_ctx = nil
begin
exec_ctx =
build_node_execution_context(
node,
input_groups,
node_context,
node_type_class,
resolver,
resolver_ctx,
runtime_state,
)
result =
node_type_class.new(
parameters: node.parameters,
credentials: node.credentials,
webhook_id: node.webhook_id,
).execute(exec_ctx)
attach_form_completion(step, node, resolver)
wait_request = runtime_state.wait_request
if wait_request
if step_mode?
raise StandardError, I18n.t("discourse_workflows.errors.step_execution.wait_requested")
end
check_deadline!
step.mark_waiting!
@waiting_node = node
@waiting_step = step
@context.store_waiting_input_sources(
input_sources_for_storage(input_sources, input_groups),
)
raise ExecutionPaused, wait_request
end
step_log = collect_step_log(exec_ctx, resolver)
if step_log&.errors?
attach_step_log(step, step_log)
step.fail!(step_log.error_summary)
raise StandardError, step.error
end
ports = node_type_class.ports(node.parameters)
output_arrays = normalize_result(result, node, ports, input_groups)
output_arrays = apply_always_output_data(output_arrays, node, input_groups)
output_arrays = route_failed_items(output_arrays, node, ports)
output_arrays = enforce_node_output_budget(output_arrays, step_log)
attach_step_log(step, step_log)
all_items = output_arrays.flatten(1)
primary_empty = output_arrays.fetch(0) { [] }.empty?
if node_type_class.branching? && primary_empty
step.filter!(output: all_items)
else
step.succeed!(output: all_items)
end
@context.store_node_output(node, all_items)
@context.store_node_run(
node,
inputs: input_groups_for_storage(input_groups),
outputs: output_arrays,
input_sources: input_sources_for_storage(input_sources, input_groups),
metadata: runtime_state.metadata,
)
route_downstream(node, output_arrays)
rescue ExecutionPaused
raise
rescue Sidekiq::Shutdown => e
step.fail!(e.message)
raise
rescue => e
if (handled_outputs = continued_error_outputs(node, input_groups, e))
step_log = collect_step_log(exec_ctx, resolver)
handled_outputs = enforce_node_output_budget(handled_outputs, step_log)
attach_step_log(step, step_log)
step.add_metadata("handled_error", Item.error_metadata(e))
all_items = handled_outputs.flatten(1)
step.succeed!(output: all_items)
step.apply_updates!("error" => nil)
@context.store_node_output(node, all_items)
@context.store_node_run(
node,
inputs: input_groups_for_storage(input_groups),
outputs: handled_outputs,
input_sources: input_sources_for_storage(input_sources, input_groups),
)
route_downstream(node, handled_outputs)
return
end
unless step.metadata&.key?("logs")
attach_step_log(step, collect_step_log(exec_ctx, resolver))
end
step.fail!(e.message, error_html: e.is_a?(NodeError) ? e.html : nil) unless step.error?
raise
ensure
resolver&.dispose
js_elapsed = (sandbox_budget_tracker.current_elapsed_ms - js_elapsed_before).round(1)
step.add_metadata("js_elapsed_ms", js_elapsed) if js_elapsed > 0
runtime_state.step_metadata.each { |key, value| step.add_metadata(key, value) }
@store.publish_progress(step:)
end
end
def collect_step_log(exec_ctx, resolver)
return unless exec_ctx
log = exec_ctx.log || StepLog.new
errors = resolver&.expression_errors || []
errors.each { |err| log.error("#{err[:expression]}: #{err[:error]}") } if errors.present?
log
end
def attach_step_log(step, step_log)
return if step_log.nil? || step_log.empty?
step.add_metadata("logs", step_log.as_json)
end
def attach_form_completion(step, node, resolver)
return unless DiscourseWorkflows::FormCompletion.completion_node?(node)
step.add_metadata(
DiscourseWorkflows::FormCompletion::METADATA_KEY,
DiscourseWorkflows::FormCompletion.for_node(node, resolver:),
)
end
def handle_unknown_node(node, input_items)
Rails.logger.warn(
"discourse-workflows: unknown node type '#{node.type}' (version: #{node.type_version}) " \
"in workflow #{@context.workflow.id}, skipping node '#{node.name}'",
)
record_step(node, input_items, status: Step::ERROR, error: "Unknown node type '#{node.type}'")
end
def handle_unavailable_node(node, node_type_class, input_items)
reason = node_type_class.unavailable_reason_key || "discourse_workflows.node_unavailable"
Rails.logger.warn(
"discourse-workflows: node type '#{node.type}' is unavailable " \
"in workflow #{@context.workflow.id}, passing through node '#{node.name}'",
)
step = record_step(node, input_items)
step.skip!(output: input_items, reason:)
@store.publish_progress(step:)
@context.store_node_output(node, input_items)
@context.store_node_run(node, inputs: [input_items], outputs: [input_items])
enqueue_downstream(node, 0, input_items)
end
def handle_node_issues(node, input_items, issues)
reason = issues.map { |i| "#{i[:path]}: #{i[:message]}" }.join(", ")
step = record_step(node, input_items)
step.skip!(output: input_items, reason:)
@store.publish_progress(step:)
@context.store_node_output(node, input_items)
@context.store_node_run(node, inputs: [input_items], outputs: [input_items])
enqueue_downstream(node, 0, input_items)
end
def handle_pinned_node(node, input_items, input_groups, input_sources, pinned_items)
step = record_step(node, input_items)
step.succeed!(output: pinned_items)
step.add_metadata("pinned", true)
@store.publish_progress(step:)
@context.store_node_output(node, pinned_items)
@context.store_node_run(
node,
inputs: input_groups_for_storage(input_groups),
outputs: [pinned_items],
input_sources: input_sources_for_storage(input_sources, input_groups),
)
enqueue_downstream(node, 0, pinned_items)
end
# Returns pinned items for this node when:
# - the run is in :manual mode (pin data is never used in :normal mode),
# - the workflow has pin data for this node name,
# - the node type has a single primary output.
# Returns nil otherwise.
def pinned_items_for(node, node_type_class: nil)
return nil if @pin_data_by_node_name.empty?
raw_items = @pin_data_by_node_name[node.name.to_s]
return nil if raw_items.blank?
node_type_class ||=
DiscourseWorkflows::Registry.find_node_type(node.type, version: node.type_version)
return nil if node_type_class.nil?
return nil if Array(node_type_class.outputs).length > 1
Item.normalize_items(raw_items)
rescue Item::InconsistentItemFormatError
nil
end
def resolved_pin_data
return {} unless @options.execution_mode == :manual
data =
if @options.workflow_snapshot
@options.workflow_snapshot.pin_data
elsif @workflow.respond_to?(:pin_data)
@workflow.pin_data
end
return {} if data.blank?
data.transform_keys(&:to_s)
end
def build_node_execution_context(
node,
input_groups,
node_context,
node_type_class,
resolver,
resolver_ctx,
runtime_state
)
NodeExecutionContext.new(
input_items: primary_input_items(input_groups),
input_groups:,
parameters: node.parameters,
credentials: node.credentials,
node_settings: DiscourseWorkflows::NodeData.direct_settings(node),
webhook_id: node.webhook_id,
property_schema: node_type_class.property_schema,
credential_schema: node_type_class.credentials,
node_context:,
user: @options.user,
resolver:,
vars: preloaded_vars,
workflow_vars: preloaded_workflow_vars,
workflow: @workflow,
workflow_version: @options.workflow_version,
execution_id: @store.execution&.id,
resume_token: @context.resume_token,
node_id: node.id.to_s,
node_name: node.name.to_s,
node_identifier: node.type,
execution_mode: @options.execution_mode,
flow_context: @context.context,
resolver_context: resolver_ctx,
workflow_dependencies: preloaded_workflow_dependencies,
workflow_snapshot: @snapshot,
webhook_context: @options.webhook_context,
workflow_call_stack: @workflow_call_stack,
runtime_state:,
static_data_state: @context.static_data_state,
deadline: @deadline,
)
end
def check_deadline!
raise ExecutionTimedOut if @deadline.past?
end
def normalize_result(result, node, ports, input_groups)
source = "#{node.name} (#{node.type})"
ItemContract.validate_output_arrays!(result, source:, ports:)
apply_item_linking_defaults!(result, input_groups:)
ItemContract.validate_output_arrays!(result, source:, ports:)
result
end
def enforce_node_output_budget(output_arrays, step_log)
total_bytes = 0
truncated = false
bounded_arrays =
output_arrays.map do |items|
bounded_items = []
items.each do |item|
item_bytes = JSON.generate(item).bytesize
if total_bytes + item_bytes > MAX_NODE_OUTPUT_BYTES
truncated = true
break
end
total_bytes += item_bytes
bounded_items << item
end
bounded_items
end
if truncated
step_log&.warn(
"Node output truncated at #{bounded_arrays.flatten(1).length} items because serialized " \
"output exceeded #{MAX_NODE_OUTPUT_BYTES} bytes",
)
end
bounded_arrays
end
def apply_item_linking_defaults!(output_arrays, input_groups:)
input_lookup = input_item_lookup(input_groups)
sole_input_pair = sole_input_pair(input_groups)
primary_items = primary_input_items(input_groups)
one_output = output_arrays.length == 1
output_arrays.each do |items|
items.each_with_index do |item, index|
pair =
if input_lookup.key?(item.object_id)
input_lookup[item.object_id]
elsif Item.paired_item(item).present?
Item.paired_item(item)
elsif sole_input_pair
sole_input_pair
elsif one_output && index < primary_items.length && items.length == primary_items.length
pair_for(input: 0, item: index)
end
items[index] = Item.with_paired_item(item, pair) if pair.present?
end
end
end
def input_item_lookup(input_groups)
input_groups.each_with_object({}) do |(input_index, items), lookup|
items.each_with_index do |item, item_index|
lookup[item.object_id] = pair_for(input: input_index, item: item_index)
end
end
end
def sole_input_pair(input_groups)
return unless input_groups.sum { |_input_index, items| items.size } == 1
input_index = input_groups.find { |_input_index, items| items.one? }.first
pair_for(input: input_index, item: 0)
end
def pair_for(input:, item:, include_input: false)
pair = { "item" => item }
pair["input"] = input if include_input || input != 0
pair
end
def apply_always_output_data(output_arrays, node, input_groups)
return output_arrays unless always_output_data?(node)
return output_arrays if output_arrays.fetch(0) { [] }.any?
pairs =
input_groups.flat_map do |input_index, items|
items.each_index.map do |item_index|
pair_for(input: input_index, item: item_index, include_input: true)
end
end
synthetic = { "json" => {}, "pairedItem" => pairs }
output_arrays = output_arrays.dup
output_arrays[0] = [synthetic]
output_arrays
end
def always_output_data?(node)
ActiveModel::Type::Boolean.new.cast(
DiscourseWorkflows::NodeData.read(node, "alwaysOutputData"),
) == true
end
def continued_error_outputs(node, input_groups, error)
return if error.is_a?(ExecutionTimedOut)
case node_error_mode(node)
when "continueRegularOutput"
[primary_input_items(input_groups)]
when "continueErrorOutput"
[[], error_output_items(primary_input_items(input_groups), error)]
end
end
def route_failed_items(output_arrays, node, ports)
if output_arrays.none? { |items| items.any? { |item| failed_item?(item) } }
return output_arrays
end
if ports.one? && node_error_mode(node) == "continueErrorOutput"
failed, succeeded = output_arrays[0].partition { |item| failed_item?(item) }
return succeeded, failed.map { |item| item.except(Item::FAILED_KEY) }
end
output_arrays.map do |items|
items.map { |item| failed_item?(item) ? item.except(Item::FAILED_KEY) : item }
end
end
def failed_item?(item)
item.key?(Item::FAILED_KEY)
end
def node_error_mode(node)
on_error = DiscourseWorkflows::NodeData.read(node, "onError").presence
return on_error if NodeDataShape::CONTINUE_ON_ERROR_MODES.include?(on_error)
return "stopWorkflow" if on_error == "stopWorkflow"
return if on_error.present?
if ActiveModel::Type::Boolean.new.cast(
DiscourseWorkflows::NodeData.read(node, "continueOnFail"),
) == true
"continueRegularOutput"
end
end
def error_output_items(input_items, error)
return [Item.with_error({ "json" => {} }, error, paired_item: nil)] if input_items.empty?
input_items.map.with_index do |item, index|
Item.with_error(item, error, paired_item: pair_for(input: 0, item: index))
end
end
def route_downstream(node, output_arrays)
output_arrays.each_with_index { |items, index| enqueue_downstream(node, index, items) }
end
def enqueue_downstream(node, output_index, items)
return if step_mode? && node.id.to_s == @step_plan.target_id
@snapshot
.connections_from_output_index(node, output_index)
.each do |conn|
target = @snapshot.target_node(conn)
next if target.nil?
next if step_mode? && !@step_plan.runnable?(target)
enqueue_target(
target,
conn.target_input_index,
items,
source: {
"node_name" => node.name,
"output_index" => output_index,
},
)
end
end
def enqueue_target(target, target_input_index, items, source:)
target_input_index = target_input_index.to_i
requirements = input_wait_requirements(target)
inputs_to_wait_for = requirements[:inputs_to_wait_for]
if requirements[:minimum_input_count]
return enqueue_minimum_input_target(target, target_input_index, items, source, requirements)
end
if inputs_to_wait_for.length <= 1
return if items.empty?
@queue << [target, { 0 => items }, { 0 => source }]
return
end
inputs = (@waiting_inputs[target.id] ||= {})
sources = (@waiting_input_sources[target.id] ||= {})
@waiting_input_targets[target.id] = target
inputs[target_input_index] = items
sources[target_input_index] = source
return unless inputs_to_wait_for.all? { |index| inputs.key?(index) }
@waiting_inputs.delete(target.id)
@waiting_input_sources.delete(target.id)
@waiting_input_targets.delete(target.id)
@queue << [target, inputs, sources]
end
def enqueue_minimum_input_target(target, target_input_index, items, source, requirements)
inputs = (@waiting_inputs[target.id] ||= {})
sources = (@waiting_input_sources[target.id] ||= {})
@waiting_input_targets[target.id] = target
inputs[target_input_index] = items
sources[target_input_index] = source
connected_inputs = requirements[:connected_inputs]
if connected_inputs.length <= requirements[:minimum_input_count] ||
connected_inputs.all? { |index| inputs.key?(index) }
return enqueue_waiting_input_target(target)
end
return if available_input_count(inputs) >= requirements[:minimum_input_count]
@waiting_inputs.delete(target.id)
@waiting_input_sources.delete(target.id)
@waiting_input_targets.delete(target.id)
end
def enqueue_waiting_input_target(target)
inputs = @waiting_inputs.delete(target.id) || {}
sources = @waiting_input_sources.delete(target.id) || {}
@waiting_input_targets.delete(target.id)
return if available_input_count(inputs).zero?
@queue << [target, inputs, sources]
end
def flush_partial_waiting_inputs
queued_any = false
waiting_target_ids = @waiting_input_targets.keys
@waiting_input_targets.dup.each_value do |target|
requirements = input_wait_requirements(target)
next unless requirements[:minimum_input_count]
next if waiting_input_parent?(target, waiting_target_ids)
inputs = @waiting_inputs[target.id] || {}
next if available_input_count(inputs) < requirements[:minimum_input_count]
enqueue_waiting_input_target(target)
queued_any = true
end
queued_any
end
def available_input_count(inputs)
inputs.count { |_index, items| items.present? }
end
def waiting_input_parent?(target, waiting_target_ids)
@snapshot
.connections_to(target)
.any? { |conn| waiting_target_ids.include?(conn.source_node_id.to_s) }
end
def input_wait_requirements(target)
@input_wait_requirements[target.id] ||= begin
node_type_class =
DiscourseWorkflows::Registry.find_node_type(target.type, version: target.type_version)
input_ports =
node_type_class&.input_ports(target.parameters) ||
[{ key: "main", index: 0, required: true }]
explicit_required_inputs = node_type_class&.required_inputs(target.parameters)
required_inputs = input_ports.select { |port| port[:required] }.map { |port| port[:index] }
connected_inputs = @snapshot.connections_to(target).map(&:target_input_index).uniq
if explicit_required_inputs.is_a?(Integer)
{
connected_inputs:,
inputs_to_wait_for: [],
minimum_input_count: explicit_required_inputs,
}
elsif explicit_required_inputs.present?
{ connected_inputs:, inputs_to_wait_for: Array(explicit_required_inputs).map(&:to_i) }
else
{ connected_inputs:, inputs_to_wait_for: (required_inputs + connected_inputs).uniq }
end
end
end
def primary_input_items(input_groups)
input_groups.fetch(0) { input_groups.values.first || [] }
end
def input_groups_for_storage(input_groups)
max_index = input_groups.keys.max || 0
Array.new(max_index + 1) { |index| input_groups[index] || [] }
end
def input_sources_for_storage(input_sources, input_groups)
max_index = [input_groups.keys.max || 0, input_sources.keys.max || 0].max
Array.new(max_index + 1) { |index| input_sources[index] || {} }
end
def capture_operation_metadata(step, node, node_type_class)
properties = node_type_class.properties
return unless properties.is_a?(Hash) && properties.key?(:operation)
operation_value = node.parameters&.dig("operation") || node.parameters&.dig(:operation)
return if operation_value.blank?
step.add_metadata("operation", operation_value)
end
def record_step(node, input_items, output: [], status: Step::RUNNING, error: nil)
step = Step.build(node:, position: @steps.size, input: input_items, output:, status:, error:)
@steps << step
@store.publish_progress(step:)
step
end
def update_waiting_step(waiting_node, response_items)
step = @steps.find { |s| s.node_id == waiting_node.id.to_s && s.status == Step::WAITING }
if step.nil?
Rails.logger.warn(
"discourse-workflows: waiting step not found for node '#{waiting_node.id}' " \
"in workflow #{@context.workflow.id}",
)
return
end
step.apply_updates!(
"status" => Step::SUCCESS,
"output" => response_items,
"finished_at" => Time.current.iso8601,
)
@store.publish_progress(step:)
end
def begin_wait!(wait_request)
store_pending_wait_state!
if wait_request.workflow_call?
begin_workflow_call_wait!(wait_request)
else
begin_timed_wait!(wait_request.waiting_until, timeout_action: wait_request.timeout_action)
end
end
def store_pending_wait_state!
@context.store_pending_input_groups(
inputs: @waiting_inputs,
sources: @waiting_input_sources,
target_ids: @waiting_input_targets.keys,
)
@context.store_pending_queue(@queue.drop(@queue_index || 0))
end
def begin_timed_wait!(waiting_until, timeout_action: nil)
now = Time.current
ceiling = now + MAX_WAIT_DURATION_SECONDS
resolved = waiting_until.blank? ? ceiling : [waiting_until, ceiling].min
execution =
@store.pause_waiting_execution!(
node: @waiting_node,
waiting_until: resolved,
timeout_action:,
steps: @steps,
)
return @store.execution unless execution
Jobs.enqueue_in(
[resolved - now, 0].max,
Jobs::DiscourseWorkflows::ResumeWaitingExecution,
execution_id: @store.execution.id,
resume_token: execution.resume_token,
)
execution
end
def begin_workflow_call_wait!(wait_request)
execution = @store.pause_waiting_execution!(node: @waiting_node, steps: @steps)
return @store.execution unless execution
DiscourseWorkflows::WorkflowCallContinuation.begin_child_call!(
execution:,
node: @waiting_node,
request: wait_request,
)
execution
end
def start_execution!
started =
if @options.job_id.present?
DistributedMutex.synchronize("discourse_workflows_job_#{@options.job_id}") do
create_execution!
end
else
create_execution!
end
return false unless started
@snapshot = @store.workflow_snapshot
@steps = []
reset_queue!
end
def create_execution!
return false if @store.find_job_execution
unless @workflow.published? || @options.draft_execution || @options.workflow_version ||
@options.workflow_snapshot
@store.create_execution_with_status(:skipped)
return false
end
if (rate_limit_message = rate_limiter.exceeded_limit_message)
@store.create_rate_limited_execution(error: rate_limit_message)
return false
end
@store.start!
end
def resume_execution!(execution)
@store.resume!(execution)
@snapshot = @store.workflow_snapshot
@steps = restore_steps_from(execution)
reset_queue!
restore_pending_queue!
restore_pending_input_groups!
end
def reset_queue!
@queue = []
@queue_index = 0
@waiting_inputs = {}
@waiting_input_sources = {}
@waiting_input_targets = {}
@input_wait_requirements = {}
end
def clear_waiting!
@store.clear_waiting_execution!
@waiting_node = nil
@waiting_step = nil
end
def restore_steps_from(execution)
entries = execution.execution_data&.entries || {}
entries.values.flatten.map { |h| Step.from_h(h) }
end
def restore_pending_input_groups!
@context.consume_pending_input_groups.each do |target_id, payload|
target = @snapshot.find_node(target_id)
next if target.nil?
@waiting_input_targets[target.id] = target
@waiting_inputs[target.id] = indexed_values_from_payload(payload["inputs"], "items")
@waiting_input_sources[target.id] = indexed_values_from_payload(
payload["sources"],
"source",
)
end
end
def restore_pending_queue!
@queue =
@context.consume_pending_queue.filter_map do |payload|
node = @snapshot.find_node(payload["node_id"])
next if node.nil?
[
node,
indexed_values_from_payload(payload["inputs"], "items"),
indexed_values_from_payload(payload["sources"], "source"),
]
end
end
def indexed_values_from_payload(payload, value_key)
Array(payload).each_with_object({}) do |entry, values|
values[entry["index"].to_i] = entry[value_key]
end
end
def build_resolver_context(node, input_groups, node_context, input_sources)
input_items = primary_input_items(input_groups)
current_item = input_items.first || { "json" => {} }
base =
@context.resolver_context(
"__input_item" => current_item,
"__input_items" => input_items,
"__input_params" => node.parameters,
"__input_context" => DiscourseWorkflows::InputContext.from_node_context(node_context),
"__current_node_id" => node.id.to_s,
"__node_parameters_by_name" => node_parameters_by_name,
"$itemIndex" => 0,
)
base["__input_sources"] = input_sources_for_storage(input_sources, input_groups)
base.merge("$json" => current_item.fetch("json") { {} })
end
def node_parameters_by_name
@node_parameters_by_name ||=
@snapshot
.nodes
.each_with_object({}) do |node, by_name|
name = node.name.to_s
next if name.blank?
by_name[name] = by_name.key?(name) ? nil : node.parameters
end
.compact
end
def build_resolver(resolver_ctx)
ExpressionResolver.new(resolver_ctx, user: @options.user, sandbox: shared_sandbox)
end
def shared_sandbox
@sandbox ||=
DiscourseWorkflows::JsSandbox.new(
@context.resolver_context,
user: @options.user,
vars: preloaded_vars,
workflow_vars: preloaded_workflow_vars,
budget_tracker: sandbox_budget_tracker,
)
end
def preloaded_vars
@preloaded_vars ||=
DiscourseWorkflows::Variable.where(workflow_id: nil).pluck(:key, :value).to_h
end
# Mirrors the same nodes/connections source-selection above: whatever
# variables schema a workflow_snapshot was built with (a resume, a node
# preview, a step execution, a form test session) travels with it, the
# same way its nodes/connections do.
def preloaded_workflow_vars
@preloaded_workflow_vars ||=
DiscourseWorkflows::WorkflowVariableValueResolver.new(variables_schema).resolve
end
def variables_schema
if @options.workflow_snapshot
@options.workflow_snapshot.variables_schema
elsif @options.workflow_version
@options.workflow_version.variables_schema
elsif @options.draft_execution
@workflow.variables_schema
else
@workflow.active_version&.variables_schema || []
end
end
def preloaded_workflow_dependencies
@preloaded_workflow_dependencies ||=
@snapshot
.nodes
.each_with_object(Hash.new { |h, k| h[k] = Set.new }) do |node, dependencies|
node_dependencies(node).each do |type, key|
dependencies[node.id.to_s] << "#{type}:#{key}"
end
end
end
def node_dependencies(node)
parameters = NodeData.parameters(node)
credentials =
NodeData.split(parameters:, credentials: NodeData.credentials(node), node_type: node.type)[
"credentials"
]
dependencies = []
credentials.each_value do |credential|
dependencies << ["credential_id", credential["id"]] if credential["id"].present?
end
if parameters["data_table_id"].present?
dependencies << ["data_table_id", parameters["data_table_id"]]
end
if node.type == DiscourseWorkflows::Nodes::WorkflowCall::V1.identifier &&
parameters["workflow_id"].present?
dependencies << ["workflow_call", parameters["workflow_id"]]
end
dependencies
end
def sandbox_budget_tracker
@sandbox_budget_tracker ||= DiscourseWorkflows::SandboxBudget.new(@context.context)
end
def rate_limiter
@rate_limiter ||= ExecutionRateLimiter.new(@workflow)
end
end
end