A platform for community discussion. Free, open, simple.
FIX: Stop the data table node from silently writing blanks or nothing
Régis Hanol4cddb3e23db0
diff --git a/plugins/discourse-workflows/config/locales/server.en.yml b/plugins/discourse-workflows/config/locales/server.en.ymlindex 85e8fafea29..d3b14acbb90 100644--- a/plugins/discourse-workflows/config/locales/server.en.yml+++ b/plugins/discourse-workflows/config/locales/server.en.yml@@ -208,6 +208,9 @@ en:invalid_query: "Invalid query: %{errors}"invalid_row: "Invalid row: %{errors}"filter_required: "Filter must not be empty"+ no_columns_to_insert: "Cannot insert a row without any column values."+ empty_column_expression: "Column '%{column}' has an empty expression."+ invalid_columns: "Columns must be an object of column names to values, got '%{value}'."workflow_version:referenced_by_workflow: "Cannot delete a workflow version that is still referenced as the current or active version."workflow_call:diff --git a/plugins/discourse-workflows/lib/discourse_workflows/data_tables/columns_resolver.rb b/plugins/discourse-workflows/lib/discourse_workflows/data_tables/columns_resolver.rbindex dd06a971a29..d577017fddd 100644--- a/plugins/discourse-workflows/lib/discourse_workflows/data_tables/columns_resolver.rb+++ b/plugins/discourse-workflows/lib/discourse_workflows/data_tables/columns_resolver.rb@@ -3,6 +3,22 @@module DiscourseWorkflowsmodule DataTablesclass ColumnsResolver+ # Returns column name => value pairs, or nil when the value isn't keyed by+ # column (e.g. an unresolved whole-object expression).+ def self.column_pairs(fields)+ case fields+ when nil+ {}+ when Hash+ fields.stringify_keys+ when Array+ fields = fields.map { |field| field.try(:with_indifferent_access) }+ return unless fields.all? { |field| field&.key?(:columnName) }++ fields.to_h { |field| [field[:columnName], field[:value]] }+ end+ end+def initialize(data_table)@column_names =data_table@@ -31,17 +47,15 @@ module DiscourseWorkflowsprivatedef normalize_fields(fields)- case fields- when Hash- fields.stringify_keys- when Array- fields.to_h do |field|- field = field.with_indifferent_access- [field[:columnName], field[:value]]- end- else- {}- end+ self.class.column_pairs(fields) || raise_invalid_columns!(fields)+ end++ def raise_invalid_columns!(fields)+ raise ArgumentError,+ I18n.t(+ "discourse_workflows.errors.data_table.invalid_columns",+ value: fields.to_s.truncate(100),+ )endendenddiff --git a/plugins/discourse-workflows/lib/discourse_workflows/data_tables/node_proxy.rb b/plugins/discourse-workflows/lib/discourse_workflows/data_tables/node_proxy.rbindex c25173d426a..a75a34c90e4 100644--- a/plugins/discourse-workflows/lib/discourse_workflows/data_tables/node_proxy.rb+++ b/plugins/discourse-workflows/lib/discourse_workflows/data_tables/node_proxy.rb@@ -53,7 +53,10 @@ module DiscourseWorkflowsenddef insert_rows(rows, return_type)- inserted_rows = rows.map { |row| @facade.insert(build_row_input(row)) }+ # Validate every row up front so an invalid row fails before any row is inserted.+ row_inputs =+ rows.map { |row| build_row_input(row).tap { |input| ensure_insertable!(input) } }+ inserted_rows = row_inputs.map { |row_input| @facade.insert(row_input) }case return_type.to_swhen "all"@@ -70,28 +73,27 @@ module DiscourseWorkflowsdef update_rows(options)options = normalize_options(options)query = build_query(options, optional_filter: false)- row_input = build_row_input(options["data"] || {})+ row_input = build_row_input(options["data"])rows = @facade.query(query)[:rows]return dry_run_update_rows(rows, row_input) if options["dry_run"]- rows.filter_map { |row| @facade.update_row(row_id: row.fetch("id"), row_input:) }+ update_matching_rows(rows, row_input)enddef upsert_row(options)options = normalize_options(options)query = build_query(options, optional_filter: false)- row_input = build_row_input(options["data"] || {})+ row_input = build_row_input(options["data"])matching_rows = @facade.query(query)[:rows]if matching_rows.present?return dry_run_update_rows(matching_rows, row_input) if options["dry_run"]- matching_rows.filter_map { |row| @facade.update_row(row_id: row.fetch("id"), row_input:) }- elsif options["dry_run"]- [dry_run_insert_row(row_input)]+ update_matching_rows(matching_rows, row_input)else- [@facade.insert(row_input)]+ ensure_insertable!(row_input)+ [options["dry_run"] ? dry_run_insert_row(row_input) : @facade.insert(row_input)]endend@@ -172,11 +174,24 @@ module DiscourseWorkflowsdef build_row_input(data)row_input =- @facade.build_row_input(data: @columns_resolver.resolve(data || {}), fill_missing: false)+ @facade.build_row_input(data: @columns_resolver.resolve(data), fill_missing: false)raise_invalid(:row, row_input) if row_input.invalid?row_inputend+ def ensure_insertable!(row_input)+ return if row_input.columns.present?++ raise ArgumentError, I18n.t("discourse_workflows.errors.data_table.no_columns_to_insert")+ end++ # Nothing to write still counts as a match, so the rows flow on unchanged.+ def update_matching_rows(rows, row_input)+ return rows if row_input.columns.empty?++ rows.filter_map { |row| @facade.update_row(row_id: row.fetch("id"), row_input:) }+ end+def dry_run_update_rows(rows, row_input)rows.flat_map do |row|[diff --git a/plugins/discourse-workflows/lib/discourse_workflows/expression_resolver.rb b/plugins/discourse-workflows/lib/discourse_workflows/expression_resolver.rbindex f1fb578a119..18cfe07d071 100644--- a/plugins/discourse-workflows/lib/discourse_workflows/expression_resolver.rb+++ b/plugins/discourse-workflows/lib/discourse_workflows/expression_resolver.rb@@ -2,6 +2,13 @@module DiscourseWorkflowsclass ExpressionResolver+ EMPTY_EXPRESSION = /\A=\s*(\{\{\s*\}\})?\s*\z/++ # Mirrored by isEmptyExpression in property-engine.js.+ def self.empty_expression?(value)+ value.is_a?(String) && EMPTY_EXPRESSION.match?(value)+ end+def self.find_matching_close(template, start)depth = 1cursor = startdiff --git a/plugins/discourse-workflows/lib/discourse_workflows/nodes/data_table/v1.rb b/plugins/discourse-workflows/lib/discourse_workflows/nodes/data_table/v1.rbindex 3397e0b1b68..2909708e7a0 100644--- a/plugins/discourse-workflows/lib/discourse_workflows/nodes/data_table/v1.rb+++ b/plugins/discourse-workflows/lib/discourse_workflows/nodes/data_table/v1.rb@@ -95,6 +95,8 @@ module DiscourseWorkflows},},},+ # A missing key leaves the column untouched (NULL/default on insert);+ # a present key is always written, and an empty value writes NULL.columns: {type: :object,required: false,@@ -105,6 +107,7 @@ module DiscourseWorkflowsoperation: %w[insert update upsert],},hide: {+ operation: ["insert"],mapping_mode: ["auto"],},},@@ -191,6 +194,7 @@ module DiscourseWorkflowsenddef execute(exec_ctx)+ empty_column = empty_column_expression(exec_ctx)items =exec_ctx.input_items.flat_map.with_index do |item, item_index|config = {@@ -208,6 +212,14 @@ module DiscourseWorkflows"sort_direction" =>exec_ctx.get_node_parameter("sort_direction", item_index, default: "asc"),}+ if empty_column && manual_columns?(config)+ raise_node_error!(+ I18n.t(+ "discourse_workflows.errors.data_table.empty_column_expression",+ column: empty_column,+ ),+ )+ endbeginpair_output_items(execute_with_config(config, item, exec_ctx), exec_ctx, item)rescue ArgumentError => e@@ -220,6 +232,28 @@ module DiscourseWorkflowsprivate+ # An empty expression resolves to "", which would silently overwrite the column.+ # Raw values don't depend on the item, so this is scanned once per run.+ def empty_column_expression(exec_ctx)+ raw_columns = exec_ctx.get_node_parameter("columns", options: { raw_expressions: true })++ DiscourseWorkflows::DataTables::ColumnsResolver+ .column_pairs(raw_columns)+ &.find { |_, value| DiscourseWorkflows::ExpressionResolver.empty_expression?(value) }+ &.first+ end++ def manual_columns?(config)+ DiscourseWorkflows::Schema.visible?(+ self.class.properties[:columns][:display_options],+ config,+ )+ end++ def auto_mapped_insert?(config)+ config["operation"] == "insert" && config["mapping_mode"] == "auto"+ end+def pair_output_items(output_items, exec_ctx, item)paired_item = exec_ctx.paired_item_for(item)@@ -230,14 +264,14 @@ module DiscourseWorkflowsnode_proxy = exec_ctx.helpers.get_data_table_proxy(config.fetch("data_table_id"))operation_name = config.fetch("operation", "insert")- if operation_name == "insert" && config["mapping_mode"] == "auto"+ if auto_mapped_insert?(config)json = item["json"] || {}config = config.merge("columns" => json.slice(*column_names(node_proxy)))endcase operation_namewhen "insert"- wrap(node_proxy.insert_rows([config["columns"] || {}], "all"))+ wrap(node_proxy.insert_rows([config["columns"]], "all"))when "get"wrap(node_proxy.get_many_rows_and_count(row_query_options(config))[:data])when "update"@@ -281,7 +315,7 @@ module DiscourseWorkflowsenddef row_mutation_options(config)- { filter: normalized_filter(config), data: config["columns"] || {} }+ { filter: normalized_filter(config), data: config["columns"] }enddef row_delete_options(config)diff --git a/plugins/discourse-workflows/spec/lib/discourse_workflows/data_tables/columns_resolver_spec.rb b/plugins/discourse-workflows/spec/lib/discourse_workflows/data_tables/columns_resolver_spec.rbindex 8dcc046b213..4372ea50d5e 100644--- a/plugins/discourse-workflows/spec/lib/discourse_workflows/data_tables/columns_resolver_spec.rb+++ b/plugins/discourse-workflows/spec/lib/discourse_workflows/data_tables/columns_resolver_spec.rb@@ -36,15 +36,27 @@ RSpec.describe DiscourseWorkflows::DataTables::ColumnsResolver doexpect(resolver.resolve(nil)).to eq({})end- it "raises for unknown column names" do- expect { resolver.resolve("unknown" => "x") }.to raise_error(- ArgumentError,- /Unknown column name/,- )+ it "raises when the input is not a hash or an array of column fields" do+ ["oops", 42, [1, 2], [{ "value" => "x" }]].each do |input|+ expect { resolver.resolve(input) }.to raise_error(+ ArgumentError,+ /Columns must be an object of column names to values/,+ )+ end+ end++ it "raises for unknown and system column names" do+ %w[unknown id].each do |name|+ expect { resolver.resolve(name => 1) }.to raise_error(ArgumentError, /Unknown column name/)+ endend+ end- it "does not resolve system columns" do- expect { resolver.resolve("id" => 1) }.to raise_error(ArgumentError, /Unknown column name/)+ describe ".column_pairs" do+ it "returns nil when the columns are not keyed by column name" do+ ["={{ $json }}", ["oops", { "columnName" => "email" }]].each do |input|+ expect(described_class.column_pairs(input)).to be_nil+ endendend@@ -52,12 +64,5 @@ RSpec.describe DiscourseWorkflows::DataTables::ColumnsResolver doit "returns nil for known columns" doexpect(resolver.validate_column!("email")).to be_nilend-- it "raises for unknown column names" do- expect { resolver.validate_column!("unknown") }.to raise_error(- ArgumentError,- /Unknown column name/,- )- endendenddiff --git a/plugins/discourse-workflows/spec/lib/discourse_workflows/data_tables/node_proxy_spec.rb b/plugins/discourse-workflows/spec/lib/discourse_workflows/data_tables/node_proxy_spec.rbindex a6fed093100..c212857b8ad 100644--- a/plugins/discourse-workflows/spec/lib/discourse_workflows/data_tables/node_proxy_spec.rb+++ b/plugins/discourse-workflows/spec/lib/discourse_workflows/data_tables/node_proxy_spec.rb@@ -50,6 +50,14 @@ RSpec.describe DiscourseWorkflows::DataTables::NodeProxy do)end+ it "raises instead of inserting a row without any columns" do+ expect { proxy.insert_rows([{ "email" => "[email protected]" }, {}], "all") }.to raise_error(+ ArgumentError,+ /Cannot insert a row without any column/,+ )+ expect(count_data_table_rows(data_table)).to eq(0)+ end+it "gets many rows and count" doinsert_data_table_row(data_table, "email" => "[email protected]", "score" => 1)insert_data_table_row(data_table, "email" => "[email protected]", "score" => 2)@@ -93,6 +101,40 @@ RSpec.describe DiscourseWorkflows::DataTables::NodeProxy do)end+ context "when there are no columns to write" do+ fab!(:row) { insert_data_table_row(data_table, "email" => "[email protected]", "score" => 1) }++ it "returns the matched rows unchanged on update and upsert" do+ %i[update_rows upsert_row].each do |method|+ expect(proxy.public_send(method, filter: email_filter("[email protected]"), data: {})).to eq(+ [row],+ ),+ method.to_s+ end+ expect(find_data_table_row(data_table, row["id"])).to eq(row)+ end++ it "returns identical before and after rows on a dry run" do+ expected = [row.merge("dryRunState" => "before"), row.merge("dryRunState" => "after")]++ %i[update_rows upsert_row].each do |method|+ expect(+ proxy.public_send(method, filter: email_filter("[email protected]"), data: {}, dry_run: true),+ ).to eq(expected),+ method.to_s+ end+ end++ it "raises on an upsert that matches nothing, including a dry run" do+ [false, true].each do |dry_run|+ expect {+ proxy.upsert_row(filter: email_filter("[email protected]"), data: {}, dry_run:)+ }.to raise_error(ArgumentError, /Cannot insert a row without any column/)+ end+ expect(count_data_table_rows(data_table)).to eq(1)+ end+ end+it "deletes rows and returns deleted rows" doinsert_data_table_row(data_table, "email" => "[email protected]", "score" => 1)insert_data_table_row(data_table, "email" => "[email protected]", "score" => 2)diff --git a/plugins/discourse-workflows/spec/lib/discourse_workflows/expression_resolver_spec.rb b/plugins/discourse-workflows/spec/lib/discourse_workflows/expression_resolver_spec.rbindex dbaad77668f..dc51642dd38 100644--- a/plugins/discourse-workflows/spec/lib/discourse_workflows/expression_resolver_spec.rb+++ b/plugins/discourse-workflows/spec/lib/discourse_workflows/expression_resolver_spec.rb@@ -642,6 +642,21 @@ RSpec.describe DiscourseWorkflows::ExpressionResolver doendend+ describe ".empty_expression?" do+ it "is true only for an expression with nothing to evaluate" do+ {+ "=" => true,+ "= " => true,+ "={{ }}" => true,+ "= {{ }} " => true,+ "={{ x }}" => false,+ "={{ }}{{ x }}" => false,+ "" => false,+ nil => false,+ }.each { |value, expected| expect(described_class.empty_expression?(value)).to eq(expected) }+ end+ end+describe ".resolve" dofab!(:user)diff --git a/plugins/discourse-workflows/spec/lib/discourse_workflows/nodes/data_table_spec.rb b/plugins/discourse-workflows/spec/lib/discourse_workflows/nodes/data_table_spec.rbindex 65d4d098c42..d67b2a4b7a2 100644--- a/plugins/discourse-workflows/spec/lib/discourse_workflows/nodes/data_table_spec.rb+++ b/plugins/discourse-workflows/spec/lib/discourse_workflows/nodes/data_table_spec.rb@@ -67,6 +67,24 @@ RSpec.describe DiscourseWorkflows::Nodes::DataTable::V1 doendend+ describe "columns property" do+ let(:display_options) { described_class.properties[:columns][:display_options] }++ it "is hidden only for insert in auto mapping mode" do+ {+ %w[insert auto] => false,+ %w[insert manual] => true,+ %w[update auto] => true,+ %w[upsert auto] => true,+ }.each do |(operation, mapping_mode), visible|+ expect(+ DiscourseWorkflows::Schema.visible?(display_options, operation:, mapping_mode:),+ ).to eq(visible),+ "#{operation}/#{mapping_mode}"+ end+ end+ end+describe "error handling" doit "raises when data table does not exist" doexpect {@@ -80,18 +98,6 @@ RSpec.describe DiscourseWorkflows::Nodes::DataTable::V1 do}.to raise_error(ActiveRecord::RecordNotFound)end- it "raises for unknown column names" do- expect {- execute_data_table(- "operation" => "insert",- "data_table_id" => data_table.id.to_s,- "columns" => {- "nonexistent" => "test",- },- )- }.to raise_error(DiscourseWorkflows::NodeError, /Unknown column name/)- end-it "raises invalid row errors with the node error prefix" doexpect {execute_data_table(@@ -119,27 +125,104 @@ RSpec.describe DiscourseWorkflows::Nodes::DataTable::V1 doexecute_data_table("operation" => "truncate", "data_table_id" => data_table.id.to_s)}.to raise_error(DiscourseWorkflows::NodeError, /Unknown operation/)end- end- describe "insert operation" doit "raises when the storage limit is exceeded" do+ insert_data_table_row(data_table, "email" => "[email protected]", "score" => 1)allow(DiscourseWorkflows::DataTables::Facade).to receive(:within_storage_limit?).and_return(false,)- expect {- execute_data_table(- "operation" => "insert",+ # upsert filters on a missing row so it takes the insert path+ {+ "insert" => "[email protected]",+ "update" => "[email protected]",+ "upsert" => "[email protected]",+ }.each do |operation, email|+ expect {+ execute_data_table(+ "operation" => operation,+ "data_table_id" => data_table.id.to_s,+ "filter" => [filter_condition("email", "equals", value: email)],+ "columns" => {+ "score" => "99",+ },+ )+ }.to raise_error(DiscourseWorkflows::NodeError, "Data table storage limit exceeded"),+ operation+ end+ end+ end++ describe "empty column expressions" do+ fab!(:row) { insert_data_table_row(data_table, "email" => "[email protected]", "score" => 1) }++ def execute_with_columns(operation, columns, mapping_mode: "manual", json: {})+ execute_data_table_with_item(+ {+ "operation" => operation,"data_table_id" => data_table.id.to_s,- "columns" => {- "email" => "[email protected]",- "score" => "42",+ "mapping_mode" => mapping_mode,+ "filter" => [filter_condition("email", "equals", value: "[email protected]")],+ "columns" => columns,+ },+ { "json" => json },+ )+ end++ it "raises instead of writing an empty string" do+ [+ %w[insert manual =],+ ["update", "manual", "={{ }}"],+ %w[update auto =],+ ["upsert", "manual", "={{ }}"],+ ].each do |operation, mapping_mode, value|+ expect { execute_with_columns(operation, { "email" => value }, mapping_mode:) }.to(+ raise_error(DiscourseWorkflows::NodeError, /Column 'email' has an empty expression/),+ "#{operation}/#{mapping_mode} with #{value.inspect}",+ )+ end++ expect {+ execute_with_columns("update", [{ "columnName" => "email", "value" => "=" }])+ }.to raise_error(DiscourseWorkflows::NodeError, /Column 'email' has an empty expression/)++ expect(find_data_table_row(data_table, row["id"])).to eq(row)+ expect(count_data_table_rows(data_table)).to eq(1)+ end++ it "ignores empty expressions when columns are not written" do+ expect(execute_with_columns("get", { "email" => "=" }).size).to eq(1)++ items =+ execute_with_columns(+ "insert",+ { "email" => "=" },+ mapping_mode: "auto",+ json: {+ "email" => "[email protected]",},)- }.to raise_error(DiscourseWorkflows::NodeError, "Data table storage limit exceeded")+ expect(items[0]["json"]).to include("email" => "[email protected]")end+ end+ describe "insert operation" docontext "when mapping_mode is auto" do+ it "raises when the item has no matching keys" do+ expect {+ execute_data_table_with_item(+ {+ "operation" => "insert",+ "data_table_id" => data_table.id.to_s,+ "mapping_mode" => "auto",+ },+ { "json" => { "ignored" => "value" } },+ )+ }.to raise_error(DiscourseWorkflows::NodeError, /Cannot insert a row without any column/)++ expect(count_data_table_rows(data_table)).to eq(0)+ end+it "picks matching keys from the incoming item JSON" doitems =execute_data_table_with_item(@@ -323,21 +406,54 @@ RSpec.describe DiscourseWorkflows::Nodes::DataTable::V1 doexpect(count_data_table_rows(data_table)).to eq(2)end- it "raises when the storage limit is exceeded" do- allow(DiscourseWorkflows::DataTables::Facade).to receive(:within_storage_limit?).and_return(- false,- )-- expect {+ describe "column values" do+ def update_columns(columns)execute_data_table("operation" => "update","data_table_id" => data_table.id.to_s,- "filter" => [filter_condition("email", "equals", value: "[email protected]")],- "columns" => {- "score" => "99",- },+ "filter" => [filter_condition("score", "equals", value: 1, type: "number")],+ "columns" => columns,)- }.to raise_error(DiscourseWorkflows::NodeError, "Data table storage limit exceeded")++ find_data_table_row(data_table, row["id"])+ end++ it "writes present keys and leaves absent ones untouched" do+ expect(update_columns("email" => "[email protected]")).to include(+ "email" => "[email protected]",+ "score" => 1,+ )++ [+ [{ "email" => nil }, nil],+ [{ "email" => "" }, ""],+ [{ "email" => "={{ null }}" }, nil],+ [{ "email" => '={{ "" }}' }, ""],+ ].each do |columns, expected|+ update_columns("email" => "[email protected]")++ expect(update_columns(columns)).to include("email" => expected, "score" => 1),+ columns.inspect+ end+ end++ it "writes columns from an expression returning an object" do+ expect(update_columns("={{ { score: 9, email: '[email protected]' } }}")).to include(+ "email" => "[email protected]",+ "score" => 9,+ )+ end++ it "raises when the columns are not an object" do+ expect { update_columns("={{ 'oops' }}") }.to raise_error(+ DiscourseWorkflows::NodeError,+ /Columns must be an object of column names to values, got 'oops'/,+ )++ expect {+ update_columns(["oops", { "columnName" => "email", "value" => "=" }])+ }.to raise_error(DiscourseWorkflows::NodeError, /Columns must be an object/)+ endendend@@ -532,23 +648,5 @@ RSpec.describe DiscourseWorkflows::Nodes::DataTable::V1 do)expect(count_data_table_rows(data_table)).to eq(1)end-- it "raises when the storage limit is exceeded" do- allow(DiscourseWorkflows::DataTables::Facade).to receive(:within_storage_limit?).and_return(- false,- )-- expect {- execute_data_table(- "operation" => "upsert",- "data_table_id" => data_table.id.to_s,- "filter" => [filter_condition("email", "equals", value: "[email protected]")],- "columns" => {- "email" => "[email protected]",- "score" => "50",- },- )- }.to raise_error(DiscourseWorkflows::NodeError, "Data table storage limit exceeded")- endendend