Skip to content

A platform for community discussion. Free, open, simple.

FIX: Stop the data table node from silently writing blanks or nothing

Régis Hanol4cddb3e23db0
diff --git a/plugins/discourse-workflows/config/locales/server.en.yml b/plugins/discourse-workflows/config/locales/server.en.yml
index 85e8fafea29..d3b14acbb90 100644
--- a/plugins/discourse-workflows/config/locales/server.en.yml
+++ b/plugins/discourse-workflows/config/locales/server.en.yml
@@ -208,6 +208,9 @@ en:
invalid_query: "Invalid query: %{errors}"
invalid_row: "Invalid row: %{errors}"
filter_required: "Filter must not be empty"
+ no_columns_to_insert: "Cannot insert a row without any column values."
+ empty_column_expression: "Column '%{column}' has an empty expression."
+ invalid_columns: "Columns must be an object of column names to values, got '%{value}'."
workflow_version:
referenced_by_workflow: "Cannot delete a workflow version that is still referenced as the current or active version."
workflow_call:
diff --git a/plugins/discourse-workflows/lib/discourse_workflows/data_tables/columns_resolver.rb b/plugins/discourse-workflows/lib/discourse_workflows/data_tables/columns_resolver.rb
index dd06a971a29..d577017fddd 100644
--- a/plugins/discourse-workflows/lib/discourse_workflows/data_tables/columns_resolver.rb
+++ b/plugins/discourse-workflows/lib/discourse_workflows/data_tables/columns_resolver.rb
@@ -3,6 +3,22 @@
module DiscourseWorkflows
module DataTables
class ColumnsResolver
+ # Returns column name => value pairs, or nil when the value isn't keyed by
+ # column (e.g. an unresolved whole-object expression).
+ def self.column_pairs(fields)
+ case fields
+ when nil
+ {}
+ when Hash
+ fields.stringify_keys
+ when Array
+ fields = fields.map { |field| field.try(:with_indifferent_access) }
+ return unless fields.all? { |field| field&.key?(:columnName) }
+
+ fields.to_h { |field| [field[:columnName], field[:value]] }
+ end
+ end
+
def initialize(data_table)
@column_names =
data_table
@@ -31,17 +47,15 @@ module DiscourseWorkflows
private
def normalize_fields(fields)
- case fields
- when Hash
- fields.stringify_keys
- when Array
- fields.to_h do |field|
- field = field.with_indifferent_access
- [field[:columnName], field[:value]]
- end
- else
- {}
- end
+ self.class.column_pairs(fields) || raise_invalid_columns!(fields)
+ end
+
+ def raise_invalid_columns!(fields)
+ raise ArgumentError,
+ I18n.t(
+ "discourse_workflows.errors.data_table.invalid_columns",
+ value: fields.to_s.truncate(100),
+ )
end
end
end
diff --git a/plugins/discourse-workflows/lib/discourse_workflows/data_tables/node_proxy.rb b/plugins/discourse-workflows/lib/discourse_workflows/data_tables/node_proxy.rb
index c25173d426a..a75a34c90e4 100644
--- a/plugins/discourse-workflows/lib/discourse_workflows/data_tables/node_proxy.rb
+++ b/plugins/discourse-workflows/lib/discourse_workflows/data_tables/node_proxy.rb
@@ -53,7 +53,10 @@ module DiscourseWorkflows
end
def insert_rows(rows, return_type)
- inserted_rows = rows.map { |row| @facade.insert(build_row_input(row)) }
+ # Validate every row up front so an invalid row fails before any row is inserted.
+ row_inputs =
+ rows.map { |row| build_row_input(row).tap { |input| ensure_insertable!(input) } }
+ inserted_rows = row_inputs.map { |row_input| @facade.insert(row_input) }
case return_type.to_s
when "all"
@@ -70,28 +73,27 @@ module DiscourseWorkflows
def update_rows(options)
options = normalize_options(options)
query = build_query(options, optional_filter: false)
- row_input = build_row_input(options["data"] || {})
+ row_input = build_row_input(options["data"])
rows = @facade.query(query)[:rows]
return dry_run_update_rows(rows, row_input) if options["dry_run"]
- rows.filter_map { |row| @facade.update_row(row_id: row.fetch("id"), row_input:) }
+ update_matching_rows(rows, row_input)
end
def upsert_row(options)
options = normalize_options(options)
query = build_query(options, optional_filter: false)
- row_input = build_row_input(options["data"] || {})
+ row_input = build_row_input(options["data"])
matching_rows = @facade.query(query)[:rows]
if matching_rows.present?
return dry_run_update_rows(matching_rows, row_input) if options["dry_run"]
- matching_rows.filter_map { |row| @facade.update_row(row_id: row.fetch("id"), row_input:) }
- elsif options["dry_run"]
- [dry_run_insert_row(row_input)]
+ update_matching_rows(matching_rows, row_input)
else
- [@facade.insert(row_input)]
+ ensure_insertable!(row_input)
+ [options["dry_run"] ? dry_run_insert_row(row_input) : @facade.insert(row_input)]
end
end
@@ -172,11 +174,24 @@ module DiscourseWorkflows
def build_row_input(data)
row_input =
- @facade.build_row_input(data: @columns_resolver.resolve(data || {}), fill_missing: false)
+ @facade.build_row_input(data: @columns_resolver.resolve(data), fill_missing: false)
raise_invalid(:row, row_input) if row_input.invalid?
row_input
end
+ def ensure_insertable!(row_input)
+ return if row_input.columns.present?
+
+ raise ArgumentError, I18n.t("discourse_workflows.errors.data_table.no_columns_to_insert")
+ end
+
+ # Nothing to write still counts as a match, so the rows flow on unchanged.
+ def update_matching_rows(rows, row_input)
+ return rows if row_input.columns.empty?
+
+ rows.filter_map { |row| @facade.update_row(row_id: row.fetch("id"), row_input:) }
+ end
+
def dry_run_update_rows(rows, row_input)
rows.flat_map do |row|
[
diff --git a/plugins/discourse-workflows/lib/discourse_workflows/expression_resolver.rb b/plugins/discourse-workflows/lib/discourse_workflows/expression_resolver.rb
index f1fb578a119..18cfe07d071 100644
--- a/plugins/discourse-workflows/lib/discourse_workflows/expression_resolver.rb
+++ b/plugins/discourse-workflows/lib/discourse_workflows/expression_resolver.rb
@@ -2,6 +2,13 @@
module DiscourseWorkflows
class ExpressionResolver
+ EMPTY_EXPRESSION = /\A=\s*(\{\{\s*\}\})?\s*\z/
+
+ # Mirrored by isEmptyExpression in property-engine.js.
+ def self.empty_expression?(value)
+ value.is_a?(String) && EMPTY_EXPRESSION.match?(value)
+ end
+
def self.find_matching_close(template, start)
depth = 1
cursor = start
diff --git a/plugins/discourse-workflows/lib/discourse_workflows/nodes/data_table/v1.rb b/plugins/discourse-workflows/lib/discourse_workflows/nodes/data_table/v1.rb
index 3397e0b1b68..2909708e7a0 100644
--- a/plugins/discourse-workflows/lib/discourse_workflows/nodes/data_table/v1.rb
+++ b/plugins/discourse-workflows/lib/discourse_workflows/nodes/data_table/v1.rb
@@ -95,6 +95,8 @@ module DiscourseWorkflows
},
},
},
+ # A missing key leaves the column untouched (NULL/default on insert);
+ # a present key is always written, and an empty value writes NULL.
columns: {
type: :object,
required: false,
@@ -105,6 +107,7 @@ module DiscourseWorkflows
operation: %w[insert update upsert],
},
hide: {
+ operation: ["insert"],
mapping_mode: ["auto"],
},
},
@@ -191,6 +194,7 @@ module DiscourseWorkflows
end
def execute(exec_ctx)
+ empty_column = empty_column_expression(exec_ctx)
items =
exec_ctx.input_items.flat_map.with_index do |item, item_index|
config = {
@@ -208,6 +212,14 @@ module DiscourseWorkflows
"sort_direction" =>
exec_ctx.get_node_parameter("sort_direction", item_index, default: "asc"),
}
+ if empty_column && manual_columns?(config)
+ raise_node_error!(
+ I18n.t(
+ "discourse_workflows.errors.data_table.empty_column_expression",
+ column: empty_column,
+ ),
+ )
+ end
begin
pair_output_items(execute_with_config(config, item, exec_ctx), exec_ctx, item)
rescue ArgumentError => e
@@ -220,6 +232,28 @@ module DiscourseWorkflows
private
+ # An empty expression resolves to "", which would silently overwrite the column.
+ # Raw values don't depend on the item, so this is scanned once per run.
+ def empty_column_expression(exec_ctx)
+ raw_columns = exec_ctx.get_node_parameter("columns", options: { raw_expressions: true })
+
+ DiscourseWorkflows::DataTables::ColumnsResolver
+ .column_pairs(raw_columns)
+ &.find { |_, value| DiscourseWorkflows::ExpressionResolver.empty_expression?(value) }
+ &.first
+ end
+
+ def manual_columns?(config)
+ DiscourseWorkflows::Schema.visible?(
+ self.class.properties[:columns][:display_options],
+ config,
+ )
+ end
+
+ def auto_mapped_insert?(config)
+ config["operation"] == "insert" && config["mapping_mode"] == "auto"
+ end
+
def pair_output_items(output_items, exec_ctx, item)
paired_item = exec_ctx.paired_item_for(item)
@@ -230,14 +264,14 @@ module DiscourseWorkflows
node_proxy = exec_ctx.helpers.get_data_table_proxy(config.fetch("data_table_id"))
operation_name = config.fetch("operation", "insert")
- if operation_name == "insert" && config["mapping_mode"] == "auto"
+ if auto_mapped_insert?(config)
json = item["json"] || {}
config = config.merge("columns" => json.slice(*column_names(node_proxy)))
end
case operation_name
when "insert"
- wrap(node_proxy.insert_rows([config["columns"] || {}], "all"))
+ wrap(node_proxy.insert_rows([config["columns"]], "all"))
when "get"
wrap(node_proxy.get_many_rows_and_count(row_query_options(config))[:data])
when "update"
@@ -281,7 +315,7 @@ module DiscourseWorkflows
end
def row_mutation_options(config)
- { filter: normalized_filter(config), data: config["columns"] || {} }
+ { filter: normalized_filter(config), data: config["columns"] }
end
def row_delete_options(config)
diff --git a/plugins/discourse-workflows/spec/lib/discourse_workflows/data_tables/columns_resolver_spec.rb b/plugins/discourse-workflows/spec/lib/discourse_workflows/data_tables/columns_resolver_spec.rb
index 8dcc046b213..4372ea50d5e 100644
--- a/plugins/discourse-workflows/spec/lib/discourse_workflows/data_tables/columns_resolver_spec.rb
+++ b/plugins/discourse-workflows/spec/lib/discourse_workflows/data_tables/columns_resolver_spec.rb
@@ -36,15 +36,27 @@ RSpec.describe DiscourseWorkflows::DataTables::ColumnsResolver do
expect(resolver.resolve(nil)).to eq({})
end
- it "raises for unknown column names" do
- expect { resolver.resolve("unknown" => "x") }.to raise_error(
- ArgumentError,
- /Unknown column name/,
- )
+ it "raises when the input is not a hash or an array of column fields" do
+ ["oops", 42, [1, 2], [{ "value" => "x" }]].each do |input|
+ expect { resolver.resolve(input) }.to raise_error(
+ ArgumentError,
+ /Columns must be an object of column names to values/,
+ )
+ end
+ end
+
+ it "raises for unknown and system column names" do
+ %w[unknown id].each do |name|
+ expect { resolver.resolve(name => 1) }.to raise_error(ArgumentError, /Unknown column name/)
+ end
end
+ end
- it "does not resolve system columns" do
- expect { resolver.resolve("id" => 1) }.to raise_error(ArgumentError, /Unknown column name/)
+ describe ".column_pairs" do
+ it "returns nil when the columns are not keyed by column name" do
+ ["={{ $json }}", ["oops", { "columnName" => "email" }]].each do |input|
+ expect(described_class.column_pairs(input)).to be_nil
+ end
end
end
@@ -52,12 +64,5 @@ RSpec.describe DiscourseWorkflows::DataTables::ColumnsResolver do
it "returns nil for known columns" do
expect(resolver.validate_column!("email")).to be_nil
end
-
- it "raises for unknown column names" do
- expect { resolver.validate_column!("unknown") }.to raise_error(
- ArgumentError,
- /Unknown column name/,
- )
- end
end
end
diff --git a/plugins/discourse-workflows/spec/lib/discourse_workflows/data_tables/node_proxy_spec.rb b/plugins/discourse-workflows/spec/lib/discourse_workflows/data_tables/node_proxy_spec.rb
index a6fed093100..c212857b8ad 100644
--- a/plugins/discourse-workflows/spec/lib/discourse_workflows/data_tables/node_proxy_spec.rb
+++ b/plugins/discourse-workflows/spec/lib/discourse_workflows/data_tables/node_proxy_spec.rb
@@ -50,6 +50,14 @@ RSpec.describe DiscourseWorkflows::DataTables::NodeProxy do
)
end
+ it "raises instead of inserting a row without any columns" do
+ expect { proxy.insert_rows([{ "email" => "[email protected]" }, {}], "all") }.to raise_error(
+ ArgumentError,
+ /Cannot insert a row without any column/,
+ )
+ expect(count_data_table_rows(data_table)).to eq(0)
+ end
+
it "gets many rows and count" do
insert_data_table_row(data_table, "email" => "[email protected]", "score" => 1)
insert_data_table_row(data_table, "email" => "[email protected]", "score" => 2)
@@ -93,6 +101,40 @@ RSpec.describe DiscourseWorkflows::DataTables::NodeProxy do
)
end
+ context "when there are no columns to write" do
+ fab!(:row) { insert_data_table_row(data_table, "email" => "[email protected]", "score" => 1) }
+
+ it "returns the matched rows unchanged on update and upsert" do
+ %i[update_rows upsert_row].each do |method|
+ expect(proxy.public_send(method, filter: email_filter("[email protected]"), data: {})).to eq(
+ [row],
+ ),
+ method.to_s
+ end
+ expect(find_data_table_row(data_table, row["id"])).to eq(row)
+ end
+
+ it "returns identical before and after rows on a dry run" do
+ expected = [row.merge("dryRunState" => "before"), row.merge("dryRunState" => "after")]
+
+ %i[update_rows upsert_row].each do |method|
+ expect(
+ proxy.public_send(method, filter: email_filter("[email protected]"), data: {}, dry_run: true),
+ ).to eq(expected),
+ method.to_s
+ end
+ end
+
+ it "raises on an upsert that matches nothing, including a dry run" do
+ [false, true].each do |dry_run|
+ expect {
+ proxy.upsert_row(filter: email_filter("[email protected]"), data: {}, dry_run:)
+ }.to raise_error(ArgumentError, /Cannot insert a row without any column/)
+ end
+ expect(count_data_table_rows(data_table)).to eq(1)
+ end
+ end
+
it "deletes rows and returns deleted rows" do
insert_data_table_row(data_table, "email" => "[email protected]", "score" => 1)
insert_data_table_row(data_table, "email" => "[email protected]", "score" => 2)
diff --git a/plugins/discourse-workflows/spec/lib/discourse_workflows/expression_resolver_spec.rb b/plugins/discourse-workflows/spec/lib/discourse_workflows/expression_resolver_spec.rb
index dbaad77668f..dc51642dd38 100644
--- a/plugins/discourse-workflows/spec/lib/discourse_workflows/expression_resolver_spec.rb
+++ b/plugins/discourse-workflows/spec/lib/discourse_workflows/expression_resolver_spec.rb
@@ -642,6 +642,21 @@ RSpec.describe DiscourseWorkflows::ExpressionResolver do
end
end
+ describe ".empty_expression?" do
+ it "is true only for an expression with nothing to evaluate" do
+ {
+ "=" => true,
+ "= " => true,
+ "={{ }}" => true,
+ "= {{ }} " => true,
+ "={{ x }}" => false,
+ "={{ }}{{ x }}" => false,
+ "" => false,
+ nil => false,
+ }.each { |value, expected| expect(described_class.empty_expression?(value)).to eq(expected) }
+ end
+ end
+
describe ".resolve" do
fab!(:user)
diff --git a/plugins/discourse-workflows/spec/lib/discourse_workflows/nodes/data_table_spec.rb b/plugins/discourse-workflows/spec/lib/discourse_workflows/nodes/data_table_spec.rb
index 65d4d098c42..d67b2a4b7a2 100644
--- a/plugins/discourse-workflows/spec/lib/discourse_workflows/nodes/data_table_spec.rb
+++ b/plugins/discourse-workflows/spec/lib/discourse_workflows/nodes/data_table_spec.rb
@@ -67,6 +67,24 @@ RSpec.describe DiscourseWorkflows::Nodes::DataTable::V1 do
end
end
+ describe "columns property" do
+ let(:display_options) { described_class.properties[:columns][:display_options] }
+
+ it "is hidden only for insert in auto mapping mode" do
+ {
+ %w[insert auto] => false,
+ %w[insert manual] => true,
+ %w[update auto] => true,
+ %w[upsert auto] => true,
+ }.each do |(operation, mapping_mode), visible|
+ expect(
+ DiscourseWorkflows::Schema.visible?(display_options, operation:, mapping_mode:),
+ ).to eq(visible),
+ "#{operation}/#{mapping_mode}"
+ end
+ end
+ end
+
describe "error handling" do
it "raises when data table does not exist" do
expect {
@@ -80,18 +98,6 @@ RSpec.describe DiscourseWorkflows::Nodes::DataTable::V1 do
}.to raise_error(ActiveRecord::RecordNotFound)
end
- it "raises for unknown column names" do
- expect {
- execute_data_table(
- "operation" => "insert",
- "data_table_id" => data_table.id.to_s,
- "columns" => {
- "nonexistent" => "test",
- },
- )
- }.to raise_error(DiscourseWorkflows::NodeError, /Unknown column name/)
- end
-
it "raises invalid row errors with the node error prefix" do
expect {
execute_data_table(
@@ -119,27 +125,104 @@ RSpec.describe DiscourseWorkflows::Nodes::DataTable::V1 do
execute_data_table("operation" => "truncate", "data_table_id" => data_table.id.to_s)
}.to raise_error(DiscourseWorkflows::NodeError, /Unknown operation/)
end
- end
- describe "insert operation" do
it "raises when the storage limit is exceeded" do
+ insert_data_table_row(data_table, "email" => "[email protected]", "score" => 1)
allow(DiscourseWorkflows::DataTables::Facade).to receive(:within_storage_limit?).and_return(
false,
)
- expect {
- execute_data_table(
- "operation" => "insert",
+ # upsert filters on a missing row so it takes the insert path
+ {
+ "insert" => "[email protected]",
+ "update" => "[email protected]",
+ "upsert" => "[email protected]",
+ }.each do |operation, email|
+ expect {
+ execute_data_table(
+ "operation" => operation,
+ "data_table_id" => data_table.id.to_s,
+ "filter" => [filter_condition("email", "equals", value: email)],
+ "columns" => {
+ "score" => "99",
+ },
+ )
+ }.to raise_error(DiscourseWorkflows::NodeError, "Data table storage limit exceeded"),
+ operation
+ end
+ end
+ end
+
+ describe "empty column expressions" do
+ fab!(:row) { insert_data_table_row(data_table, "email" => "[email protected]", "score" => 1) }
+
+ def execute_with_columns(operation, columns, mapping_mode: "manual", json: {})
+ execute_data_table_with_item(
+ {
+ "operation" => operation,
"data_table_id" => data_table.id.to_s,
- "columns" => {
- "email" => "[email protected]",
- "score" => "42",
+ "mapping_mode" => mapping_mode,
+ "filter" => [filter_condition("email", "equals", value: "[email protected]")],
+ "columns" => columns,
+ },
+ { "json" => json },
+ )
+ end
+
+ it "raises instead of writing an empty string" do
+ [
+ %w[insert manual =],
+ ["update", "manual", "={{ }}"],
+ %w[update auto =],
+ ["upsert", "manual", "={{ }}"],
+ ].each do |operation, mapping_mode, value|
+ expect { execute_with_columns(operation, { "email" => value }, mapping_mode:) }.to(
+ raise_error(DiscourseWorkflows::NodeError, /Column 'email' has an empty expression/),
+ "#{operation}/#{mapping_mode} with #{value.inspect}",
+ )
+ end
+
+ expect {
+ execute_with_columns("update", [{ "columnName" => "email", "value" => "=" }])
+ }.to raise_error(DiscourseWorkflows::NodeError, /Column 'email' has an empty expression/)
+
+ expect(find_data_table_row(data_table, row["id"])).to eq(row)
+ expect(count_data_table_rows(data_table)).to eq(1)
+ end
+
+ it "ignores empty expressions when columns are not written" do
+ expect(execute_with_columns("get", { "email" => "=" }).size).to eq(1)
+
+ items =
+ execute_with_columns(
+ "insert",
+ { "email" => "=" },
+ mapping_mode: "auto",
+ json: {
+ "email" => "[email protected]",
},
)
- }.to raise_error(DiscourseWorkflows::NodeError, "Data table storage limit exceeded")
+ expect(items[0]["json"]).to include("email" => "[email protected]")
end
+ end
+ describe "insert operation" do
context "when mapping_mode is auto" do
+ it "raises when the item has no matching keys" do
+ expect {
+ execute_data_table_with_item(
+ {
+ "operation" => "insert",
+ "data_table_id" => data_table.id.to_s,
+ "mapping_mode" => "auto",
+ },
+ { "json" => { "ignored" => "value" } },
+ )
+ }.to raise_error(DiscourseWorkflows::NodeError, /Cannot insert a row without any column/)
+
+ expect(count_data_table_rows(data_table)).to eq(0)
+ end
+
it "picks matching keys from the incoming item JSON" do
items =
execute_data_table_with_item(
@@ -323,21 +406,54 @@ RSpec.describe DiscourseWorkflows::Nodes::DataTable::V1 do
expect(count_data_table_rows(data_table)).to eq(2)
end
- it "raises when the storage limit is exceeded" do
- allow(DiscourseWorkflows::DataTables::Facade).to receive(:within_storage_limit?).and_return(
- false,
- )
-
- expect {
+ describe "column values" do
+ def update_columns(columns)
execute_data_table(
"operation" => "update",
"data_table_id" => data_table.id.to_s,
- "filter" => [filter_condition("email", "equals", value: "[email protected]")],
- "columns" => {
- "score" => "99",
- },
+ "filter" => [filter_condition("score", "equals", value: 1, type: "number")],
+ "columns" => columns,
)
- }.to raise_error(DiscourseWorkflows::NodeError, "Data table storage limit exceeded")
+
+ find_data_table_row(data_table, row["id"])
+ end
+
+ it "writes present keys and leaves absent ones untouched" do
+ expect(update_columns("email" => "[email protected]")).to include(
+ "email" => "[email protected]",
+ "score" => 1,
+ )
+
+ [
+ [{ "email" => nil }, nil],
+ [{ "email" => "" }, ""],
+ [{ "email" => "={{ null }}" }, nil],
+ [{ "email" => '={{ "" }}' }, ""],
+ ].each do |columns, expected|
+ update_columns("email" => "[email protected]")
+
+ expect(update_columns(columns)).to include("email" => expected, "score" => 1),
+ columns.inspect
+ end
+ end
+
+ it "writes columns from an expression returning an object" do
+ expect(update_columns("={{ { score: 9, email: '[email protected]' } }}")).to include(
+ "email" => "[email protected]",
+ "score" => 9,
+ )
+ end
+
+ it "raises when the columns are not an object" do
+ expect { update_columns("={{ 'oops' }}") }.to raise_error(
+ DiscourseWorkflows::NodeError,
+ /Columns must be an object of column names to values, got 'oops'/,
+ )
+
+ expect {
+ update_columns(["oops", { "columnName" => "email", "value" => "=" }])
+ }.to raise_error(DiscourseWorkflows::NodeError, /Columns must be an object/)
+ end
end
end
@@ -532,23 +648,5 @@ RSpec.describe DiscourseWorkflows::Nodes::DataTable::V1 do
)
expect(count_data_table_rows(data_table)).to eq(1)
end
-
- it "raises when the storage limit is exceeded" do
- allow(DiscourseWorkflows::DataTables::Facade).to receive(:within_storage_limit?).and_return(
- false,
- )
-
- expect {
- execute_data_table(
- "operation" => "upsert",
- "data_table_id" => data_table.id.to_s,
- "filter" => [filter_condition("email", "equals", value: "[email protected]")],
- "columns" => {
- "email" => "[email protected]",
- "score" => "50",
- },
- )
- }.to raise_error(DiscourseWorkflows::NodeError, "Data table storage limit exceeded")
- end
end
end