Skip to content

A platform for community discussion. Free, open, simple.

FIX: Keep both values when Summarize split-by keys collide (#44549)

Régis Hanol62eba994606a
diff --git a/plugins/discourse-workflows/admin/assets/javascripts/admin/components/workflows/configurators/summarize-aggregations.gjs b/plugins/discourse-workflows/admin/assets/javascripts/admin/components/workflows/configurators/summarize-aggregations.gjs
index 1cd7290ccc9..e8e310b5d3f 100644
--- a/plugins/discourse-workflows/admin/assets/javascripts/admin/components/workflows/configurators/summarize-aggregations.gjs
+++ b/plugins/discourse-workflows/admin/assets/javascripts/admin/components/workflows/configurators/summarize-aggregations.gjs
@@ -2,6 +2,7 @@ import Component from "@glimmer/component";
import { i18n } from "discourse-i18n";
import {
fieldVisible,
+ fixedCollectionRows,
normalizeOptions,
normalizeSchema,
propertyLabel,
@@ -22,9 +23,12 @@ function isDerived(item) {
return summarizeOutputKeyIsDerived(item || {});
}
-function derivedPlaceholder(item) {
+function derivedPlaceholder(item, configuration, fieldName) {
return summarizeOutputKeyIsDerived(item || {})
- ? summarizeOutputKey(item || {})
+ ? summarizeOutputKey(
+ item || {},
+ fixedCollectionRows(configuration?.[fieldName])
+ )
: "";
}
@@ -186,7 +190,11 @@ export default class SummarizeAggregations extends Component {
'workflows-summarize-row__control--derived'
}}"
@name="output_field_name"
- @placeholder={{derivedPlaceholder row.item}}
+ @placeholder={{derivedPlaceholder
+ row.item
+ @configuration
+ @fieldName
+ }}
@showTitle={{false}}
@title={{this.fieldTitle "output_field_name"}}
as |field|
diff --git a/plugins/discourse-workflows/admin/assets/javascripts/admin/lib/workflows/output-schemas/summarize.js b/plugins/discourse-workflows/admin/assets/javascripts/admin/lib/workflows/output-schemas/summarize.js
index 9a2cb060e0a..bfffe14646e 100644
--- a/plugins/discourse-workflows/admin/assets/javascripts/admin/lib/workflows/output-schemas/summarize.js
+++ b/plugins/discourse-workflows/admin/assets/javascripts/admin/lib/workflows/output-schemas/summarize.js
@@ -1,5 +1,5 @@
import { fixedCollectionRows } from "../property-engine";
-import { summarizeOutputKey } from "../summarize-output-key";
+import { fieldKey, summarizeOutputKey } from "../summarize-output-key";
const DRAFT_URI = "https://json-schema.org/draft/2020-12/schema";
const NUMBER_TYPE = { type: ["number", "null"] };
@@ -34,18 +34,17 @@ function splitFieldList(value) {
.filter(Boolean);
}
-function leafName(field) {
- return field.includes(".") ? field.split(".").pop() : field;
-}
-
export default function summarizeOutputSchemas(configuration = {}) {
const properties = {};
- for (const field of splitFieldList(configuration.fields_to_split_by)) {
- properties[leafName(field)] = ANY_TYPE;
+ const splitFields = splitFieldList(configuration.fields_to_split_by);
+ const rows = fixedCollectionRows(configuration.fields_to_summarize);
+
+ for (const field of splitFields) {
+ properties[fieldKey(field, splitFields)] = ANY_TYPE;
}
- for (const row of fixedCollectionRows(configuration.fields_to_summarize)) {
- properties[summarizeOutputKey(row)] = schemaTypeFor(row);
+ for (const row of rows) {
+ properties[summarizeOutputKey(row, rows)] = schemaTypeFor(row);
}
return [{ $schema: DRAFT_URI, type: "object", properties }];
diff --git a/plugins/discourse-workflows/admin/assets/javascripts/admin/lib/workflows/summarize-output-key.js b/plugins/discourse-workflows/admin/assets/javascripts/admin/lib/workflows/summarize-output-key.js
index bc9b3e1a825..adf11ab4818 100644
--- a/plugins/discourse-workflows/admin/assets/javascripts/admin/lib/workflows/summarize-output-key.js
+++ b/plugins/discourse-workflows/admin/assets/javascripts/admin/lib/workflows/summarize-output-key.js
@@ -25,7 +25,27 @@ function sanitizeKey(name) {
.replace(/[ .]/g, "_");
}
-export function summarizeOutputKey(row = {}) {
+export function fieldKey(field, fields) {
+ const leaf = leafName(field);
+ if (fields.filter((other) => leafName(other) === leaf).length < 2) {
+ return leaf;
+ }
+
+ return sanitizeKey(field);
+}
+
+function derivedFields(rows, aggregation) {
+ return rows
+ .filter(
+ (row) =>
+ String(row.aggregation ?? "") === aggregation &&
+ summarizeOutputKeyIsDerived(row)
+ )
+ .map((row) => String(row.field ?? "").trim())
+ .filter(Boolean);
+}
+
+export function summarizeOutputKey(row = {}, rows = [row]) {
const explicit = String(row.output_field_name ?? "").trim();
if (explicit) {
return explicit;
@@ -39,7 +59,9 @@ export function summarizeOutputKey(row = {}) {
return prefix.replace(/_$/, "");
}
- return `${prefix}${sanitizeKey(leafName(field))}`;
+ return `${prefix}${sanitizeKey(
+ fieldKey(field, derivedFields(rows, aggregation))
+ )}`;
}
export function summarizeOutputKeyIsDerived(row = {}) {
diff --git a/plugins/discourse-workflows/lib/discourse_workflows/nodes/summarize/v1.rb b/plugins/discourse-workflows/lib/discourse_workflows/nodes/summarize/v1.rb
index 56a71d7d5e3..0e23948acd6 100644
--- a/plugins/discourse-workflows/lib/discourse_workflows/nodes/summarize/v1.rb
+++ b/plugins/discourse-workflows/lib/discourse_workflows/nodes/summarize/v1.rb
@@ -145,8 +145,8 @@ module DiscourseWorkflows
split_fields = split_field_list(config["fields_to_split_by"])
properties = {}
- split_fields.each { |field| properties[leaf_name(field)] = ANY_TYPE }
- rows.each { |row| properties[output_key(row)] = schema_type_for(row) }
+ split_fields.each { |field| properties[field_key(field, split_fields)] = ANY_TYPE }
+ rows.each { |row| properties[output_key(row, rows)] = schema_type_for(row) }
[Schema.document(properties)]
end
@@ -160,7 +160,14 @@ module DiscourseWorkflows
field.split(".").last
end
- def self.output_key(row)
+ def self.field_key(field, fields)
+ leaf = leaf_name(field)
+ return leaf if fields.count { |other| leaf_name(other) == leaf } < 2
+
+ sanitize_key(field)
+ end
+
+ def self.output_key(row, rows)
row = row.deep_stringify_keys
explicit = row["output_field_name"].to_s.strip
return explicit if explicit.present?
@@ -169,7 +176,18 @@ module DiscourseWorkflows
field = row["field"].to_s.strip
return prefix.chomp("_") if field.empty?
- "#{prefix}#{sanitize_key(leaf_name(field))}"
+ "#{prefix}#{sanitize_key(field_key(field, derived_fields(rows, row["aggregation"])))}"
+ end
+
+ def self.derived_fields(rows, aggregation)
+ rows.filter_map do |row|
+ row = row.deep_stringify_keys
+ if row["aggregation"] != aggregation || row["output_field_name"].to_s.strip.present?
+ next
+ end
+
+ row["field"].to_s.strip.presence
+ end
end
def self.sanitize_key(name)
@@ -196,10 +214,13 @@ module DiscourseWorkflows
self.class.split_field_list(
exec_ctx.get_node_parameter("fields_to_split_by", 0, default: ""),
)
+ split_keys = split_fields.map { |field| self.class.field_key(field, split_fields) }
rows = normalized_rows(exec_ctx)
+ keyed_rows = rows.map { |row| [self.class.output_key(row, rows), row] }
+ validate_unique_output_keys!(keyed_rows.map(&:first))
groups = group_items(exec_ctx.input_items, split_fields)
- [separate_items(groups, split_fields, rows, exec_ctx)]
+ [separate_items(groups, split_keys, keyed_rows, exec_ctx)]
end
private
@@ -218,7 +239,6 @@ module DiscourseWorkflows
end
rows.each { |row| validate_aggregation!(row) }
- validate_unique_output_keys!(rows)
rows
end
@@ -234,8 +254,7 @@ module DiscourseWorkflows
)
end
- def validate_unique_output_keys!(rows)
- keys = rows.map { |row| self.class.output_key(row) }
+ def validate_unique_output_keys!(keys)
duplicate = keys.tally.find { |_key, count| count > 1 }&.first
return if duplicate.nil?
@@ -253,24 +272,16 @@ module DiscourseWorkflows
.map { |keys, items| { keys: keys, items: items } }
end
- def separate_items(groups, split_fields, rows, exec_ctx)
+ def separate_items(groups, split_keys, keyed_rows, exec_ctx)
groups.map do |group|
- json = {}
-
- split_fields.each_with_index do |field, index|
- json[self.class.leaf_name(field)] = group[:keys][index]
- end
-
- json.merge!(aggregate(group[:items], rows))
+ json = split_keys.zip(group[:keys]).to_h.merge!(aggregate(group[:items], keyed_rows))
wrap(json, paired_item: paired_items_for(group[:items], exec_ctx))
end
end
- def aggregate(items, rows)
- rows.each_with_object({}) do |row, result|
- result[self.class.output_key(row)] = compute(row, items)
- end
+ def aggregate(items, keyed_rows)
+ keyed_rows.to_h { |key, row| [key, compute(row, items)] }
end
def compute(row, items)
diff --git a/plugins/discourse-workflows/spec/lib/discourse_workflows/nodes/summarize_spec.rb b/plugins/discourse-workflows/spec/lib/discourse_workflows/nodes/summarize_spec.rb
index a3ce84de34b..1fc37989600 100644
--- a/plugins/discourse-workflows/spec/lib/discourse_workflows/nodes/summarize_spec.rb
+++ b/plugins/discourse-workflows/spec/lib/discourse_workflows/nodes/summarize_spec.rb
@@ -108,6 +108,35 @@ RSpec.describe DiscourseWorkflows::Nodes::Summarize::V1 do
expect(json(result)).to eq([{ "topic_id" => 7, "count" => 1 }])
end
+
+ it "writes colliding dotted group keys under their full path" do
+ input = [{ "json" => { "topic" => { "id" => 7 }, "user" => { "id" => 3 } } }]
+ config = {
+ "fields_to_split_by" => "topic.id, user.id",
+ "fields_to_summarize" => aggregations({ "aggregation" => "count" }),
+ }
+
+ result = execute(input, config)
+
+ expect(json(result)).to eq([{ "topic_id" => 7, "user_id" => 3, "count" => 1 }])
+ expect(json(result).first).to match_node_output_schema(described_class, configuration: config)
+ end
+
+ it "names colliding derived aggregates after their full path" do
+ input = [{ "json" => { "topic" => { "id" => 7 }, "user" => { "id" => 3 } } }]
+ config = {
+ "fields_to_summarize" =>
+ aggregations(
+ { "aggregation" => "count", "field" => "topic.id" },
+ { "aggregation" => "count", "field" => "user.id" },
+ ),
+ }
+
+ result = execute(input, config)
+
+ expect(json(result)).to eq([{ "count_topic_id" => 1, "count_user_id" => 1 }])
+ expect(json(result).first).to match_node_output_schema(described_class, configuration: config)
+ end
end
describe "collect" do
diff --git a/plugins/discourse-workflows/test/javascripts/unit/lib/workflows-summarize-output-key-test.js b/plugins/discourse-workflows/test/javascripts/unit/lib/workflows-summarize-output-key-test.js
index 0f105abf689..d0afae30654 100644
--- a/plugins/discourse-workflows/test/javascripts/unit/lib/workflows-summarize-output-key-test.js
+++ b/plugins/discourse-workflows/test/javascripts/unit/lib/workflows-summarize-output-key-test.js
@@ -38,6 +38,27 @@ module("Unit | lib | discourse-workflows | summarizeOutputKey", function () {
);
});
+ test("uses the full path when derived leaves collide", function (assert) {
+ const rows = [
+ { aggregation: "count", field: "topic.id" },
+ { aggregation: "count", field: "user.id" },
+ { aggregation: "sum", field: "post.id" },
+ ];
+
+ assert.strictEqual(summarizeOutputKey(rows[0], rows), "count_topic_id");
+ assert.strictEqual(summarizeOutputKey(rows[1], rows), "count_user_id");
+ assert.strictEqual(summarizeOutputKey(rows[2], rows), "sum_id");
+ });
+
+ test("ignores rows with explicit names when checking collisions", function (assert) {
+ const rows = [
+ { aggregation: "count", field: "topic.id" },
+ { aggregation: "count", field: "user.id", output_field_name: "users" },
+ ];
+
+ assert.strictEqual(summarizeOutputKey(rows[0], rows), "count_id");
+ });
+
test("drops the trailing underscore when there is no field", function (assert) {
assert.strictEqual(summarizeOutputKey({ aggregation: "count" }), "count");
assert.strictEqual(
diff --git a/plugins/discourse-workflows/test/javascripts/unit/lib/workflows-summarize-output-schemas-test.js b/plugins/discourse-workflows/test/javascripts/unit/lib/workflows-summarize-output-schemas-test.js
index b6f5a0654a9..cd91cbee997 100644
--- a/plugins/discourse-workflows/test/javascripts/unit/lib/workflows-summarize-output-schemas-test.js
+++ b/plugins/discourse-workflows/test/javascripts/unit/lib/workflows-summarize-output-schemas-test.js
@@ -36,6 +36,25 @@ module(
assert.deepEqual(Object.keys(schema.properties), ["topic_id", "tags"]);
});
+ test("uses the full path when group key leaves collide", function (assert) {
+ const [schema] = summarizeOutputSchemas({
+ fields_to_split_by: "topic.id, user.id",
+ fields_to_summarize: {
+ values: [
+ { aggregation: "count", field: "topic.id" },
+ { aggregation: "count", field: "user.id" },
+ ],
+ },
+ });
+
+ assert.deepEqual(Object.keys(schema.properties), [
+ "topic_id",
+ "user_id",
+ "count_topic_id",
+ "count_user_id",
+ ]);
+ });
+
test("resolves an empty shape without configuration", function (assert) {
const [schema] = summarizeOutputSchemas();