A platform for community discussion. Free, open, simple.
FIX: Keep both values when Summarize split-by keys collide (#44549)
Régis Hanol62eba994606a
diff --git a/plugins/discourse-workflows/admin/assets/javascripts/admin/components/workflows/configurators/summarize-aggregations.gjs b/plugins/discourse-workflows/admin/assets/javascripts/admin/components/workflows/configurators/summarize-aggregations.gjsindex 1cd7290ccc9..e8e310b5d3f 100644--- a/plugins/discourse-workflows/admin/assets/javascripts/admin/components/workflows/configurators/summarize-aggregations.gjs+++ b/plugins/discourse-workflows/admin/assets/javascripts/admin/components/workflows/configurators/summarize-aggregations.gjs@@ -2,6 +2,7 @@ import Component from "@glimmer/component";import { i18n } from "discourse-i18n";import {fieldVisible,+ fixedCollectionRows,normalizeOptions,normalizeSchema,propertyLabel,@@ -22,9 +23,12 @@ function isDerived(item) {return summarizeOutputKeyIsDerived(item || {});}-function derivedPlaceholder(item) {+function derivedPlaceholder(item, configuration, fieldName) {return summarizeOutputKeyIsDerived(item || {})- ? summarizeOutputKey(item || {})+ ? summarizeOutputKey(+ item || {},+ fixedCollectionRows(configuration?.[fieldName])+ ): "";}@@ -186,7 +190,11 @@ export default class SummarizeAggregations extends Component {'workflows-summarize-row__control--derived'}}"@name="output_field_name"- @placeholder={{derivedPlaceholder row.item}}+ @placeholder={{derivedPlaceholder+ row.item+ @configuration+ @fieldName+ }}@showTitle={{false}}@title={{this.fieldTitle "output_field_name"}}as |field|diff --git a/plugins/discourse-workflows/admin/assets/javascripts/admin/lib/workflows/output-schemas/summarize.js b/plugins/discourse-workflows/admin/assets/javascripts/admin/lib/workflows/output-schemas/summarize.jsindex 9a2cb060e0a..bfffe14646e 100644--- a/plugins/discourse-workflows/admin/assets/javascripts/admin/lib/workflows/output-schemas/summarize.js+++ b/plugins/discourse-workflows/admin/assets/javascripts/admin/lib/workflows/output-schemas/summarize.js@@ -1,5 +1,5 @@import { fixedCollectionRows } from "../property-engine";-import { summarizeOutputKey } from "../summarize-output-key";+import { fieldKey, summarizeOutputKey } from "../summarize-output-key";const DRAFT_URI = "https://json-schema.org/draft/2020-12/schema";const NUMBER_TYPE = { type: ["number", "null"] };@@ -34,18 +34,17 @@ function splitFieldList(value) {.filter(Boolean);}-function leafName(field) {- return field.includes(".") ? field.split(".").pop() : field;-}-export default function summarizeOutputSchemas(configuration = {}) {const properties = {};- for (const field of splitFieldList(configuration.fields_to_split_by)) {- properties[leafName(field)] = ANY_TYPE;+ const splitFields = splitFieldList(configuration.fields_to_split_by);+ const rows = fixedCollectionRows(configuration.fields_to_summarize);++ for (const field of splitFields) {+ properties[fieldKey(field, splitFields)] = ANY_TYPE;}- for (const row of fixedCollectionRows(configuration.fields_to_summarize)) {- properties[summarizeOutputKey(row)] = schemaTypeFor(row);+ for (const row of rows) {+ properties[summarizeOutputKey(row, rows)] = schemaTypeFor(row);}return [{ $schema: DRAFT_URI, type: "object", properties }];diff --git a/plugins/discourse-workflows/admin/assets/javascripts/admin/lib/workflows/summarize-output-key.js b/plugins/discourse-workflows/admin/assets/javascripts/admin/lib/workflows/summarize-output-key.jsindex bc9b3e1a825..adf11ab4818 100644--- a/plugins/discourse-workflows/admin/assets/javascripts/admin/lib/workflows/summarize-output-key.js+++ b/plugins/discourse-workflows/admin/assets/javascripts/admin/lib/workflows/summarize-output-key.js@@ -25,7 +25,27 @@ function sanitizeKey(name) {.replace(/[ .]/g, "_");}-export function summarizeOutputKey(row = {}) {+export function fieldKey(field, fields) {+ const leaf = leafName(field);+ if (fields.filter((other) => leafName(other) === leaf).length < 2) {+ return leaf;+ }++ return sanitizeKey(field);+}++function derivedFields(rows, aggregation) {+ return rows+ .filter(+ (row) =>+ String(row.aggregation ?? "") === aggregation &&+ summarizeOutputKeyIsDerived(row)+ )+ .map((row) => String(row.field ?? "").trim())+ .filter(Boolean);+}++export function summarizeOutputKey(row = {}, rows = [row]) {const explicit = String(row.output_field_name ?? "").trim();if (explicit) {return explicit;@@ -39,7 +59,9 @@ export function summarizeOutputKey(row = {}) {return prefix.replace(/_$/, "");}- return `${prefix}${sanitizeKey(leafName(field))}`;+ return `${prefix}${sanitizeKey(+ fieldKey(field, derivedFields(rows, aggregation))+ )}`;}export function summarizeOutputKeyIsDerived(row = {}) {diff --git a/plugins/discourse-workflows/lib/discourse_workflows/nodes/summarize/v1.rb b/plugins/discourse-workflows/lib/discourse_workflows/nodes/summarize/v1.rbindex 56a71d7d5e3..0e23948acd6 100644--- a/plugins/discourse-workflows/lib/discourse_workflows/nodes/summarize/v1.rb+++ b/plugins/discourse-workflows/lib/discourse_workflows/nodes/summarize/v1.rb@@ -145,8 +145,8 @@ module DiscourseWorkflowssplit_fields = split_field_list(config["fields_to_split_by"])properties = {}- split_fields.each { |field| properties[leaf_name(field)] = ANY_TYPE }- rows.each { |row| properties[output_key(row)] = schema_type_for(row) }+ split_fields.each { |field| properties[field_key(field, split_fields)] = ANY_TYPE }+ rows.each { |row| properties[output_key(row, rows)] = schema_type_for(row) }[Schema.document(properties)]end@@ -160,7 +160,14 @@ module DiscourseWorkflowsfield.split(".").lastend- def self.output_key(row)+ def self.field_key(field, fields)+ leaf = leaf_name(field)+ return leaf if fields.count { |other| leaf_name(other) == leaf } < 2++ sanitize_key(field)+ end++ def self.output_key(row, rows)row = row.deep_stringify_keysexplicit = row["output_field_name"].to_s.stripreturn explicit if explicit.present?@@ -169,7 +176,18 @@ module DiscourseWorkflowsfield = row["field"].to_s.stripreturn prefix.chomp("_") if field.empty?- "#{prefix}#{sanitize_key(leaf_name(field))}"+ "#{prefix}#{sanitize_key(field_key(field, derived_fields(rows, row["aggregation"])))}"+ end++ def self.derived_fields(rows, aggregation)+ rows.filter_map do |row|+ row = row.deep_stringify_keys+ if row["aggregation"] != aggregation || row["output_field_name"].to_s.strip.present?+ next+ end++ row["field"].to_s.strip.presence+ endenddef self.sanitize_key(name)@@ -196,10 +214,13 @@ module DiscourseWorkflowsself.class.split_field_list(exec_ctx.get_node_parameter("fields_to_split_by", 0, default: ""),)+ split_keys = split_fields.map { |field| self.class.field_key(field, split_fields) }rows = normalized_rows(exec_ctx)+ keyed_rows = rows.map { |row| [self.class.output_key(row, rows), row] }+ validate_unique_output_keys!(keyed_rows.map(&:first))groups = group_items(exec_ctx.input_items, split_fields)- [separate_items(groups, split_fields, rows, exec_ctx)]+ [separate_items(groups, split_keys, keyed_rows, exec_ctx)]endprivate@@ -218,7 +239,6 @@ module DiscourseWorkflowsendrows.each { |row| validate_aggregation!(row) }- validate_unique_output_keys!(rows)rowsend@@ -234,8 +254,7 @@ module DiscourseWorkflows)end- def validate_unique_output_keys!(rows)- keys = rows.map { |row| self.class.output_key(row) }+ def validate_unique_output_keys!(keys)duplicate = keys.tally.find { |_key, count| count > 1 }&.firstreturn if duplicate.nil?@@ -253,24 +272,16 @@ module DiscourseWorkflows.map { |keys, items| { keys: keys, items: items } }end- def separate_items(groups, split_fields, rows, exec_ctx)+ def separate_items(groups, split_keys, keyed_rows, exec_ctx)groups.map do |group|- json = {}-- split_fields.each_with_index do |field, index|- json[self.class.leaf_name(field)] = group[:keys][index]- end-- json.merge!(aggregate(group[:items], rows))+ json = split_keys.zip(group[:keys]).to_h.merge!(aggregate(group[:items], keyed_rows))wrap(json, paired_item: paired_items_for(group[:items], exec_ctx))endend- def aggregate(items, rows)- rows.each_with_object({}) do |row, result|- result[self.class.output_key(row)] = compute(row, items)- end+ def aggregate(items, keyed_rows)+ keyed_rows.to_h { |key, row| [key, compute(row, items)] }enddef compute(row, items)diff --git a/plugins/discourse-workflows/spec/lib/discourse_workflows/nodes/summarize_spec.rb b/plugins/discourse-workflows/spec/lib/discourse_workflows/nodes/summarize_spec.rbindex a3ce84de34b..1fc37989600 100644--- a/plugins/discourse-workflows/spec/lib/discourse_workflows/nodes/summarize_spec.rb+++ b/plugins/discourse-workflows/spec/lib/discourse_workflows/nodes/summarize_spec.rb@@ -108,6 +108,35 @@ RSpec.describe DiscourseWorkflows::Nodes::Summarize::V1 doexpect(json(result)).to eq([{ "topic_id" => 7, "count" => 1 }])end++ it "writes colliding dotted group keys under their full path" do+ input = [{ "json" => { "topic" => { "id" => 7 }, "user" => { "id" => 3 } } }]+ config = {+ "fields_to_split_by" => "topic.id, user.id",+ "fields_to_summarize" => aggregations({ "aggregation" => "count" }),+ }++ result = execute(input, config)++ expect(json(result)).to eq([{ "topic_id" => 7, "user_id" => 3, "count" => 1 }])+ expect(json(result).first).to match_node_output_schema(described_class, configuration: config)+ end++ it "names colliding derived aggregates after their full path" do+ input = [{ "json" => { "topic" => { "id" => 7 }, "user" => { "id" => 3 } } }]+ config = {+ "fields_to_summarize" =>+ aggregations(+ { "aggregation" => "count", "field" => "topic.id" },+ { "aggregation" => "count", "field" => "user.id" },+ ),+ }++ result = execute(input, config)++ expect(json(result)).to eq([{ "count_topic_id" => 1, "count_user_id" => 1 }])+ expect(json(result).first).to match_node_output_schema(described_class, configuration: config)+ endenddescribe "collect" dodiff --git a/plugins/discourse-workflows/test/javascripts/unit/lib/workflows-summarize-output-key-test.js b/plugins/discourse-workflows/test/javascripts/unit/lib/workflows-summarize-output-key-test.jsindex 0f105abf689..d0afae30654 100644--- a/plugins/discourse-workflows/test/javascripts/unit/lib/workflows-summarize-output-key-test.js+++ b/plugins/discourse-workflows/test/javascripts/unit/lib/workflows-summarize-output-key-test.js@@ -38,6 +38,27 @@ module("Unit | lib | discourse-workflows | summarizeOutputKey", function () {);});+ test("uses the full path when derived leaves collide", function (assert) {+ const rows = [+ { aggregation: "count", field: "topic.id" },+ { aggregation: "count", field: "user.id" },+ { aggregation: "sum", field: "post.id" },+ ];++ assert.strictEqual(summarizeOutputKey(rows[0], rows), "count_topic_id");+ assert.strictEqual(summarizeOutputKey(rows[1], rows), "count_user_id");+ assert.strictEqual(summarizeOutputKey(rows[2], rows), "sum_id");+ });++ test("ignores rows with explicit names when checking collisions", function (assert) {+ const rows = [+ { aggregation: "count", field: "topic.id" },+ { aggregation: "count", field: "user.id", output_field_name: "users" },+ ];++ assert.strictEqual(summarizeOutputKey(rows[0], rows), "count_id");+ });+test("drops the trailing underscore when there is no field", function (assert) {assert.strictEqual(summarizeOutputKey({ aggregation: "count" }), "count");assert.strictEqual(diff --git a/plugins/discourse-workflows/test/javascripts/unit/lib/workflows-summarize-output-schemas-test.js b/plugins/discourse-workflows/test/javascripts/unit/lib/workflows-summarize-output-schemas-test.jsindex b6f5a0654a9..cd91cbee997 100644--- a/plugins/discourse-workflows/test/javascripts/unit/lib/workflows-summarize-output-schemas-test.js+++ b/plugins/discourse-workflows/test/javascripts/unit/lib/workflows-summarize-output-schemas-test.js@@ -36,6 +36,25 @@ module(assert.deepEqual(Object.keys(schema.properties), ["topic_id", "tags"]);});+ test("uses the full path when group key leaves collide", function (assert) {+ const [schema] = summarizeOutputSchemas({+ fields_to_split_by: "topic.id, user.id",+ fields_to_summarize: {+ values: [+ { aggregation: "count", field: "topic.id" },+ { aggregation: "count", field: "user.id" },+ ],+ },+ });++ assert.deepEqual(Object.keys(schema.properties), [+ "topic_id",+ "user_id",+ "count_topic_id",+ "count_user_id",+ ]);+ });+test("resolves an empty shape without configuration", function (assert) {const [schema] = summarizeOutputSchemas();