A platform for community discussion. Free, open, simple.
FIX: `type: objects` uploads should be stored as IDs (#40178)
Gabriel Grubbae52b385ae1ab
diff --git a/app/models/site_setting.rb b/app/models/site_setting.rbindex 5a4b206cc5c..945ba9b5aba 100644--- a/app/models/site_setting.rb+++ b/app/models/site_setting.rb@@ -93,24 +93,12 @@ class SiteSetting < ActiveRecord::Baseupload_ids = value.split("|").compact.uniqUploadReference.ensure_exist!(upload_ids: upload_ids, target: self)elsif data_type == SiteSettings::TypeSupervisor.types[:objects] && value.present?- upload_values =- SchemaSettingsObjectValidator.property_values_of_type(+ upload_ids =+ SchemaSettingsObjectValidator.upload_ids(schema: SiteSetting.type_supervisor.type_hash(name.to_sym)[:schema],objects: JSON.parse(value),- type: "upload",)- # Convert URLs to upload IDs (values can be either integer IDs or URL strings)- upload_ids =- upload_values.filter_map do |value|- if value.is_a?(Integer)- value- elsif value.is_a?(String) && value.present?- ::Upload.get_from_url(value)&.id- end- end-- # Always call ensure_exist! to clean up old references when uploads are removedUploadReference.ensure_exist!(upload_ids: upload_ids, target: self)endenddiff --git a/app/models/theme_setting.rb b/app/models/theme_setting.rbindex 651e3a43f1e..38dcfaa0d9e 100644--- a/app/models/theme_setting.rb+++ b/app/models/theme_setting.rb@@ -23,30 +23,18 @@ class ThemeSetting < ActiveRecord::BaseUploadReference.ensure_exist!(upload_ids: [value], target: self)elsif data_type == ThemeSetting.types[:objects] && saved_change_to_json_value? &&json_value.present?- upload_values =- SchemaSettingsObjectValidator.property_values_of_type(+ upload_ids =+ SchemaSettingsObjectValidator.upload_ids(schema: theme.settings[name.to_sym].schema,objects: json_value,- type: "upload",)- # Convert URLs to upload IDs (values can be either integer IDs or URL strings)- upload_ids =- upload_values.filter_map do |value|- if value.is_a?(Integer)- value- elsif value.is_a?(String) && value.present?- Upload.get_from_url(value)&.id- end- end-- # Always call ensure_exist! to clean up old references when uploads are removedUploadReference.ensure_exist!(upload_ids: upload_ids, target: self)endif theme.theme_modifier_set.refresh_theme_setting_modifiers(target_setting_name: name,- target_setting_value: self.value,+ target_setting_value: value,)theme.theme_modifier_set.save!enddiff --git a/lib/schema_settings_object_validator.rb b/lib/schema_settings_object_validator.rbindex fad4e605f2e..cc4f7b2c82b 100644--- a/lib/schema_settings_object_validator.rb+++ b/lib/schema_settings_object_validator.rb@@ -26,8 +26,84 @@ class SchemaSettingsObjectValidatorvalues.to_aend+ def upload_ids(schema:, objects:)+ property_values_of_type(schema:, objects:, type: "upload").filter_map do |value|+ # TODO(gabriel): this if branch was to deal wit ha legacy logic bug where some upload fields were stored as URLs instead of IDs.+ # This can be removed by May 2027.+ if value.is_a?(Integer)+ value+ elsif value.is_a?(String) && value.present?+ Upload.get_from_url(value)&.id+ end+ end+ end++ def normalize_uploads(schema:, objects:)+ return objects if objects.blank?++ transform_uploads_in_objects(objects.deep_dup, schema[:properties]) do |value|+ if value.is_a?(String) && value.present?+ Upload.get_from_url(value)&.id || value+ else+ value+ end+ end+ end++ def hydrate_uploads(schema:, objects:, cdn: false)+ return objects if objects.blank?++ upload_ids =+ property_values_of_type(schema: schema, objects: objects, type: "upload").select do |value|+ value.is_a?(Integer)+ end+ return objects if upload_ids.empty?++ uploads_by_id = Upload.where(id: upload_ids).index_by(&:id)+ transform_uploads_in_objects(objects.deep_dup, schema[:properties]) do |value|+ if upload = uploads_by_id[value]+ cdn ? Discourse.store.cdn_url(upload.url) : upload.url+ else+ value+ end+ end+ end+private+ def transform_uploads_in_objects(objects, properties, &block)+ objects.each do |object|+ properties.each do |property_name, property_attributes|+ key = object_key(object, property_name)+ next if key.nil?++ case property_attributes[:type]+ when "upload"+ object[key] = block.call(object[key])+ when "objects"+ nested_objects = object[key]+ if nested_objects.is_a?(Array)+ transform_uploads_in_objects(+ nested_objects,+ property_attributes[:schema][:properties],+ &block+ )+ end+ end+ end+ end++ objects+ end++ def object_key(object, property_name)+ string_key = property_name.to_s+ return string_key if object.key?(string_key)++ symbol_key = property_name.to_sym+ symbol_key if object.key?(symbol_key)+ end+def humanize_error_messages(errors, index:, error_messages:)errors.each do |property_json_pointer, error_details|error_messages.push(*error_details.humanize_messages("/#{index}#{property_json_pointer}"))diff --git a/lib/site_setting_extension.rb b/lib/site_setting_extension.rbindex e9f16146607..c93fbde9b01 100644--- a/lib/site_setting_extension.rb+++ b/lib/site_setting_extension.rb@@ -307,6 +307,11 @@ module SiteSettingExtensionvalue = value.to_selsif type == :uploaded_image_list && value.present?value = value.map(&:to_s).join("|")+ elsif type == :objects && value.present?+ type_hash = type_supervisor.type_hash(name)+ if type_hash[:schema]+ value = hydrate_objects_setting_value(name, value, type_hash:).to_json+ endend[name, value]@@ -456,8 +461,7 @@ module SiteSettingExtension# For uploads nested in objects type, hydrate upload IDs to URLsif type_hash[:type].to_s == "objects" && type_hash[:schema]- parsed_value = JSON.parse(value)- value = hydrate_uploads_in_objects(parsed_value, type_hash[:schema])+ value = hydrate_objects_setting_value(s, value, type_hash:)endopts = {@@ -1382,38 +1386,11 @@ module SiteSettingExtensionprivate- def hydrate_uploads_in_objects(objects, schema)- return objects if objects.blank?-- upload_ids =- SchemaSettingsObjectValidator.property_values_of_type(- schema: schema,- objects: objects,- type: "upload",- )-- uploads_by_id = Upload.where(id: upload_ids).index_by(&:id)- objects.map { |obj| hydrate_uploads_in_object(obj, schema[:properties], uploads_by_id) }- end-- def hydrate_uploads_in_object(object, properties, uploads_by_id)- properties.each do |prop_key, prop_value|- case prop_value[:type]- when "upload"- key = prop_key.to_s- upload_id = object[key]- upload = uploads_by_id[upload_id]- object[key] = upload.url if upload- when "objects"- nested_objects = object[prop_key.to_s]- if nested_objects.is_a?(Array)- nested_objects.each do |nested_obj|- hydrate_uploads_in_object(nested_obj, prop_value[:schema][:properties], uploads_by_id)- end- end- end- end+ def hydrate_objects_setting_value(name, value, type_hash: nil)+ type_hash ||= type_supervisor.type_hash(name)+ return value if !type_hash[:schema] || value.blank?- object+ objects = value.is_a?(String) ? JSON.parse(value) : value+ SchemaSettingsObjectValidator.hydrate_uploads(objects:, schema: type_hash[:schema])endenddiff --git a/lib/site_settings/object_uploads_normalizer.rb b/lib/site_settings/object_uploads_normalizer.rbnew file mode 100644index 00000000000..fa183e04259--- /dev/null+++ b/lib/site_settings/object_uploads_normalizer.rb@@ -0,0 +1,99 @@+# frozen_string_literal: true++# TODO(gabriel): This was added as a fix for a bug in `type: object` settings.+# This is only used by a rake task. This class can be removed by May 2027++module SiteSettings+ class ObjectUploadsNormalizer+ def initialize(dry_run: false, output: $stdout)+ @dry_run = dry_run+ @output = output+ @counts = Hash.new(0)+ end++ def normalize+ normalize_site_settings+ normalize_theme_settings++ SiteSetting.refresh!+ Theme.expire_site_setting_cache!++ @counts+ end++ private++ def normalize_site_settings+ SiteSetting+ .where(data_type: SiteSettings::TypeSupervisor.types[:objects])+ .find_each do |setting|+ normalize_json_setting(+ setting,+ schema: site_setting_schema(setting.name),+ value: setting.value,+ label: "site setting #{setting.name}",+ ) { |normalized| setting.update!(value: JSON.generate(normalized)) }+ end+ end++ def normalize_theme_settings+ ThemeSetting+ .where(data_type: ThemeSetting.types[:objects])+ .includes(:theme)+ .find_each do |setting|+ normalize_json_setting(+ setting,+ schema: theme_setting_schema(setting),+ value: setting.json_value,+ label: "theme setting #{setting.theme.name}:#{setting.name}",+ ) { |normalized| setting.update!(json_value: normalized) }+ end+ end++ def normalize_json_setting(record, schema:, value:, label:)+ @counts[:processed] += 1++ if schema.blank?+ @counts[:skipped] += 1+ @output.puts("Skipping #{label}: schema not found")+ return+ end++ objects = parse_objects(value)+ if !objects.is_a?(Array)+ @counts[:invalid_json] += 1+ @output.puts("Skipping #{label}: value is not an objects array")+ return+ end++ normalized_objects = SchemaSettingsObjectValidator.normalize_uploads(schema:, objects:)++ if normalized_objects == objects+ @counts[:unchanged] += 1+ return+ end++ @counts[:changed] += 1+ @output.puts("#{@dry_run ? "Would normalize" : "Normalizing"} #{label}")+ yield normalized_objects if !@dry_run+ rescue JSON::ParserError+ @counts[:invalid_json] += 1+ @output.puts("Skipping #{label}: invalid JSON")+ rescue => error+ @counts[:errors] += 1+ @output.puts("Failed #{label}: #{error.class}: #{error.message}")+ end++ def parse_objects(value)+ value.is_a?(String) ? JSON.parse(value) : value+ end++ def site_setting_schema(name)+ SiteSetting.type_supervisor.type_hash(name.to_sym)[:schema]+ end++ def theme_setting_schema(setting)+ setting.theme.settings[setting.name.to_sym]&.schema+ end+ end+enddiff --git a/lib/site_settings/type_supervisor.rb b/lib/site_settings/type_supervisor.rbindex c1439d9e834..475d0d2d989 100644--- a/lib/site_settings/type_supervisor.rb+++ b/lib/site_settings/type_supervisor.rb@@ -346,6 +346,21 @@ class SiteSettings::TypeSupervisorval = val.is_a?(String) ? val : val.map(&:id).join("|")elsif type == self.class.types[:upload] && val.present?val = val.is_a?(Integer) ? val : val.id+ elsif type == self.class.types[:objects] && val.present?+ begin+ objects = val.is_a?(String) ? JSON.parse(val) : val++ if objects.is_a?(Array) && @schemas[name]+ val =+ JSON.generate(+ SchemaSettingsObjectValidator.normalize_uploads(+ schema: @schemas[name],+ objects: objects,+ ),+ )+ end+ rescue JSON::ParserError+ endend[val, type]diff --git a/lib/tasks/site_settings.rake b/lib/tasks/site_settings.rakeindex 78c8901d375..19136c41ae0 100644--- a/lib/tasks/site_settings.rake+++ b/lib/tasks/site_settings.rake@@ -1,6 +1,7 @@# frozen_string_literal: truerequire "yaml"+require_relative "../site_settings/object_uploads_normalizer"desc "Exports site settings"task "site_settings:export" => :environment do@@ -90,3 +91,24 @@ task "site_settings:find_dead" => :environment doputs "No dead settings found."endend++# TODO(gabriel): This was added as a fix for a bug in `type: object` settings.+# Some settings were storing the upload URLs instead of upload IDs.+# This rake task can be removed by May 2027+# Remove `spec/tasks/site_settings_spec.rb` too.+desc "Normalize upload URLs in objects site/theme settings to upload IDs"+task "site_settings:normalize_object_uploads" => :environment do+ dry_run = ENV["DRY_RUN"] == "1"+ counts = SiteSettings::ObjectUploadsNormalizer.new(dry_run:).normalize++ puts+ puts "Results:"+ puts " Processed: #{counts[:processed]}"+ puts " Changed: #{counts[:changed]}"+ puts " Unchanged: #{counts[:unchanged]}"+ puts " Skipped: #{counts[:skipped]}"+ puts " Invalid JSON: #{counts[:invalid_json]}"+ puts " Errors: #{counts[:errors]}"++ exit 1 if counts[:errors] > 0+enddiff --git a/lib/theme_settings_manager/objects.rb b/lib/theme_settings_manager/objects.rbindex ee71eb55944..c20f0600f3b 100644--- a/lib/theme_settings_manager/objects.rb+++ b/lib/theme_settings_manager/objects.rb@@ -5,13 +5,18 @@ class ThemeSettingsManager::Objects < ThemeSettingsManagerrow.json_valueend+ def default+ hydrate_uploads(@default.map(&:deep_stringify_keys))+ end+def value- has_record? ? db_record.json_value : default.map!(&:deep_stringify_keys)+ has_record? ? hydrate_uploads(db_record.json_value) : defaultenddef value=(objects)objects = JSON.parse(objects) if objects.is_a?(::String)ensure_is_valid_value!(objects)+ objects = SchemaSettingsObjectValidator.normalize_uploads(schema:, objects:)record = has_record? ? update_record!(json_value: objects) : create_record!(json_value: objects)theme.reloadrecord.json_value@@ -21,6 +26,10 @@ class ThemeSettingsManager::Objects < ThemeSettingsManager@opts[:schema]end+ def hydrate_uploads(objects)+ SchemaSettingsObjectValidator.hydrate_uploads(schema:, objects:, cdn: true)+ end+def categories(guardian)category_ids = Set.newdiff --git a/spec/models/site_setting_spec.rb b/spec/models/site_setting_spec.rbindex d11e545efae..a9116f3d53b 100644--- a/spec/models/site_setting_spec.rb+++ b/spec/models/site_setting_spec.rb@@ -351,6 +351,56 @@ RSpec.describe SiteSetting doexpect(upload_references.pluck(:upload_id)).to contain_exactly(upload.id, upload2.id)end+ it "stores object upload fields as upload IDs when set with upload URLs" do+ old_provider = SiteSetting.provider+ SiteSetting.provider = provider+ SiteSetting.refresh!++ begin+ SiteSetting.ui_cards_setting =+ JSON.generate([{ "title" => "Build a community", "image" => upload.url }])++ setting = provider.find("ui_cards_setting")+ expect(JSON.parse(setting.value)).to eq(+ [{ "title" => "Build a community", "image" => upload.id }],+ )+ ensure+ SiteSetting.provider = old_provider+ SiteSetting.refresh!+ end+ end++ it "hydrates object upload fields in client settings" do+ settings = new_settings(SiteSettings::LocalProcessProvider.new)+ settings.setting(+ :ui_cards_setting,+ "[]",+ type: :objects,+ client: true,+ schema: {+ name: "card",+ identifier: "title",+ properties: {+ title: {+ type: "string",+ required: true,+ },+ image: {+ type: "upload",+ },+ },+ },+ )++ settings.ui_cards_setting =+ JSON.generate([{ "title" => "Build a community", "image" => upload.url }])++ client_settings = JSON.parse(settings.client_settings_json_uncached)+ expect(JSON.parse(client_settings["ui_cards_setting"])).to eq(+ [{ "title" => "Build a community", "image" => upload.url }],+ )+ end+it "removes upload references when uploads are removed from objects" do# First save with two uploadsobjects_value =diff --git a/spec/models/theme_setting_spec.rb b/spec/models/theme_setting_spec.rbindex d543a0a9843..2f369a9f7f2 100644--- a/spec/models/theme_setting_spec.rb+++ b/spec/models/theme_setting_spec.rb@@ -20,6 +20,22 @@ RSpec.describe ThemeSetting dotype: uploadYAML+ let(:ui_cards_setting_yaml) { <<~YAML }+ ui_cards_setting:+ type: objects+ default:+ - title: "Build a community"+ schema:+ name: card+ identifier: title+ properties:+ title:+ type: string+ required: true+ image:+ type: upload+ YAML+it "creates upload references for type objects settings with upload fields" dotheme.set_field(target: :settings, name: "yaml", value: objects_setting_yaml)theme.save!@@ -34,6 +50,23 @@ RSpec.describe ThemeSetting doexpect(upload_references.pluck(:upload_id)).to contain_exactly(upload.id, upload2.id)end+ it "stores object upload fields as upload IDs when set with upload URLs" do+ theme.set_field(target: :settings, name: "yaml", value: ui_cards_setting_yaml)+ theme.save!++ theme.settings[:ui_cards_setting].value = [+ { "title" => "Build a community", "image" => upload.url },+ ]++ theme_setting = theme.theme_settings.find_by(name: "ui_cards_setting")+ expect(theme_setting.json_value).to eq(+ [{ "title" => "Build a community", "image" => upload.id }],+ )+ expect(theme.cached_settings[:ui_cards_setting].first["image"]).to eq(+ Discourse.store.cdn_url(upload.url),+ )+ end+it "destroys upload references for type objects setting when the setting is destroyed" dotheme.set_field(target: :settings, name: "yaml", value: objects_setting_yaml)theme.save!diff --git a/spec/support/sample_plugin_site_settings.yml b/spec/support/sample_plugin_site_settings.ymlindex d137a1e7f85..da66ef088fa 100644--- a/spec/support/sample_plugin_site_settings.yml+++ b/spec/support/sample_plugin_site_settings.yml@@ -24,3 +24,15 @@ site_settings:type: stringupload_id:type: upload+ ui_cards_setting:+ type: objects+ default: "[]"+ schema:+ name: card+ identifier: title+ properties:+ title:+ type: string+ required: true+ image:+ type: uploaddiff --git a/spec/tasks/site_settings_spec.rb b/spec/tasks/site_settings_spec.rbnew file mode 100644index 00000000000..f612b6aa08c--- /dev/null+++ b/spec/tasks/site_settings_spec.rb@@ -0,0 +1,72 @@+# frozen_string_literal: true++RSpec.describe "tasks/site_settings" do+ describe "site_settings:normalize_object_uploads" do+ fab!(:theme)+ fab!(:upload)++ it "normalizes upload URLs in object settings" do+ SiteSetting.create!(+ name: "ui_cards_setting",+ data_type: SiteSettings::TypeSupervisor.types[:objects],+ value: JSON.generate([{ "title" => "Site card", "image" => upload.url }]),+ )++ theme.set_field(target: :settings, name: "yaml", value: <<~YAML)+ cards:+ type: objects+ default: []+ schema:+ name: card+ identifier: title+ properties:+ title:+ type: string+ required: true+ image:+ type: upload+ YAML+ theme.save!++ ThemeSetting.create!(+ theme:,+ name: "cards",+ data_type: ThemeSetting.types[:objects],+ json_value: [{ "title" => "Theme card", "image" => upload.url }],+ )++ output = capture_stdout { invoke_rake_task("site_settings:normalize_object_uploads") }++ expect(output).to include("Changed: 2")+ expect(JSON.parse(SiteSetting.find_by(name: "ui_cards_setting").value)).to eq(+ [{ "title" => "Site card", "image" => upload.id }],+ )+ expect(ThemeSetting.find_by(name: "cards").json_value).to eq(+ [{ "title" => "Theme card", "image" => upload.id }],+ )+ end++ it "does not persist changes in dry run mode" do+ SiteSetting.create!(+ name: "ui_cards_setting",+ data_type: SiteSettings::TypeSupervisor.types[:objects],+ value: JSON.generate([{ "title" => "Site card", "image" => upload.url }]),+ )++ previous_dry_run = ENV["DRY_RUN"]+ ENV["DRY_RUN"] = "1"+ begin+ output = capture_stdout { invoke_rake_task("site_settings:normalize_object_uploads") }++ expect(output).to include("Would normalize site setting ui_cards_setting")+ expect(output).to include("Changed: 1")+ ensure+ ENV["DRY_RUN"] = previous_dry_run+ end++ expect(JSON.parse(SiteSetting.find_by(name: "ui_cards_setting").value)).to eq(+ [{ "title" => "Site card", "image" => upload.url }],+ )+ end+ end+end