Skip to content

A platform for community discussion. Free, open, simple.

FIX: `type: objects` uploads should be stored as IDs (#40178)

Gabriel Grubbae52b385ae1ab
diff --git a/app/models/site_setting.rb b/app/models/site_setting.rb
index 5a4b206cc5c..945ba9b5aba 100644
--- a/app/models/site_setting.rb
+++ b/app/models/site_setting.rb
@@ -93,24 +93,12 @@ class SiteSetting < ActiveRecord::Base
upload_ids = value.split("|").compact.uniq
UploadReference.ensure_exist!(upload_ids: upload_ids, target: self)
elsif data_type == SiteSettings::TypeSupervisor.types[:objects] && value.present?
- upload_values =
- SchemaSettingsObjectValidator.property_values_of_type(
+ upload_ids =
+ SchemaSettingsObjectValidator.upload_ids(
schema: SiteSetting.type_supervisor.type_hash(name.to_sym)[:schema],
objects: JSON.parse(value),
- type: "upload",
)
- # Convert URLs to upload IDs (values can be either integer IDs or URL strings)
- upload_ids =
- upload_values.filter_map do |value|
- if value.is_a?(Integer)
- value
- elsif value.is_a?(String) && value.present?
- ::Upload.get_from_url(value)&.id
- end
- end
-
- # Always call ensure_exist! to clean up old references when uploads are removed
UploadReference.ensure_exist!(upload_ids: upload_ids, target: self)
end
end
diff --git a/app/models/theme_setting.rb b/app/models/theme_setting.rb
index 651e3a43f1e..38dcfaa0d9e 100644
--- a/app/models/theme_setting.rb
+++ b/app/models/theme_setting.rb
@@ -23,30 +23,18 @@ class ThemeSetting < ActiveRecord::Base
UploadReference.ensure_exist!(upload_ids: [value], target: self)
elsif data_type == ThemeSetting.types[:objects] && saved_change_to_json_value? &&
json_value.present?
- upload_values =
- SchemaSettingsObjectValidator.property_values_of_type(
+ upload_ids =
+ SchemaSettingsObjectValidator.upload_ids(
schema: theme.settings[name.to_sym].schema,
objects: json_value,
- type: "upload",
)
- # Convert URLs to upload IDs (values can be either integer IDs or URL strings)
- upload_ids =
- upload_values.filter_map do |value|
- if value.is_a?(Integer)
- value
- elsif value.is_a?(String) && value.present?
- Upload.get_from_url(value)&.id
- end
- end
-
- # Always call ensure_exist! to clean up old references when uploads are removed
UploadReference.ensure_exist!(upload_ids: upload_ids, target: self)
end
if theme.theme_modifier_set.refresh_theme_setting_modifiers(
target_setting_name: name,
- target_setting_value: self.value,
+ target_setting_value: value,
)
theme.theme_modifier_set.save!
end
diff --git a/lib/schema_settings_object_validator.rb b/lib/schema_settings_object_validator.rb
index fad4e605f2e..cc4f7b2c82b 100644
--- a/lib/schema_settings_object_validator.rb
+++ b/lib/schema_settings_object_validator.rb
@@ -26,8 +26,84 @@ class SchemaSettingsObjectValidator
values.to_a
end
+ def upload_ids(schema:, objects:)
+ property_values_of_type(schema:, objects:, type: "upload").filter_map do |value|
+ # TODO(gabriel): this if branch was to deal wit ha legacy logic bug where some upload fields were stored as URLs instead of IDs.
+ # This can be removed by May 2027.
+ if value.is_a?(Integer)
+ value
+ elsif value.is_a?(String) && value.present?
+ Upload.get_from_url(value)&.id
+ end
+ end
+ end
+
+ def normalize_uploads(schema:, objects:)
+ return objects if objects.blank?
+
+ transform_uploads_in_objects(objects.deep_dup, schema[:properties]) do |value|
+ if value.is_a?(String) && value.present?
+ Upload.get_from_url(value)&.id || value
+ else
+ value
+ end
+ end
+ end
+
+ def hydrate_uploads(schema:, objects:, cdn: false)
+ return objects if objects.blank?
+
+ upload_ids =
+ property_values_of_type(schema: schema, objects: objects, type: "upload").select do |value|
+ value.is_a?(Integer)
+ end
+ return objects if upload_ids.empty?
+
+ uploads_by_id = Upload.where(id: upload_ids).index_by(&:id)
+ transform_uploads_in_objects(objects.deep_dup, schema[:properties]) do |value|
+ if upload = uploads_by_id[value]
+ cdn ? Discourse.store.cdn_url(upload.url) : upload.url
+ else
+ value
+ end
+ end
+ end
+
private
+ def transform_uploads_in_objects(objects, properties, &block)
+ objects.each do |object|
+ properties.each do |property_name, property_attributes|
+ key = object_key(object, property_name)
+ next if key.nil?
+
+ case property_attributes[:type]
+ when "upload"
+ object[key] = block.call(object[key])
+ when "objects"
+ nested_objects = object[key]
+ if nested_objects.is_a?(Array)
+ transform_uploads_in_objects(
+ nested_objects,
+ property_attributes[:schema][:properties],
+ &block
+ )
+ end
+ end
+ end
+ end
+
+ objects
+ end
+
+ def object_key(object, property_name)
+ string_key = property_name.to_s
+ return string_key if object.key?(string_key)
+
+ symbol_key = property_name.to_sym
+ symbol_key if object.key?(symbol_key)
+ end
+
def humanize_error_messages(errors, index:, error_messages:)
errors.each do |property_json_pointer, error_details|
error_messages.push(*error_details.humanize_messages("/#{index}#{property_json_pointer}"))
diff --git a/lib/site_setting_extension.rb b/lib/site_setting_extension.rb
index e9f16146607..c93fbde9b01 100644
--- a/lib/site_setting_extension.rb
+++ b/lib/site_setting_extension.rb
@@ -307,6 +307,11 @@ module SiteSettingExtension
value = value.to_s
elsif type == :uploaded_image_list && value.present?
value = value.map(&:to_s).join("|")
+ elsif type == :objects && value.present?
+ type_hash = type_supervisor.type_hash(name)
+ if type_hash[:schema]
+ value = hydrate_objects_setting_value(name, value, type_hash:).to_json
+ end
end
[name, value]
@@ -456,8 +461,7 @@ module SiteSettingExtension
# For uploads nested in objects type, hydrate upload IDs to URLs
if type_hash[:type].to_s == "objects" && type_hash[:schema]
- parsed_value = JSON.parse(value)
- value = hydrate_uploads_in_objects(parsed_value, type_hash[:schema])
+ value = hydrate_objects_setting_value(s, value, type_hash:)
end
opts = {
@@ -1382,38 +1386,11 @@ module SiteSettingExtension
private
- def hydrate_uploads_in_objects(objects, schema)
- return objects if objects.blank?
-
- upload_ids =
- SchemaSettingsObjectValidator.property_values_of_type(
- schema: schema,
- objects: objects,
- type: "upload",
- )
-
- uploads_by_id = Upload.where(id: upload_ids).index_by(&:id)
- objects.map { |obj| hydrate_uploads_in_object(obj, schema[:properties], uploads_by_id) }
- end
-
- def hydrate_uploads_in_object(object, properties, uploads_by_id)
- properties.each do |prop_key, prop_value|
- case prop_value[:type]
- when "upload"
- key = prop_key.to_s
- upload_id = object[key]
- upload = uploads_by_id[upload_id]
- object[key] = upload.url if upload
- when "objects"
- nested_objects = object[prop_key.to_s]
- if nested_objects.is_a?(Array)
- nested_objects.each do |nested_obj|
- hydrate_uploads_in_object(nested_obj, prop_value[:schema][:properties], uploads_by_id)
- end
- end
- end
- end
+ def hydrate_objects_setting_value(name, value, type_hash: nil)
+ type_hash ||= type_supervisor.type_hash(name)
+ return value if !type_hash[:schema] || value.blank?
- object
+ objects = value.is_a?(String) ? JSON.parse(value) : value
+ SchemaSettingsObjectValidator.hydrate_uploads(objects:, schema: type_hash[:schema])
end
end
diff --git a/lib/site_settings/object_uploads_normalizer.rb b/lib/site_settings/object_uploads_normalizer.rb
new file mode 100644
index 00000000000..fa183e04259
--- /dev/null
+++ b/lib/site_settings/object_uploads_normalizer.rb
@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+# TODO(gabriel): This was added as a fix for a bug in `type: object` settings.
+# This is only used by a rake task. This class can be removed by May 2027
+
+module SiteSettings
+ class ObjectUploadsNormalizer
+ def initialize(dry_run: false, output: $stdout)
+ @dry_run = dry_run
+ @output = output
+ @counts = Hash.new(0)
+ end
+
+ def normalize
+ normalize_site_settings
+ normalize_theme_settings
+
+ SiteSetting.refresh!
+ Theme.expire_site_setting_cache!
+
+ @counts
+ end
+
+ private
+
+ def normalize_site_settings
+ SiteSetting
+ .where(data_type: SiteSettings::TypeSupervisor.types[:objects])
+ .find_each do |setting|
+ normalize_json_setting(
+ setting,
+ schema: site_setting_schema(setting.name),
+ value: setting.value,
+ label: "site setting #{setting.name}",
+ ) { |normalized| setting.update!(value: JSON.generate(normalized)) }
+ end
+ end
+
+ def normalize_theme_settings
+ ThemeSetting
+ .where(data_type: ThemeSetting.types[:objects])
+ .includes(:theme)
+ .find_each do |setting|
+ normalize_json_setting(
+ setting,
+ schema: theme_setting_schema(setting),
+ value: setting.json_value,
+ label: "theme setting #{setting.theme.name}:#{setting.name}",
+ ) { |normalized| setting.update!(json_value: normalized) }
+ end
+ end
+
+ def normalize_json_setting(record, schema:, value:, label:)
+ @counts[:processed] += 1
+
+ if schema.blank?
+ @counts[:skipped] += 1
+ @output.puts("Skipping #{label}: schema not found")
+ return
+ end
+
+ objects = parse_objects(value)
+ if !objects.is_a?(Array)
+ @counts[:invalid_json] += 1
+ @output.puts("Skipping #{label}: value is not an objects array")
+ return
+ end
+
+ normalized_objects = SchemaSettingsObjectValidator.normalize_uploads(schema:, objects:)
+
+ if normalized_objects == objects
+ @counts[:unchanged] += 1
+ return
+ end
+
+ @counts[:changed] += 1
+ @output.puts("#{@dry_run ? "Would normalize" : "Normalizing"} #{label}")
+ yield normalized_objects if !@dry_run
+ rescue JSON::ParserError
+ @counts[:invalid_json] += 1
+ @output.puts("Skipping #{label}: invalid JSON")
+ rescue => error
+ @counts[:errors] += 1
+ @output.puts("Failed #{label}: #{error.class}: #{error.message}")
+ end
+
+ def parse_objects(value)
+ value.is_a?(String) ? JSON.parse(value) : value
+ end
+
+ def site_setting_schema(name)
+ SiteSetting.type_supervisor.type_hash(name.to_sym)[:schema]
+ end
+
+ def theme_setting_schema(setting)
+ setting.theme.settings[setting.name.to_sym]&.schema
+ end
+ end
+end
diff --git a/lib/site_settings/type_supervisor.rb b/lib/site_settings/type_supervisor.rb
index c1439d9e834..475d0d2d989 100644
--- a/lib/site_settings/type_supervisor.rb
+++ b/lib/site_settings/type_supervisor.rb
@@ -346,6 +346,21 @@ class SiteSettings::TypeSupervisor
val = val.is_a?(String) ? val : val.map(&:id).join("|")
elsif type == self.class.types[:upload] && val.present?
val = val.is_a?(Integer) ? val : val.id
+ elsif type == self.class.types[:objects] && val.present?
+ begin
+ objects = val.is_a?(String) ? JSON.parse(val) : val
+
+ if objects.is_a?(Array) && @schemas[name]
+ val =
+ JSON.generate(
+ SchemaSettingsObjectValidator.normalize_uploads(
+ schema: @schemas[name],
+ objects: objects,
+ ),
+ )
+ end
+ rescue JSON::ParserError
+ end
end
[val, type]
diff --git a/lib/tasks/site_settings.rake b/lib/tasks/site_settings.rake
index 78c8901d375..19136c41ae0 100644
--- a/lib/tasks/site_settings.rake
+++ b/lib/tasks/site_settings.rake
@@ -1,6 +1,7 @@
# frozen_string_literal: true
require "yaml"
+require_relative "../site_settings/object_uploads_normalizer"
desc "Exports site settings"
task "site_settings:export" => :environment do
@@ -90,3 +91,24 @@ task "site_settings:find_dead" => :environment do
puts "No dead settings found."
end
end
+
+# TODO(gabriel): This was added as a fix for a bug in `type: object` settings.
+# Some settings were storing the upload URLs instead of upload IDs.
+# This rake task can be removed by May 2027
+# Remove `spec/tasks/site_settings_spec.rb` too.
+desc "Normalize upload URLs in objects site/theme settings to upload IDs"
+task "site_settings:normalize_object_uploads" => :environment do
+ dry_run = ENV["DRY_RUN"] == "1"
+ counts = SiteSettings::ObjectUploadsNormalizer.new(dry_run:).normalize
+
+ puts
+ puts "Results:"
+ puts " Processed: #{counts[:processed]}"
+ puts " Changed: #{counts[:changed]}"
+ puts " Unchanged: #{counts[:unchanged]}"
+ puts " Skipped: #{counts[:skipped]}"
+ puts " Invalid JSON: #{counts[:invalid_json]}"
+ puts " Errors: #{counts[:errors]}"
+
+ exit 1 if counts[:errors] > 0
+end
diff --git a/lib/theme_settings_manager/objects.rb b/lib/theme_settings_manager/objects.rb
index ee71eb55944..c20f0600f3b 100644
--- a/lib/theme_settings_manager/objects.rb
+++ b/lib/theme_settings_manager/objects.rb
@@ -5,13 +5,18 @@ class ThemeSettingsManager::Objects < ThemeSettingsManager
row.json_value
end
+ def default
+ hydrate_uploads(@default.map(&:deep_stringify_keys))
+ end
+
def value
- has_record? ? db_record.json_value : default.map!(&:deep_stringify_keys)
+ has_record? ? hydrate_uploads(db_record.json_value) : default
end
def value=(objects)
objects = JSON.parse(objects) if objects.is_a?(::String)
ensure_is_valid_value!(objects)
+ objects = SchemaSettingsObjectValidator.normalize_uploads(schema:, objects:)
record = has_record? ? update_record!(json_value: objects) : create_record!(json_value: objects)
theme.reload
record.json_value
@@ -21,6 +26,10 @@ class ThemeSettingsManager::Objects < ThemeSettingsManager
@opts[:schema]
end
+ def hydrate_uploads(objects)
+ SchemaSettingsObjectValidator.hydrate_uploads(schema:, objects:, cdn: true)
+ end
+
def categories(guardian)
category_ids = Set.new
diff --git a/spec/models/site_setting_spec.rb b/spec/models/site_setting_spec.rb
index d11e545efae..a9116f3d53b 100644
--- a/spec/models/site_setting_spec.rb
+++ b/spec/models/site_setting_spec.rb
@@ -351,6 +351,56 @@ RSpec.describe SiteSetting do
expect(upload_references.pluck(:upload_id)).to contain_exactly(upload.id, upload2.id)
end
+ it "stores object upload fields as upload IDs when set with upload URLs" do
+ old_provider = SiteSetting.provider
+ SiteSetting.provider = provider
+ SiteSetting.refresh!
+
+ begin
+ SiteSetting.ui_cards_setting =
+ JSON.generate([{ "title" => "Build a community", "image" => upload.url }])
+
+ setting = provider.find("ui_cards_setting")
+ expect(JSON.parse(setting.value)).to eq(
+ [{ "title" => "Build a community", "image" => upload.id }],
+ )
+ ensure
+ SiteSetting.provider = old_provider
+ SiteSetting.refresh!
+ end
+ end
+
+ it "hydrates object upload fields in client settings" do
+ settings = new_settings(SiteSettings::LocalProcessProvider.new)
+ settings.setting(
+ :ui_cards_setting,
+ "[]",
+ type: :objects,
+ client: true,
+ schema: {
+ name: "card",
+ identifier: "title",
+ properties: {
+ title: {
+ type: "string",
+ required: true,
+ },
+ image: {
+ type: "upload",
+ },
+ },
+ },
+ )
+
+ settings.ui_cards_setting =
+ JSON.generate([{ "title" => "Build a community", "image" => upload.url }])
+
+ client_settings = JSON.parse(settings.client_settings_json_uncached)
+ expect(JSON.parse(client_settings["ui_cards_setting"])).to eq(
+ [{ "title" => "Build a community", "image" => upload.url }],
+ )
+ end
+
it "removes upload references when uploads are removed from objects" do
# First save with two uploads
objects_value =
diff --git a/spec/models/theme_setting_spec.rb b/spec/models/theme_setting_spec.rb
index d543a0a9843..2f369a9f7f2 100644
--- a/spec/models/theme_setting_spec.rb
+++ b/spec/models/theme_setting_spec.rb
@@ -20,6 +20,22 @@ RSpec.describe ThemeSetting do
type: upload
YAML
+ let(:ui_cards_setting_yaml) { <<~YAML }
+ ui_cards_setting:
+ type: objects
+ default:
+ - title: "Build a community"
+ schema:
+ name: card
+ identifier: title
+ properties:
+ title:
+ type: string
+ required: true
+ image:
+ type: upload
+ YAML
+
it "creates upload references for type objects settings with upload fields" do
theme.set_field(target: :settings, name: "yaml", value: objects_setting_yaml)
theme.save!
@@ -34,6 +50,23 @@ RSpec.describe ThemeSetting do
expect(upload_references.pluck(:upload_id)).to contain_exactly(upload.id, upload2.id)
end
+ it "stores object upload fields as upload IDs when set with upload URLs" do
+ theme.set_field(target: :settings, name: "yaml", value: ui_cards_setting_yaml)
+ theme.save!
+
+ theme.settings[:ui_cards_setting].value = [
+ { "title" => "Build a community", "image" => upload.url },
+ ]
+
+ theme_setting = theme.theme_settings.find_by(name: "ui_cards_setting")
+ expect(theme_setting.json_value).to eq(
+ [{ "title" => "Build a community", "image" => upload.id }],
+ )
+ expect(theme.cached_settings[:ui_cards_setting].first["image"]).to eq(
+ Discourse.store.cdn_url(upload.url),
+ )
+ end
+
it "destroys upload references for type objects setting when the setting is destroyed" do
theme.set_field(target: :settings, name: "yaml", value: objects_setting_yaml)
theme.save!
diff --git a/spec/support/sample_plugin_site_settings.yml b/spec/support/sample_plugin_site_settings.yml
index d137a1e7f85..da66ef088fa 100644
--- a/spec/support/sample_plugin_site_settings.yml
+++ b/spec/support/sample_plugin_site_settings.yml
@@ -24,3 +24,15 @@ site_settings:
type: string
upload_id:
type: upload
+ ui_cards_setting:
+ type: objects
+ default: "[]"
+ schema:
+ name: card
+ identifier: title
+ properties:
+ title:
+ type: string
+ required: true
+ image:
+ type: upload
diff --git a/spec/tasks/site_settings_spec.rb b/spec/tasks/site_settings_spec.rb
new file mode 100644
index 00000000000..f612b6aa08c
--- /dev/null
+++ b/spec/tasks/site_settings_spec.rb
@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+RSpec.describe "tasks/site_settings" do
+ describe "site_settings:normalize_object_uploads" do
+ fab!(:theme)
+ fab!(:upload)
+
+ it "normalizes upload URLs in object settings" do
+ SiteSetting.create!(
+ name: "ui_cards_setting",
+ data_type: SiteSettings::TypeSupervisor.types[:objects],
+ value: JSON.generate([{ "title" => "Site card", "image" => upload.url }]),
+ )
+
+ theme.set_field(target: :settings, name: "yaml", value: <<~YAML)
+ cards:
+ type: objects
+ default: []
+ schema:
+ name: card
+ identifier: title
+ properties:
+ title:
+ type: string
+ required: true
+ image:
+ type: upload
+ YAML
+ theme.save!
+
+ ThemeSetting.create!(
+ theme:,
+ name: "cards",
+ data_type: ThemeSetting.types[:objects],
+ json_value: [{ "title" => "Theme card", "image" => upload.url }],
+ )
+
+ output = capture_stdout { invoke_rake_task("site_settings:normalize_object_uploads") }
+
+ expect(output).to include("Changed: 2")
+ expect(JSON.parse(SiteSetting.find_by(name: "ui_cards_setting").value)).to eq(
+ [{ "title" => "Site card", "image" => upload.id }],
+ )
+ expect(ThemeSetting.find_by(name: "cards").json_value).to eq(
+ [{ "title" => "Theme card", "image" => upload.id }],
+ )
+ end
+
+ it "does not persist changes in dry run mode" do
+ SiteSetting.create!(
+ name: "ui_cards_setting",
+ data_type: SiteSettings::TypeSupervisor.types[:objects],
+ value: JSON.generate([{ "title" => "Site card", "image" => upload.url }]),
+ )
+
+ previous_dry_run = ENV["DRY_RUN"]
+ ENV["DRY_RUN"] = "1"
+ begin
+ output = capture_stdout { invoke_rake_task("site_settings:normalize_object_uploads") }
+
+ expect(output).to include("Would normalize site setting ui_cards_setting")
+ expect(output).to include("Changed: 1")
+ ensure
+ ENV["DRY_RUN"] = previous_dry_run
+ end
+
+ expect(JSON.parse(SiteSetting.find_by(name: "ui_cards_setting").value)).to eq(
+ [{ "title" => "Site card", "image" => upload.url }],
+ )
+ end
+ end
+end