Skip to content

A platform for community discussion. Free, open, simple.

MT: Add `disco check` — a single entrypoint for all schema and converter checks

Gerhard Schlager15be00530a41
diff --git a/.github/workflows/migration-tests.yml b/.github/workflows/migration-tests.yml
index 2c62476b30e..bd9a6a09578 100644
--- a/.github/workflows/migration-tests.yml
+++ b/.github/workflows/migration-tests.yml
@@ -43,6 +43,15 @@ jobs:
(cd "migrations/$gem" && bundle install --jobs $(($(nproc) - 1)) && bundle exec rspec --tag ~rails)
done
+ - name: Converter coverage (no Rails)
+ run: |
+ bundle config set --local with migrations
+ bundle install --jobs $(($(nproc) - 1))
+ # Fails if the reference (Discourse) converter leaves an IntermediateDB
+ # column unwritten, or writes columns or models that don't exist in the
+ # schema. Runs without Rails or a source database.
+ migrations/bin/disco check coverage
+
tests:
if: github.event_name == 'pull_request' || github.repository != 'discourse/discourse-private-mirror'
name: Tests
@@ -99,39 +108,13 @@ jobs:
bin/rake db:create
script/silence_successful_output bin/rake db:migrate
- - name: Validate IntermediateDB schema
+ - name: Run all schema and converter checks (disco check)
run: |
- # Show structured differences between DSL config and database
- migrations/bin/disco schema diff --db intermediate_db || true
-
- # Regenerate schema from DSL config
- migrations/bin/disco schema generate --db intermediate_db
-
- # Check if generated output matches what's committed
- if [ ! -z "$(git status --porcelain migrations/core/db/intermediate_db_schema/ migrations/core/lib/migrations/database/intermediate_db/)" ]; then
- echo ""
- echo "=========================================="
- echo "IntermediateDB schema is not up to date."
- echo "=========================================="
- echo ""
- echo "The generated schema files differ from what is committed."
- echo "This usually means the schema DSL config or the database"
- echo "structure changed without regenerating."
- echo ""
- echo "To fix:"
- echo " 1. Run: migrations/bin/disco schema diff"
- echo " to see what changed between config and database"
- echo " 2. Update config files in migrations/tooling/config/schema/intermediate_db/"
- echo " - New tables: migrations/bin/disco schema add <table>"
- echo " - Ignore tables: migrations/bin/disco schema ignore <table> --reason '...'"
- echo " - New columns: add to the table's include list or ignore with reason"
- echo " 3. Run: migrations/bin/disco schema generate"
- echo " 4. Commit the updated config and generated files"
- echo ""
- echo "Generated file diff:"
- git -c color.ui=always diff -- migrations/core/db/intermediate_db_schema/ migrations/core/lib/migrations/database/intermediate_db/
- exit 1
- fi
+ # Runs every check in dependency order: pending migrations, schema
+ # config validity, config ↔ database drift, committed generated files
+ # vs regeneration (in a temp directory), and converter coverage.
+ # Prints the actionable differences and fix commands on failure.
+ migrations/bin/disco check
- name: RSpec (Rails integration)
env:
diff --git a/Gemfile.lock b/Gemfile.lock
index aaf4ecf4f35..314c6354a12 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1011,7 +1011,7 @@ CHECKSUMS
bootsnap (1.24.5) sha256=36b677448524d279b470469aabd5dff4a980e3fa4931a0df68da4a500eb1b6c4
builder (3.3.0) sha256=497918d2f9dca528fdca4b88d84e4ef4387256d984b8154e9d5d3fe5a9c8835f
bullet (8.1.3) sha256=c8163c527324fe1180775be48719875726891622f0a73784658b2992acf3c7cb
- bundler (4.0.11) sha256=5bcec0fb78302e48d02ee46f10ee6e6942be647ba5b44a6d1ddfda9a240ce785
+ bundler (4.0.14) sha256=d09a0a965cf772266a7e49e83610be7c2f4e49e61134c42a56804bb383cc24b8
capybara (3.40.0) sha256=42dba720578ea1ca65fd7a41d163dd368502c191804558f6e0f71b391054aeef
capybara-playwright-driver (0.5.9) sha256=4c17fed20817b7e1bde2cfc8186e7bf5cd72017ce1aa695e35130f8e600e7282
cbor (0.5.10.2) sha256=df5104f7a62c881123e6505441b1e276208be1771540c2cc3b1de8a210a7c52c
@@ -1336,4 +1336,4 @@ RUBY VERSION
ruby 3.4.7p58
BUNDLED WITH
- 4.0.11
+ 4.0.14
diff --git a/migrations/AGENTS.md b/migrations/AGENTS.md
index d94afe3a039..3720ec210e0 100644
--- a/migrations/AGENTS.md
+++ b/migrations/AGENTS.md
@@ -5,5 +5,26 @@ project: gem layout and namespaces, the `disco` CLI, converters, the schema DSL,
and the install / test / lint workflow.
This file is reserved for **agent-specific** guidance that does not belong in the
-README (conventions, gotchas, do/don't notes for automated contributors). There is
-none yet; add it here as it comes up.
+README (conventions, gotchas, do/don't notes for automated contributors).
+
+## Gotchas
+
+- **Samovar reserves `name` on commands.** `Nested#parse` instantiates a
+ sub-command with `name:` (its invocation name), which Samovar stores and exposes
+ as `name`. So don't declare a positional `one :name` on a `disco` command — when
+ the argument is omitted the accessor silently reads back the command's own name
+ (e.g. a `schema add` command would read back `"add"`) instead of `nil`. Name the
+ positional something else (`one :table_name, …`).
+
+- **Samovar positionals are never required.** Don't use `one :x, required: true` —
+ it raises during parsing, before `call` runs, which breaks the `-h/--help`
+ handling. Leave positionals optional and validate them at the top of `call` with
+ `require_positional!` (see `Migrations::CLI::Command`); it raises a presentable
+ error, so the user gets a clean message instead of a backtrace.
+
+- **Don't give command groups a `-h/--help` option.** The option hoisting in
+ `Command#parse` moves recognized flags to the front, so a group-level help
+ option steals `--help` from the subcommands — `group sub --help` would run
+ the subcommand instead of printing its help. Leave groups without options;
+ a bare `group --help` surfaces as an unparsable token, which `Bootstrap`
+ turns into usage with exit 0.
diff --git a/migrations/converters/lib/migrations/converters/cli/convert_command.rb b/migrations/converters/lib/migrations/converters/cli/convert_command.rb
index 8364909e47d..448fdbf724d 100644
--- a/migrations/converters/lib/migrations/converters/cli/convert_command.rb
+++ b/migrations/converters/lib/migrations/converters/cli/convert_command.rb
@@ -29,7 +29,12 @@ module Migrations
def call
return print_usage if @options[:help]
- type = converter_type.downcase
+ type =
+ require_positional!(
+ converter_type,
+ "converter_type",
+ hint: valid_names_message,
+ ).downcase
validate_converter_type!(type)
settings = load_settings(type)
@@ -43,15 +48,18 @@ module Migrations
private
def validate_converter_type!(type)
- names = Converters.names
- return if names.include?(type)
+ return if Converters.names.include?(type)
raise Error, <<~MSG
Unknown converter name: #{type}
- Valid names are: #{names.join(", ")}
+ #{valid_names_message}
MSG
end
+ def valid_names_message
+ "Valid names are: #{Converters.names.join(", ")}"
+ end
+
def load_settings(type)
settings_path = @options[:settings] || Converters.default_settings_path(type)
settings_path = File.expand_path(settings_path, Dir.pwd)
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/categories.rb b/migrations/converters/lib/migrations/converters/discourse/steps/categories.rb
index a8b11ae1c8b..401fee4eda3 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/categories.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/categories.rb
@@ -87,6 +87,7 @@ module Migrations
emoji: item[:emoji],
existing_id: item[:existing_id],
icon: item[:icon],
+ locale: item[:locale],
mailinglist_mirror: item[:mailinglist_mirror],
minimum_required_tags: item[:minimum_required_tags],
name: item[:name],
@@ -96,6 +97,7 @@ module Migrations
position: item[:position],
read_only_banner: item[:read_only_banner],
read_restricted: item[:read_restricted],
+ search_priority: item[:search_priority],
show_subcategory_list: item[:show_subcategory_list],
slug: item[:slug],
sort_ascending: item[:sort_ascending],
@@ -106,6 +108,7 @@ module Migrations
topic_featured_link_allowed: item[:topic_featured_link_allowed],
topic_id: item[:topic_id],
topic_template: item[:topic_template],
+ topic_title_placeholder: item[:topic_title_placeholder],
uploaded_background_dark_id: @background_dark_upload_creator.create_for(item),
uploaded_background_id: @background_upload_creator.create_for(item),
uploaded_logo_dark_id: @logo_dark_upload_creator.create_for(item),
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/groups.rb
index 1ba23bd83b2..1c3d891f165 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/groups.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/groups.rb
@@ -48,6 +48,7 @@ module Migrations
flair_upload_id: @flair_upload_creator.create_for(item),
full_name: item[:full_name],
grant_trust_level: item[:grant_trust_level],
+ incoming_email: item[:incoming_email],
members_visibility_level: item[:members_visibility_level],
membership_request_template: item[:membership_request_template],
mentionable_level: item[:mentionable_level],
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/tags.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tags.rb
index 1e8490612f9..d7838c3d40a 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/tags.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tags.rb
@@ -24,6 +24,8 @@ module Migrations
created_at: item[:created_at],
description: item[:description],
name: item[:name],
+ locale: item[:locale],
+ slug: item[:slug],
)
if item[:target_tag_id]
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb
index a61b225c6b9..473f860cddb 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb
@@ -31,9 +31,11 @@ module Migrations
deleted_by_id: item[:deleted_by_id],
external_id: item[:external_id],
featured_link: item[:featured_link],
+ locale: item[:locale],
pinned_at: item[:pinned_at],
pinned_globally: item[:pinned_globally],
pinned_until: item[:pinned_until],
+ slow_mode_seconds: item[:slow_mode_seconds],
subtype: item[:subtype],
title: item[:title],
user_id: item[:user_id],
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_custom_fields.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_custom_fields.rb
new file mode 100644
index 00000000000..7944dca2127
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_custom_fields.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ class UserCustomFields < Conversion::ProgressStep
+ attr_accessor :source_db
+
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*)
+ FROM (
+ SELECT 1
+ FROM user_custom_fields
+ WHERE user_id > 0
+ AND value IS NOT NULL
+ AND name NOT LIKE '#{UserFieldValues::USER_FIELD_LIKE_PATTERN}' ESCAPE '\\'
+ GROUP BY user_id, name, value
+ ) custom_fields
+ SQL
+ end
+
+ def items
+ # User field values are converted by the `UserFieldValues` step.
+ # The GROUP BY drops exact duplicates because the IntermediateDB
+ # uses (user_id, name, value) as primary key.
+ @source_db.query <<~SQL
+ SELECT user_id, name, value, MIN(created_at) AS created_at
+ FROM user_custom_fields
+ WHERE user_id > 0
+ AND value IS NOT NULL
+ AND name NOT LIKE '#{UserFieldValues::USER_FIELD_LIKE_PATTERN}' ESCAPE '\\'
+ GROUP BY user_id, name, value
+ ORDER BY user_id, name, value
+ SQL
+ end
+
+ def process_item(item)
+ IntermediateDB::UserCustomField.create(
+ user_id: item[:user_id],
+ name: item[:name],
+ value: item[:value],
+ created_at: item[:created_at],
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rb
index 78b96458f32..801caa2510a 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rb
@@ -6,6 +6,12 @@ module Migrations
class UserFieldValues < Conversion::ProgressStep
USER_FIELD_PREFIX = "user_field_"
+ # `USER_FIELD_PREFIX` as a `LIKE` pattern: the `_` characters are literal
+ # here, not single-character wildcards, so they're escaped for use with
+ # an `ESCAPE '\'` clause. This keeps the `user_field_*` split between
+ # this step and `UserCustomFields` exact and complementary.
+ USER_FIELD_LIKE_PATTERN = "#{USER_FIELD_PREFIX.gsub("_") { '\_' }}%"
+
attr_accessor :source_db
def max_progress
@@ -13,7 +19,7 @@ module Migrations
SELECT COUNT(*)
FROM user_custom_fields
WHERE user_id > 0
- AND name LIKE '#{USER_FIELD_PREFIX}%'
+ AND name LIKE '#{USER_FIELD_LIKE_PATTERN}' ESCAPE '\\'
SQL
end
@@ -24,7 +30,7 @@ module Migrations
(COUNT(*) OVER (PARTITION BY user_id, name) > 1) AS is_multiselect_field
FROM user_custom_fields
WHERE user_id > 0
- AND name LIKE '#{USER_FIELD_PREFIX}%'
+ AND name LIKE '#{USER_FIELD_LIKE_PATTERN}' ESCAPE '\\'
ORDER BY user_id, name
SQL
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rb
index 74b05ba4b9c..46344a25de8 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rb
@@ -34,6 +34,7 @@ module Migrations
requirement: item[:requirement],
searchable: item[:searchable],
show_on_profile: item[:show_on_profile],
+ show_on_signup: item[:show_on_signup],
show_on_user_card: item[:show_on_user_card],
)
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb
index 5c8fc591a46..9f2d6168707 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb
@@ -26,25 +26,15 @@ module Migrations
def process_item(item)
IntermediateDB::UserOption.create(
user_id: item[:user_id],
- ai_search_discoveries: item[:ai_search_discoveries],
allow_private_messages: item[:allow_private_messages],
- auto_image_caption: item[:auto_image_caption],
auto_track_topics_after_msecs: item[:auto_track_topics_after_msecs],
automatically_unpin_topics: item[:automatically_unpin_topics],
bookmark_auto_delete_preference: item[:bookmark_auto_delete_preference],
- chat_email_frequency: item[:chat_email_frequency],
- chat_enabled: item[:chat_enabled],
- chat_header_indicator_preference: item[:chat_header_indicator_preference],
- chat_send_shortcut: item[:chat_send_shortcut],
- chat_separate_sidebar_mode: item[:chat_separate_sidebar_mode],
- chat_sound: item[:chat_sound],
color_scheme_id: item[:color_scheme_id],
composition_mode: item[:composition_mode],
dark_scheme_id: item[:dark_scheme_id],
default_calendar: item[:default_calendar],
digest_after_minutes: item[:digest_after_minutes],
- dismissed_channel_retention_reminder: item[:dismissed_channel_retention_reminder],
- dismissed_dm_retention_reminder: item[:dismissed_dm_retention_reminder],
dynamic_favicon: item[:dynamic_favicon],
email_digests: item[:email_digests],
email_in_reply_to: item[:email_in_reply_to],
@@ -56,12 +46,13 @@ module Migrations
enable_markdown_monospace_font: item[:enable_markdown_monospace_font],
enable_quoting: item[:enable_quoting],
enable_smart_lists: item[:enable_smart_lists],
+ enable_upcoming_change_available_notifications:
+ item[:enable_upcoming_change_available_notifications],
external_links_in_new_tab: item[:external_links_in_new_tab],
hide_presence: item[:hide_presence],
hide_profile: item[:hide_profile],
hide_profile_and_presence: item[:hide_profile_and_presence],
homepage_id: item[:homepage_id],
- ignore_channel_wide_mention: item[:ignore_channel_wide_mention],
include_tl0_in_digests: item[:include_tl0_in_digests],
interface_color_mode: item[:interface_color_mode],
last_redirected_to_top_at: item[:last_redirected_to_top_at],
@@ -69,14 +60,11 @@ module Migrations
mailing_list_mode: item[:mailing_list_mode],
mailing_list_mode_frequency: item[:mailing_list_mode_frequency],
new_topic_duration_minutes: item[:new_topic_duration_minutes],
- notification_level_when_assigned: item[:notification_level_when_assigned],
notification_level_when_replying: item[:notification_level_when_replying],
notify_on_linked_posts: item[:notify_on_linked_posts],
oldest_search_log_date: item[:oldest_search_log_date],
- only_chat_push_notifications: item[:only_chat_push_notifications],
- policy_email_frequency: item[:policy_email_frequency],
seen_popups: item[:seen_popups],
- show_thread_title_prompts: item[:show_thread_title_prompts],
+ show_original_content: item[:show_original_content],
sidebar_link_to_filtered_list: item[:sidebar_link_to_filtered_list],
sidebar_show_count_of_new_items: item[:sidebar_show_count_of_new_items],
skip_new_user_tips: item[:skip_new_user_tips],
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/users.rb
index 3cc696305a1..f746774033d 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/users.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/users.rb
@@ -71,6 +71,7 @@ module Migrations
manual_locked_trust_level: item[:manual_locked_trust_level],
moderator: item[:moderator],
name: item[:name],
+ original_username: nil,
primary_group_id: item[:primary_group_id],
registration_ip_address: item[:registration_ip_address],
silenced_till: item[:silenced_till],
diff --git a/migrations/core/db/intermediate_db_schema/100-base-schema.sql b/migrations/core/db/intermediate_db_schema/100-base-schema.sql
index faa2a580ec1..a6e3e09792c 100644
--- a/migrations/core/db/intermediate_db_schema/100-base-schema.sql
+++ b/migrations/core/db/intermediate_db_schema/100-base-schema.sql
@@ -143,6 +143,7 @@ CREATE TABLE "groups"
flair_upload_id TEXT,
full_name TEXT,
grant_trust_level INTEGER,
+ incoming_email TEXT,
members_visibility_level INTEGER,
membership_request_template TEXT,
mentionable_level INTEGER,
@@ -305,9 +306,11 @@ CREATE TABLE topics
deleted_by_id NUMERIC,
external_id NUMERIC,
featured_link TEXT,
+ locale TEXT,
pinned_at DATETIME,
pinned_globally BOOLEAN,
pinned_until DATETIME,
+ slow_mode_seconds INTEGER,
subtype TEXT,
title TEXT NOT NULL,
user_id NUMERIC,
diff --git a/migrations/core/lib/migrations/cli.rb b/migrations/core/lib/migrations/cli.rb
new file mode 100644
index 00000000000..b8089179fd6
--- /dev/null
+++ b/migrations/core/lib/migrations/cli.rb
@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Migrations
+ module CLI
+ # Lives here, not in `command.rb`, so it resolves without loading the command stack.
+ BIN = "migrations/bin/disco"
+ end
+end
diff --git a/migrations/core/lib/migrations/cli/bootstrap.rb b/migrations/core/lib/migrations/cli/bootstrap.rb
index cf0ccee49fa..b3979068588 100644
--- a/migrations/core/lib/migrations/cli/bootstrap.rb
+++ b/migrations/core/lib/migrations/cli/bootstrap.rb
@@ -18,8 +18,20 @@ module Migrations
top.call
rescue Samovar::Error => e
+ # The top-level command has no `--help` option, so a help request
+ # surfaces as an unparsable token.
+ help_request = e.is_a?(Samovar::InvalidInputError) && %w[--help -h].include?(e.token)
+
+ if !help_request
+ # e.g. `Could not parse token "validat"` — without it, an
+ # unrecognized command or argument prints bare usage with no
+ # explanation of what was wrong.
+ puts e.message.red
+ puts
+ end
+
e.command.print_usage
- exit(1)
+ exit(help_request ? 0 : 1)
end
# Samovar expects `--opt value`; the previous Thor-based CLI also accepted
diff --git a/migrations/core/lib/migrations/cli/command.rb b/migrations/core/lib/migrations/cli/command.rb
index c3099d278fb..b9601774cec 100644
--- a/migrations/core/lib/migrations/cli/command.rb
+++ b/migrations/core/lib/migrations/cli/command.rb
@@ -9,6 +9,11 @@ module Migrations
# such a command has been selected, keeping help and Rails-free commands
# fast.
class Command < Samovar::Command
+ # Raised by {#require_positional!} when a required positional is missing.
+ class MissingPositionalError < StandardError
+ include PresentableError
+ end
+
# Coerces a comma-separated `--only`/`--skip` value into a list of
# normalized step names. Shared as the `type:` for those options.
STEP_LIST = ->(value) do
@@ -40,6 +45,18 @@ module Migrations
private
+ # Samovar can't enforce required positionals: `one :x, required: true`
+ # raises during parsing, before `call` runs, which would break the
+ # `-h/--help` handling. Commands validate required positionals at the
+ # top of `call` with this helper instead.
+ def require_positional!(value, name, hint: nil)
+ return value unless value.nil?
+
+ message = +"Missing required argument: <#{name}>"
+ message << "\n#{hint}" if hint
+ raise MissingPositionalError, message
+ end
+
def hoist_options(input)
options = self.class.table.merged[:options]
return input unless options
diff --git a/migrations/core/lib/migrations/database/intermediate_db/group.rb b/migrations/core/lib/migrations/database/intermediate_db/group.rb
index bb46b916f9f..3a644e2ce88 100644
--- a/migrations/core/lib/migrations/database/intermediate_db/group.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/group.rb
@@ -24,6 +24,7 @@ module Migrations
flair_upload_id,
full_name,
grant_trust_level,
+ incoming_email,
members_visibility_level,
membership_request_template,
mentionable_level,
@@ -37,7 +38,7 @@ module Migrations
visibility_level
)
VALUES (
- ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)
SQL
private_constant :SQL
@@ -58,6 +59,7 @@ module Migrations
# @param flair_upload_id [String, nil]
# @param full_name [String, nil]
# @param grant_trust_level [Integer, nil]
+ # @param incoming_email [String, nil]
# @param members_visibility_level [Integer, nil]
# @param membership_request_template [String, nil]
# @param mentionable_level [Integer, nil]
@@ -86,6 +88,7 @@ module Migrations
flair_upload_id: nil,
full_name: nil,
grant_trust_level: nil,
+ incoming_email: nil,
members_visibility_level: nil,
membership_request_template: nil,
mentionable_level: nil,
@@ -114,6 +117,7 @@ module Migrations
flair_upload_id,
full_name,
grant_trust_level,
+ incoming_email,
members_visibility_level,
membership_request_template,
mentionable_level,
diff --git a/migrations/core/lib/migrations/database/intermediate_db/topic.rb b/migrations/core/lib/migrations/database/intermediate_db/topic.rb
index 42446e018bd..1ae9340a6c0 100644
--- a/migrations/core/lib/migrations/database/intermediate_db/topic.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/topic.rb
@@ -21,9 +21,11 @@ module Migrations
deleted_by_id,
external_id,
featured_link,
+ locale,
pinned_at,
pinned_globally,
pinned_until,
+ slow_mode_seconds,
subtype,
title,
user_id,
@@ -32,7 +34,7 @@ module Migrations
visible
)
VALUES (
- ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)
SQL
private_constant :SQL
@@ -50,9 +52,11 @@ module Migrations
# @param deleted_by_id [Integer, String, nil]
# @param external_id [Integer, String, nil]
# @param featured_link [String, nil]
+ # @param locale [String, nil]
# @param pinned_at [Time, nil]
# @param pinned_globally [Boolean, nil]
# @param pinned_until [Time, nil]
+ # @param slow_mode_seconds [Integer, nil]
# @param subtype [String, nil]
# @param title [String]
# @param user_id [Integer, String, nil]
@@ -73,9 +77,11 @@ module Migrations
deleted_by_id: nil,
external_id: nil,
featured_link: nil,
+ locale: nil,
pinned_at: nil,
pinned_globally: nil,
pinned_until: nil,
+ slow_mode_seconds: nil,
subtype: nil,
title:,
user_id: nil,
@@ -96,9 +102,11 @@ module Migrations
deleted_by_id,
external_id,
featured_link,
+ locale,
Migrations::Database.format_datetime(pinned_at),
Migrations::Database.format_boolean(pinned_globally),
Migrations::Database.format_datetime(pinned_until),
+ slow_mode_seconds,
subtype,
title,
user_id,
diff --git a/migrations/core/spec/lib/cli/command_spec.rb b/migrations/core/spec/lib/cli/command_spec.rb
new file mode 100644
index 00000000000..353a3faf8be
--- /dev/null
+++ b/migrations/core/spec/lib/cli/command_spec.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::CLI::Command do
+ describe "#require_positional!" do
+ subject(:command) { described_class.new }
+
+ it "returns the value when it is present" do
+ expect(command.send(:require_positional!, "discourse", "converter_type")).to eq("discourse")
+ end
+
+ it "raises a presentable error when the value is missing" do
+ expect { command.send(:require_positional!, nil, "table_name") }.to raise_error(
+ described_class::MissingPositionalError,
+ "Missing required argument: <table_name>",
+ )
+ end
+
+ it "appends the hint to the error message" do
+ expect {
+ command.send(
+ :require_positional!,
+ nil,
+ "converter_type",
+ hint: "Valid names are: discourse",
+ )
+ }.to raise_error(
+ described_class::MissingPositionalError,
+ "Missing required argument: <converter_type>\nValid names are: discourse",
+ )
+ end
+
+ it "raises an error that the exception handler presents without a backtrace" do
+ expect(described_class::MissingPositionalError.ancestors).to include(
+ Migrations::CLI::PresentableError,
+ )
+ end
+ end
+end
diff --git a/migrations/docs/schema-configuration.md b/migrations/docs/schema-configuration.md
index 2f50a1fe5a9..149a762eba6 100644
--- a/migrations/docs/schema-configuration.md
+++ b/migrations/docs/schema-configuration.md
@@ -27,15 +27,29 @@ migrations/tooling/config/schema/intermediate_db/
| Command | Description |
|----------------------------------------|------------------------------------------------------------|
| `schema add TABLE` | Create a config file for a new table |
-| `schema validate` | Validate config against the database |
| `schema diff` | Show differences between config and database |
| `schema generate` | Generate SQL schema, Ruby models, and enum files |
| `schema list` | List configured tables and enums, plus ignored table count |
| `schema ignore TABLE [--reason "..."]` | Add a table to `ignored.rb` |
+| `schema unignore TABLE` | Remove a table from `ignored.rb` |
| `schema refresh-plugins` | Regenerate the plugin manifest |
All commands accept `--db NAME` (default: `intermediate_db`).
+To check everything at once, use `disco check`: it verifies that the database
+has no pending migrations, that the config is valid and in sync with the
+database, that the committed generated files match what generation produces
+(without touching the working tree), that the reference converter covers every
+column, and that no converter writes columns or models that don't exist in the
+schema. It
+exits non-zero on the first failing check — CI runs exactly this command, so a
+clean local run means a green CI check. `disco check schema` and
+`disco check coverage` run the respective subsets.
+
+To see which columns a single converter writes (with per-model `N/M` coverage),
+use `disco check coverage --inspect <converter>`. This is a read-only report for
+debugging coverage gaps; it never changes the exit status.
+
## Table configuration
Each table has its own file in `tables/`. The basic structure:
@@ -380,6 +394,6 @@ end
1. **Add** a new table: `schema add users`
2. **Edit** the generated file in `tables/users.rb`
-3. **Validate** your config: `schema validate`
-4. **Check differences**: `schema diff`
-5. **Generate** the schema, models, and enums: `schema generate`
+3. **Check differences**: `schema diff`
+4. **Generate** the schema, models, and enums: `schema generate`
+5. **Verify** everything is consistent and committed: `disco check`
diff --git a/migrations/tooling/config/schema/intermediate_db/ignored.rb b/migrations/tooling/config/schema/intermediate_db/ignored.rb
index 7f9e1f43a64..a854d0f38db 100644
--- a/migrations/tooling/config/schema/intermediate_db/ignored.rb
+++ b/migrations/tooling/config/schema/intermediate_db/ignored.rb
@@ -150,6 +150,7 @@ Migrations::Tooling::Schema.ignored do
tables :external_upload_stubs, :optimized_images, :optimized_videos, :upload_references, :uploads
tables :admin_dashboard_reports,
+ :admin_dashboard_sections,
:browser_pageview_country_daily_rollups,
:browser_pageview_referrer_daily_rollups
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/groups.rb b/migrations/tooling/config/schema/intermediate_db/tables/groups.rb
index a93fb231775..9f49f281a1e 100644
--- a/migrations/tooling/config/schema/intermediate_db/tables/groups.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/groups.rb
@@ -14,6 +14,7 @@ Migrations::Tooling::Schema.table :groups do
:full_name,
:grant_trust_level,
:id,
+ :incoming_email,
:members_visibility_level,
:membership_request_template,
:mentionable_level,
@@ -37,7 +38,6 @@ Migrations::Tooling::Schema.table :groups do
:email_password,
:email_username,
:has_messages,
- :incoming_email,
reason: "TODO: Figure out if we need to import these settings"
ignore :smtp_enabled,
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/topics.rb b/migrations/tooling/config/schema/intermediate_db/tables/topics.rb
index 61d6e6757b4..d4365216697 100644
--- a/migrations/tooling/config/schema/intermediate_db/tables/topics.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/topics.rb
@@ -18,7 +18,6 @@ Migrations::Tooling::Schema.table :topics do
:last_post_user_id,
:last_posted_at,
:like_count,
- :locale,
:moderator_posts_count,
:notify_moderators_count,
:participant_count,
@@ -28,7 +27,6 @@ Migrations::Tooling::Schema.table :topics do
:reviewable_score,
:score,
:slug,
- :slow_mode_seconds,
:spam_count,
:word_count,
reason: "Calculated columns"
diff --git a/migrations/tooling/config/schema/plugin_manifest.yml b/migrations/tooling/config/schema/plugin_manifest.yml
index 10cd29e404a..6622f7c1362 100644
--- a/migrations/tooling/config/schema/plugin_manifest.yml
+++ b/migrations/tooling/config/schema/plugin_manifest.yml
@@ -114,6 +114,7 @@ plugins:
- discourse_calendar_post_event_dates
- discourse_post_event_events
- discourse_post_event_invitees
+ - livestream_topic_chat_channels
columns: {}
discourse-data-explorer:
tables:
@@ -213,18 +214,19 @@ plugins:
columns: {}
plugin_checksums:
automation: e69972690991bc04d92b2d161aac8921
- chat: af4a593f3c48a945ff09a44128f6efb9
+ chat: eed419ab29ae9cb0a73a32c0e7be9d6c
discourse-adplugin: 9e52419decf1099c5bc301a90e6cc91a
discourse-affiliate: '08c6e1a851441a082e9be9ddddc63910'
discourse-ai: 0eb3819de9ca86f22434718c56a58d1b
discourse-assign: 7a7a36e8ac0214e8a6aca1a1ed9899aa
discourse-cakeday: a5bb91366c8dc8d267088eb0c2679187
- discourse-calendar: 3f8dd26f85e1c26146fbf1c9a882d233
+ discourse-calendar: f61c87be8f574bba58a658b73748aa84
discourse-chat-integration: ffa3bf5dba9089491d2fc02971eb96f8
discourse-data-explorer: ba288745bea628996590debdf02fbaa4
discourse-gamification: 633256e241b030fddcc059076f7474f8
discourse-github: 3bd17b2c09828e232e7565edbf2c746f
- discourse-graphviz: 2182f4029ce73b9b77ea682a4558087c
+ discourse-graphviz: 8a1835520fbf8f7f5d1ef8a25d737e20
+ discourse-hcaptcha: fd538c1338062354276ccfda4cbafd55
discourse-lazy-videos: a916a4f265d1cc6f99a3e53f5bcd1e9e
discourse-local-dates: 18d3ec627bf8546fe13bae24c6a5bb6c
discourse-microsoft-auth: 3a95a5b50edc156ad06f40aadda7d6bc
diff --git a/migrations/tooling/lib/migrations/tooling/cli/check_command.rb b/migrations/tooling/lib/migrations/tooling/cli/check_command.rb
new file mode 100644
index 00000000000..526b05fef87
--- /dev/null
+++ b/migrations/tooling/lib/migrations/tooling/cli/check_command.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "colored2"
+
+module Migrations
+ module Tooling
+ module CLI
+ # `disco check` — the single entrypoint for all source-tree checks.
+ # Without a subcommand it runs every check in dependency order and
+ # stops at the first failing link, since everything downstream of a
+ # stale link would report against stale inputs.
+ class CheckCommand < Migrations::CLI::Command
+ self.description = "Run all schema and converter checks"
+
+ # NOTE: no group-level `-h/--help` option — the option hoisting in
+ # `Command#parse` would steal `--help` from the subcommands
+ # (`check schema --help` would run the check). A bare `--help`
+ # surfaces as an unparsable token, which Bootstrap turns into usage.
+
+ nested :command,
+ {
+ "schema" => CheckCommands::SchemaCommand,
+ "coverage" => CheckCommands::CoverageCommand,
+ }
+
+ def call
+ if @command
+ @command.call
+ else
+ run_all
+ end
+ end
+
+ private
+
+ def run_all
+ # The schema checks need Rails. This command itself doesn't declare
+ # `requires_rails!` so that `check coverage` and `--help` stay
+ # Rails-free; the all-checks mode boots it here instead.
+ Migrations.load_rails_environment(quiet: true)
+
+ puts "Checking schema config and generated files...".bold
+ unless CheckCommands::SchemaCommand.new([]).run
+ puts
+ puts "Skipping the remaining checks, they would run against stale inputs.".red
+ exit 1
+ end
+
+ puts
+ puts "Checking converter coverage...".bold
+ exit 1 unless CheckCommands::CoverageCommand.new([]).run
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/tooling/lib/migrations/tooling/cli/check_commands/coverage_command.rb b/migrations/tooling/lib/migrations/tooling/cli/check_commands/coverage_command.rb
new file mode 100644
index 00000000000..8cbe5660755
--- /dev/null
+++ b/migrations/tooling/lib/migrations/tooling/cli/check_commands/coverage_command.rb
@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "colored2"
+
+module Migrations
+ module Tooling
+ module CLI
+ module CheckCommands
+ # `disco check coverage` — asserts that the reference converter covers
+ # every IntermediateDB column. Runs without Rails.
+ #
+ # `--inspect <converter>` switches from the gate to a read-only report of
+ # the columns a single converter writes, with per-model N/M coverage. It
+ # is a debugging aid, not a gate, so it never changes the exit status.
+ class CoverageCommand < Migrations::CLI::Command
+ self.description = "Check converter coverage of the IntermediateDB schema"
+
+ class Error < StandardError
+ include Migrations::CLI::PresentableError
+ end
+
+ options do
+ option "-h/--help", "Print out help."
+ option "--inspect <converter>",
+ "Print one converter's covered columns instead of running the gate."
+ end
+
+ def call
+ return print_usage if @options[:help]
+
+ if (converter = @options[:inspect])
+ inspect_converter(converter.downcase)
+ else
+ exit 1 unless run
+ end
+ end
+
+ # Called directly by `CheckCommand#run_all`, so it must return the
+ # pass/fail boolean rather than exiting.
+ def run
+ Coverage::ReferenceCheck.run
+ end
+
+ private
+
+ def inspect_converter(converter_name)
+ validate_converter!(converter_name)
+
+ result =
+ Coverage::ConverterAnalyzer.new(
+ Migrations::Converters.path_of(converter_name),
+ ).analyze
+ written = result.written_columns
+ expected = Coverage::SchemaColumns.call
+
+ puts "Columns written by the '#{converter_name}' converter:"
+ puts
+
+ if written.empty? && result.unknown_models.empty?
+ puts " (no IntermediateDB.create calls found)"
+ return
+ end
+
+ written.keys.sort.each do |model_name|
+ columns = written[model_name].to_a.sort
+ schema_count = expected[model_name]&.columns&.size
+ coverage = schema_count ? "#{columns.size}/#{schema_count}" : columns.size.to_s
+ puts " #{model_name} (#{coverage}): #{columns.join(", ")}"
+
+ if (model = expected[model_name]) && (extra = columns - model.columns).any?
+ puts " unknown #{"column".pluralize(extra.size)}: #{extra.join(", ")}".red
+ end
+ end
+
+ result.unknown_models.keys.sort.each do |model_name|
+ locations = result.unknown_models[model_name]
+ puts " #{model_name}: model does not exist (#{locations.join(", ")})".red
+ end
+ end
+
+ def validate_converter!(converter_name)
+ names = Migrations::Converters.names
+ return if names.include?(converter_name)
+
+ raise Error, <<~MSG.strip
+ Unknown converter name: #{converter_name}
+ Valid names are: #{names.join(", ")}
+ MSG
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/tooling/lib/migrations/tooling/cli/check_commands/schema_command.rb b/migrations/tooling/lib/migrations/tooling/cli/check_commands/schema_command.rb
new file mode 100644
index 00000000000..b876eae19fb
--- /dev/null
+++ b/migrations/tooling/lib/migrations/tooling/cli/check_commands/schema_command.rb
@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "colored2"
+
+module Migrations
+ module Tooling
+ module CLI
+ module CheckCommands
+ # `disco check schema` — validates the schema pipeline up to the
+ # generated artifacts, in dependency order:
+ #
+ # 1. the dev database has no pending migrations (otherwise every
+ # later answer compares against a stale baseline)
+ # 2. the schema config is internally valid
+ # 3. every database table and column is either configured or ignored
+ # (drift — reported with `schema add`/`ignore`/`generate` suggestions)
+ # 4. the committed generated files match what generation produces
+ #
+ # It stops at the first failing link, since everything downstream
+ # would report against stale inputs.
+ class SchemaCommand < SchemaCommands::BaseCommand
+ include SchemaCommands::DiffOutput
+
+ self.description = "Check schema config against the database and generated files"
+
+ options do
+ option "-h/--help", "Print out help."
+ option "--db <name>", "Database configuration to use.", default: "intermediate_db"
+ end
+
+ def call
+ return print_usage if @options[:help]
+
+ exit 1 unless run
+ end
+
+ def run
+ database = selected_database
+
+ check_pending_migrations && check_config_validity(database) &&
+ check_config_drift(database) && check_artifacts(database)
+ end
+
+ private
+
+ def check_config_validity(database)
+ errors = schema.validate(database:)
+ return true if errors.empty?
+
+ puts "✗ The schema config is invalid:".red
+ errors.each { |error| puts " - #{error}" }
+ false
+ end
+
+ def check_pending_migrations
+ return true unless ActiveRecord::Base.connection_pool.migration_context.needs_migration?
+
+ puts "✗ The database has pending migrations, the checks would run against a stale baseline.".red
+ puts " Run `bin/rake db:migrate` first."
+ false
+ end
+
+ def check_config_drift(database)
+ result = schema.diff(database:)
+ return true unless result.actionable?
+
+ puts "✗ The schema config is out of sync with the database.".red
+ puts
+ display_diff(result, database:)
+ false
+ end
+
+ def check_artifacts(database)
+ result = Schema::DSL::ArtifactsChecker.new(schema, database:).check
+
+ if result.clean?
+ puts "✓ Schema config matches the database and the generated files.".green
+ return true
+ end
+
+ puts "✗ The generated files are out of date.".red
+ result.changed.each { |path| puts " ~ #{path} (outdated)".yellow }
+ result.missing.each { |path| puts " + #{path} (not committed)".green }
+ result.stale.each { |path| puts " - #{path} (no longer generated)".red }
+ puts
+ puts "Run `#{Migrations::CLI::BIN} schema generate` and commit the result."
+ false
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_command.rb
index 10ef8360db6..ff941c28b39 100644
--- a/migrations/tooling/lib/migrations/tooling/cli/schema_command.rb
+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_command.rb
@@ -4,16 +4,16 @@ module Migrations
module Tooling
module CLI
class SchemaCommand < Migrations::CLI::Command
- self.description = "Manage the IntermediateDB schema"
+ self.description = "Manage database schemas"
nested :command,
{
"generate" => SchemaCommands::GenerateCommand,
- "validate" => SchemaCommands::ValidateCommand,
"list" => SchemaCommands::ListCommand,
"diff" => SchemaCommands::DiffCommand,
"add" => SchemaCommands::AddCommand,
"ignore" => SchemaCommands::IgnoreCommand,
+ "unignore" => SchemaCommands::UnignoreCommand,
"refresh-plugins" => SchemaCommands::RefreshPluginsCommand,
}
diff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/add_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/add_command.rb
index 30ffbf0f337..ed94fe1fa4c 100644
--- a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/add_command.rb
+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/add_command.rb
@@ -18,6 +18,7 @@ module Migrations
def call
return print_usage if @options[:help]
+ require_positional!(table_name, "table_name")
database = selected_database
path = schema.add_table(table_name, database:)
@@ -25,7 +26,7 @@ module Migrations
puts
puts "Next steps:"
puts " 1. Edit the file to configure columns"
- puts " 2. Run 'disco schema validate'"
+ puts " 2. Run '#{Migrations::CLI::BIN} check schema'"
end
end
end
diff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/diff_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/diff_command.rb
index 3a16519cfa7..1cef542c09d 100644
--- a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/diff_command.rb
+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/diff_command.rb
@@ -7,6 +7,8 @@ module Migrations
module CLI
module SchemaCommands
class DiffCommand < BaseCommand
+ include DiffOutput
+
self.description = "Show differences between configuration and database"
options do
@@ -20,79 +22,7 @@ module Migrations
database = selected_database
result = schema.diff(database:)
- display_diff(result, verbose: @options[:verbose])
- end
-
- private
-
- def display_diff(result, verbose: false)
- sections = []
-
- if result.unconfigured_tables.any?
- lines = ["Unconfigured tables (add to tables/ or ignored.rb):".bold]
- result.unconfigured_tables.each do |t|
- plugin_info = t.plugin ? " [#{t.plugin}]" : ""
- lines << " + #{t.name}#{plugin_info}".green
- end
- sections << lines.join("\n")
- end
-
- if result.missing_tables.any?
- lines = ["Missing tables (configured but not in database):".bold]
- result.missing_tables.each { |t| lines << " - #{t.name}".red }
- sections << lines.join("\n")
- end
-
- if result.stale_ignored_tables.any?
- lines = ["Stale ignored tables (no longer in database):".bold]
- result.stale_ignored_tables.each { |t| lines << " ~ #{t.name}".yellow }
- sections << lines.join("\n")
- end
-
- table_diffs = filter_table_diffs(result.table_diffs, verbose:)
-
- if table_diffs.any?
- lines = ["Column differences:".bold]
- table_diffs.each do |table_diff|
- lines << " #{table_diff.table_name}:".bold
-
- table_diff.unconfigured_columns.each do |c|
- plugin_info = c.plugin ? " [#{c.plugin}]" : ""
- lines << " + #{c.name}#{plugin_info}".green
- end
-
- table_diff.missing_columns.each { |c| lines << " - #{c.name}".red }
- table_diff.stale_ignored_columns.each do |c|
- lines << " ~ #{c.name} (ignored but gone)".yellow
- end
-
- if verbose
- table_diff.auto_ignored_columns.each do |c|
- lines << " #{c.name} [#{c.plugin}] (auto-ignored from plugin)".cyan
- end
- end
- end
- sections << lines.join("\n")
- end
-
- if sections.any?
- puts sections.join("\n\n")
- puts
- puts "Suggested actions:".bold
- puts " disco schema add <table>"
- puts " disco schema ignore <table> [--reason \"...\"]"
- else
- puts "✓ No differences found".green
- end
- end
-
- def filter_table_diffs(table_diffs, verbose:)
- return table_diffs if verbose
-
- table_diffs.select do |td|
- td.unconfigured_columns.any? || td.missing_columns.any? ||
- td.stale_ignored_columns.any?
- end
+ display_diff(result, database:, verbose: @options[:verbose])
end
end
end
diff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/diff_output.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/diff_output.rb
new file mode 100644
index 00000000000..0ca970795db
--- /dev/null
+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/diff_output.rb
@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require "colored2"
+
+module Migrations
+ module Tooling
+ module CLI
+ module SchemaCommands
+ # Shared presentation of `Schema.diff` results for the `schema diff`
+ # and `check schema` commands.
+ module DiffOutput
+ private
+
+ def display_diff(result, database:, verbose: false)
+ sections = []
+
+ if result.unconfigured_tables.any?
+ lines = ["Unconfigured tables (add to tables/ or ignored.rb):".bold]
+ result.unconfigured_tables.each do |t|
+ plugin_info = t.plugin ? " [#{t.plugin}]" : ""
+ lines << " + #{t.name}#{plugin_info}".green
+ end
+ sections << lines.join("\n")
+ end
+
+ if result.missing_tables.any?
+ lines = ["Missing tables (configured but not in database):".bold]
+ result.missing_tables.each { |t| lines << " - #{t.name}".red }
+ sections << lines.join("\n")
+ end
+
+ if result.stale_ignored_tables.any?
+ lines = ["Stale ignored tables (no longer in database):".bold]
+ result.stale_ignored_tables.each { |t| lines << " ~ #{t.name}".yellow }
+ sections << lines.join("\n")
+ end
+
+ table_diffs = filter_table_diffs(result.table_diffs, verbose:)
+
+ if table_diffs.any?
+ lines = ["Column differences:".bold]
+ table_diffs.each do |table_diff|
+ lines << " #{table_diff.table_name}:".bold
+
+ table_diff.unconfigured_columns.each do |c|
+ plugin_info = c.plugin ? " [#{c.plugin}]" : ""
+ lines << " + #{c.name}#{plugin_info}".green
+ end
+
+ table_diff.missing_columns.each { |c| lines << " - #{c.name}".red }
+ table_diff.stale_ignored_columns.each do |c|
+ lines << " ~ #{c.name} (ignored but gone)".yellow
+ end
+
+ if verbose
+ table_diff.auto_ignored_columns.each do |c|
+ lines << " #{c.name} [#{c.plugin}] (auto-ignored from plugin)".cyan
+ end
+ end
+ end
+ sections << lines.join("\n")
+ end
+
+ if sections.any?
+ puts sections.join("\n\n")
+ display_suggested_actions(result, database)
+ else
+ puts "✓ No differences found".green
+ end
+ end
+
+ # Only suggests actions that match the actual findings: commands
+ # first, then the file edits, grouped under the config file path.
+ def display_suggested_actions(result, database)
+ commands = suggested_commands(result)
+ file_edits = suggested_file_edits(result)
+ return if commands.empty? && file_edits.empty?
+
+ puts
+ puts "Suggested actions:".bold
+ commands.each { |command| puts " #{command}" }
+
+ if file_edits.any?
+ puts if commands.any?
+ tables_dir = File.join(relative_config_path(database), "tables")
+ puts " In #{tables_dir}/<table>.rb:"
+ file_edits.each { |file_edit| puts " - #{file_edit}" }
+ end
+ end
+
+ def suggested_commands(result)
+ commands = []
+
+ if result.unconfigured_tables.any?
+ commands << "#{Migrations::CLI::BIN} schema add <table>"
+ commands << "#{Migrations::CLI::BIN} schema ignore <table> [--reason \"...\"]"
+ end
+
+ if result.stale_ignored_tables.any?
+ commands << "#{Migrations::CLI::BIN} schema unignore <table>"
+ end
+
+ commands
+ end
+
+ def suggested_file_edits(result)
+ file_edits = []
+
+ if result.table_diffs.any? { |td| td.unconfigured_columns.any? }
+ file_edits << "add new columns to the `include` list or `ignore` them with a reason"
+ end
+
+ if result.table_diffs.any? { |td| td.missing_columns.any? }
+ file_edits << "remove columns that no longer exist from the `include` list"
+ end
+
+ if result.table_diffs.any? { |td| td.stale_ignored_columns.any? }
+ file_edits << "remove columns that no longer exist from the `ignore` list"
+ end
+
+ if result.missing_tables.any?
+ file_edits << "delete the file if the table no longer exists"
+ end
+
+ file_edits
+ end
+
+ def relative_config_path(database)
+ path = Pathname.new(schema.config_path(database))
+ path.relative_path_from(Pathname.pwd).to_s
+ rescue ArgumentError
+ path.to_s
+ end
+
+ def filter_table_diffs(table_diffs, verbose:)
+ return table_diffs if verbose
+
+ table_diffs.select(&:actionable?)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/generate_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/generate_command.rb
index 5517869789c..3a0d4a516db 100644
--- a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/generate_command.rb
+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/generate_command.rb
@@ -18,7 +18,8 @@ module Migrations
return print_usage if @options[:help]
database = selected_database
- resolved = schema.generate(database:)
+ result = schema.generate(database:)
+ resolved = result.resolved
puts
table_count = resolved.tables.size
@@ -26,6 +27,10 @@ module Migrations
tables_str = "#{table_count} #{"table".pluralize(table_count)}"
enums_str = "#{enum_count} #{"enum".pluralize(enum_count)}"
puts "✓ Generated #{tables_str}, #{enums_str}".green
+
+ result.deleted_files.each do |path|
+ puts "✓ Deleted #{path} (no longer generated)".yellow
+ end
end
end
end
diff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/ignore_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/ignore_command.rb
index cbb7ada090a..203a41f5b7a 100644
--- a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/ignore_command.rb
+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/ignore_command.rb
@@ -19,6 +19,7 @@ module Migrations
def call
return print_usage if @options[:help]
+ require_positional!(table_name, "table_name")
database = selected_database
schema.ignore_table(table_name, reason: @options[:reason], database:)
diff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/unignore_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/unignore_command.rb
new file mode 100644
index 00000000000..b1ac3eeaef1
--- /dev/null
+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/unignore_command.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "colored2"
+
+module Migrations
+ module Tooling
+ module CLI
+ module SchemaCommands
+ class UnignoreCommand < BaseCommand
+ self.description = "Remove a table from ignored.rb"
+
+ options do
+ option "-h/--help", "Print out help."
+ option "--db <name>", "Database configuration to use.", default: "intermediate_db"
+ end
+
+ one :table_name, "The name of the table to remove from ignored.rb."
+
+ def call
+ return print_usage if @options[:help]
+ require_positional!(table_name, "table_name")
+
+ database = selected_database
+ schema.unignore_table(table_name, database:)
+ puts "✓ Removed #{table_name} from ignored.rb".green
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/validate_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/validate_command.rb
deleted file mode 100644
index 46d8b08899f..00000000000
--- a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/validate_command.rb
+++ /dev/null
@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-require "colored2"
-
-module Migrations
- module Tooling
- module CLI
- module SchemaCommands
- class ValidateCommand < BaseCommand
- self.description = "Validate schema configuration against the database"
-
- options do
- option "-h/--help", "Print out help."
- option "--db <name>", "Database configuration to use.", default: "intermediate_db"
- end
-
- def call
- return print_usage if @options[:help]
-
- database = selected_database
- errors = schema.validate(database:)
- print_validation_errors(errors)
-
- puts "✓ Schema valid".green
- end
-
- private
-
- def print_validation_errors(errors)
- return if errors.empty?
-
- errors.each { |e| puts "✗ #{e}".red }
- puts
- error_count = errors.size
- puts "#{error_count} #{"error".pluralize(error_count)}"
- exit 1
- end
- end
- end
- end
- end
-end
diff --git a/migrations/tooling/lib/migrations/tooling/coverage/analysis_error.rb b/migrations/tooling/lib/migrations/tooling/coverage/analysis_error.rb
new file mode 100644
index 00000000000..a4e9f3fa16c
--- /dev/null
+++ b/migrations/tooling/lib/migrations/tooling/coverage/analysis_error.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Tooling
+ module Coverage
+ # Raised when a converter source cannot be analysed with confidence: a
+ # parse failure, or a `.create` call site that passes a `**` splat or a
+ # non-literal keyword. Such call sites can't be verified statically, so the
+ # analyser fails loudly rather than silently under-reporting coverage.
+ class AnalysisError < StandardError
+ include Migrations::CLI::PresentableError
+ end
+ end
+ end
+end
diff --git a/migrations/tooling/lib/migrations/tooling/coverage/converter_analyzer.rb b/migrations/tooling/lib/migrations/tooling/coverage/converter_analyzer.rb
new file mode 100644
index 00000000000..5944b043620
--- /dev/null
+++ b/migrations/tooling/lib/migrations/tooling/coverage/converter_analyzer.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Tooling
+ module Coverage
+ # Analyses a single converter's sources: the union of IntermediateDB
+ # columns it writes via `.create` across all of its step and helper
+ # sources, and the call sites that reference models that don't exist.
+ # Unioning per model across every call site means a column written in
+ # only one branch of a conditional still counts as covered.
+ class ConverterAnalyzer
+ # `written_columns` per model name; `unknown_models` call site
+ # locations per non-resolving model name.
+ Result = Data.define(:written_columns, :unknown_models)
+
+ # @param converter_path [String] the converter's root source directory
+ def initialize(converter_path)
+ @converter_path = converter_path
+ end
+
+ # @return [Result]
+ def analyze
+ written = {}
+ unknown = {}
+
+ source_files.each do |file|
+ scan = CreateCallScanner.scan(File.read(file), path: display_path(file))
+
+ scan.columns.each { |model, columns| (written[model] ||= Set.new).merge(columns) }
+ scan.unknown_models.each do |model, locations|
+ (unknown[model] ||= []).concat(locations)
+ end
+ end
+
+ Result.new(written_columns: written, unknown_models: unknown)
+ end
+
+ private
+
+ def source_files
+ Dir[File.join(@converter_path, "**", "*.rb")].sort
+ end
+
+ def display_path(file)
+ Pathname.new(file).relative_path_from(Pathname.pwd).to_s
+ rescue ArgumentError
+ file
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/tooling/lib/migrations/tooling/coverage/create_call_scanner.rb b/migrations/tooling/lib/migrations/tooling/coverage/create_call_scanner.rb
new file mode 100644
index 00000000000..9914f21f445
--- /dev/null
+++ b/migrations/tooling/lib/migrations/tooling/coverage/create_call_scanner.rb
@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require "prism"
+
+module Migrations
+ module Tooling
+ module Coverage
+ # Walks a Ruby source string with Prism and collects, per IntermediateDB
+ # model, the keyword names passed to every `IntermediateDB::<Model>.create`
+ # call site.
+ #
+ # Converter call sites use the bare `IntermediateDB::<Model>` receiver
+ # (resolved lexically through `Conversion::Step`), but the same receiver may
+ # also be written with leading qualification (e.g.
+ # `Database::IntermediateDB::Upload`). We therefore match on the trailing
+ # `IntermediateDB::<Const>` segment regardless of qualification, then resolve
+ # `<Const>` against `Migrations::Database::IntermediateDB`.
+ #
+ # A `.create` call on a constant that doesn't resolve to a model — e.g. the
+ # model of a table that was removed from the schema — would raise at
+ # runtime, so such call sites are reported as unknown models instead of
+ # being silently skipped.
+ #
+ # A column passed via `**splat` or a non-literal keyword can't be verified
+ # statically, so the scanner raises an {AnalysisError} rather than silently
+ # under-reporting.
+ class CreateCallScanner < Prism::Visitor
+ INTERMEDIATE_DB = :IntermediateDB
+ private_constant :INTERMEDIATE_DB
+
+ # `columns` are the written columns per model name; `unknown_models` are
+ # the call site locations per non-resolving model name.
+ Result = Data.define(:columns, :unknown_models)
+
+ # @param source [String] Ruby source to analyse
+ # @param path [String] source location, used in error messages and
+ # unknown model call site locations
+ # @return [Result]
+ def self.scan(source, path:)
+ result = Prism.parse(source)
+
+ unless result.success?
+ details = result.errors.map { |e| "#{e.message} (line #{e.location.start_line})" }
+ raise AnalysisError, "Failed to parse #{path}: #{details.join(", ")}"
+ end
+
+ scanner = new(path)
+ result.value.accept(scanner)
+ Result.new(columns: scanner.columns, unknown_models: scanner.unknown_models)
+ end
+
+ attr_reader :columns, :unknown_models
+
+ def initialize(path)
+ super()
+ @path = path
+ @columns = Hash.new { |hash, key| hash[key] = Set.new }
+ @unknown_models = Hash.new { |hash, key| hash[key] = [] }
+ end
+
+ def visit_call_node(node)
+ record_create_call(node)
+ super
+ end
+
+ private
+
+ def record_create_call(node)
+ return unless node.name == :create
+
+ receiver = node.receiver
+ return unless receiver.is_a?(Prism::ConstantPathNode)
+ return unless const_name(receiver.parent) == INTERMEDIATE_DB
+
+ model_name = receiver.name.to_s
+
+ if model?(model_name)
+ collect_keywords(node, model_name)
+ else
+ @unknown_models[model_name] << "#{@path}:#{node.location.start_line}"
+ end
+ end
+
+ # Whether the constant resolves to a model that responds to `create`.
+ def model?(model_name)
+ model = Migrations::Database::IntermediateDB.const_get(model_name, false)
+ model.respond_to?(:create)
+ rescue NameError
+ false
+ end
+
+ def const_name(node)
+ case node
+ when Prism::ConstantReadNode, Prism::ConstantPathNode
+ node.name
+ end
+ end
+
+ def collect_keywords(node, model_name)
+ # Ensure the model is recorded even when a call site passes no
+ # columns at all.
+ @columns[model_name]
+
+ arguments = node.arguments
+ return unless arguments
+
+ arguments.arguments.each do |argument|
+ next unless argument.is_a?(Prism::KeywordHashNode)
+
+ argument.elements.each { |element| record_keyword(element, model_name, node) }
+ end
+ end
+
+ def record_keyword(element, model_name, node)
+ if element.is_a?(Prism::AssocSplatNode)
+ raise AnalysisError, unverifiable_message("a `**` splat", model_name, node)
+ end
+
+ key = element.key if element.is_a?(Prism::AssocNode)
+ unless key.is_a?(Prism::SymbolNode)
+ raise AnalysisError, unverifiable_message("a non-literal keyword", model_name, node)
+ end
+
+ @columns[model_name] << key.unescaped.to_sym
+ end
+
+ def unverifiable_message(what, model_name, node)
+ "Cannot statically analyse `IntermediateDB::#{model_name}.create` at " \
+ "#{@path}:#{node.location.start_line}: it passes #{what}. " \
+ "Pass each column as an explicit keyword so coverage can be verified."
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/tooling/lib/migrations/tooling/coverage/reference_check.rb b/migrations/tooling/lib/migrations/tooling/coverage/reference_check.rb
new file mode 100644
index 00000000000..a00ca6f8376
--- /dev/null
+++ b/migrations/tooling/lib/migrations/tooling/coverage/reference_check.rb
@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+require "colored2"
+
+module Migrations
+ module Tooling
+ module Coverage
+ # Asserts that the reference converter writes every IntermediateDB
+ # column, and that no converter writes columns or models that don't
+ # exist in the schema — those call sites would raise at runtime, and
+ # typically are leftovers of a schema change. Analysing every discovered
+ # converter also means an unverifiable call site (a `**` splat or
+ # non-literal keyword) fails loudly even in a non-reference converter.
+ # Prints its findings and returns whether the check passed.
+ class ReferenceCheck
+ # The Discourse converter is the reference implementation: the
+ # IntermediateDB schema is modelled on Discourse's own schema, so it
+ # is the one converter expected to populate every column. This is
+ # structural, not a configurable role, so it is hardcoded.
+ REFERENCE_CONVERTER = "discourse"
+
+ class Error < StandardError
+ include Migrations::CLI::PresentableError
+ end
+
+ def self.run
+ new.run
+ end
+
+ def run
+ converters = Migrations::Converters.names
+ ensure_reference_present!(converters)
+
+ results = converters.to_h { |name| [name, analyze(name)] }
+ expected = SchemaColumns.call
+
+ passed = true
+
+ # Unknown columns and models are errors for every converter, not
+ # just the reference: there is no legitimate reason to write
+ # something the schema doesn't know.
+ converters.each do |name|
+ unknown_columns = unknown_columns(expected, results.fetch(name).written_columns)
+ unknown_models = results.fetch(name).unknown_models
+
+ if unknown_columns.any? || unknown_models.any?
+ report_unknown(name, expected, unknown_columns, unknown_models)
+ passed = false
+ end
+ end
+
+ # Only the reference is asserted against the full schema; every
+ # other converter writes a subset of the schema by design.
+ missing = missing_columns(expected, results.fetch(REFERENCE_CONVERTER).written_columns)
+ if missing.any?
+ report_missing(expected, missing)
+ passed = false
+ end
+
+ if passed
+ column_count = expected.values.sum { |model| model.columns.size }
+ puts "✓ The #{REFERENCE_CONVERTER} converter covers all #{column_count} IntermediateDB columns across #{expected.size} tables.".green
+ end
+
+ passed
+ end
+
+ private
+
+ # @return [Hash{String => Array<Symbol>}] uncovered columns per model,
+ # only for models that have at least one.
+ def missing_columns(expected, written)
+ expected.each_with_object({}) do |(model_name, model), missing|
+ uncovered = model.columns - written.fetch(model_name, Set.new).to_a
+ missing[model_name] = uncovered if uncovered.any?
+ end
+ end
+
+ # @return [Hash{String => Array<Symbol>}] written columns the model's
+ # `create` doesn't accept, per model. Models outside the generated
+ # contract (manual models like Upload) have a bespoke API and are
+ # skipped.
+ def unknown_columns(expected, written)
+ written.each_with_object({}) do |(model_name, columns), unknown|
+ model = expected[model_name]
+ next unless model
+
+ extra = columns.to_a - model.columns
+ unknown[model_name] = extra.sort if extra.any?
+ end
+ end
+
+ def report_unknown(converter_name, expected, unknown_columns, unknown_models)
+ if unknown_columns.any?
+ puts "✗ The #{converter_name} converter writes columns that don't exist in the IntermediateDB schema".red
+ puts " (stale call sites after a schema change?):"
+ puts
+
+ unknown_columns
+ .keys
+ .sort_by { |model_name| expected[model_name].table_name }
+ .each do |model_name|
+ puts " #{expected[model_name].table_name}: #{unknown_columns[model_name].join(", ")}"
+ end
+
+ puts
+ end
+
+ if unknown_models.any?
+ puts "✗ The #{converter_name} converter writes to IntermediateDB models that don't exist:".red
+ puts
+
+ unknown_models.keys.sort.each do |model_name|
+ puts " #{model_name} (#{unknown_models[model_name].join(", ")})"
+ end
+
+ puts
+ end
+ end
+
+ def report_missing(expected, missing)
+ puts "✗ The #{REFERENCE_CONVERTER} converter does not write every IntermediateDB column.".red
+ puts " Acknowledge each column in the converter (pass it explicitly, `column: nil` if the source has no value):"
+ puts
+
+ column_count = 0
+ missing
+ .keys
+ .sort_by { |model_name| expected[model_name].table_name }
+ .each do |model_name|
+ columns = missing[model_name].sort
+ column_count += columns.size
+ puts " #{expected[model_name].table_name}: #{columns.join(", ")}"
+ end
+
+ puts
+ puts "#{column_count} #{"column".pluralize(column_count)} across #{missing.size} #{"table".pluralize(missing.size)} not covered.".red
+ end
+
+ def analyze(converter_name)
+ ConverterAnalyzer.new(Migrations::Converters.path_of(converter_name)).analyze
+ end
+
+ def ensure_reference_present!(converters)
+ return if converters.include?(REFERENCE_CONVERTER)
+
+ raise Error, <<~MSG.strip
+ Reference converter '#{REFERENCE_CONVERTER}' was not found among the discovered converters: #{converters.join(", ")}.
+ The coverage check cannot run without it.
+ MSG
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/tooling/lib/migrations/tooling/coverage/schema_columns.rb b/migrations/tooling/lib/migrations/tooling/coverage/schema_columns.rb
new file mode 100644
index 00000000000..010029e35e9
--- /dev/null
+++ b/migrations/tooling/lib/migrations/tooling/coverage/schema_columns.rb
@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Tooling
+ module Coverage
+ # Introspects the generated IntermediateDB models to discover the column
+ # contract each one expects.
+ #
+ # A model's columns are read from the keyword signature of its generated
+ # `.create` method: `:keyreq` parameters are required, `:key` parameters are
+ # optional. Required columns are already enforced by Ruby (omitting one
+ # raises), so the value this check adds is over the optional ones — but both
+ # are part of the contract and returned here.
+ #
+ # Only models produced by `disco schema generate` are part of the contract:
+ # manual (hand-written) models such as `Upload` and `LogEntry` are excluded,
+ # because they intentionally expose a bespoke API rather than the generated
+ # column-per-keyword `.create`.
+ module SchemaColumns
+ # Substring of the header `disco schema generate` writes into every
+ # generated model file. Manual models lack it, which is how we tell the
+ # two apart from migrations-core alone. Shared with the generator and the
+ # artifacts checker so the three agree on what marks a generated file.
+ GENERATED_MARKER = Schema::DSL::GeneratedFiles::MARKER
+ private_constant :GENERATED_MARKER
+
+ Model =
+ Data.define(:name, :required, :optional) do
+ # All columns the generated `.create` accepts, required first.
+ def columns
+ required + optional
+ end
+
+ # The IntermediateDB table the model writes to, e.g.
+ # `UserCustomField` => `user_custom_fields`. Display only.
+ def table_name
+ name.underscore.pluralize
+ end
+ end
+
+ # @return [Hash{String => Model}] generated models keyed by their constant
+ # name (e.g. "User"), in sorted order.
+ def self.call
+ namespace = Migrations::Database::IntermediateDB
+
+ namespace
+ .constants(false)
+ .sort
+ .each_with_object({}) do |const_name, models|
+ model = build_model(namespace, const_name)
+ models[model.name] = model if model
+ end
+ end
+
+ def self.build_model(namespace, const_name)
+ value = namespace.const_get(const_name)
+ return unless value.is_a?(Module)
+ return unless value.respond_to?(:create)
+ return unless generated?(value)
+
+ required = []
+ optional = []
+
+ value
+ .method(:create)
+ .parameters
+ .each do |type, name|
+ case type
+ when :keyreq
+ required << name
+ when :key
+ optional << name
+ end
+ end
+
+ Model.new(name: const_name.to_s, required:, optional:)
+ end
+ private_class_method :build_model
+
+ def self.generated?(model)
+ path, = model.method(:create).source_location
+ return false unless path
+
+ File.read(path).include?(GENERATED_MARKER)
+ end
+ private_class_method :generated?
+ end
+ end
+ end
+end
diff --git a/migrations/tooling/lib/migrations/tooling/register.rb b/migrations/tooling/lib/migrations/tooling/register.rb
index 882a736e7b8..0eb52e9a771 100644
--- a/migrations/tooling/lib/migrations/tooling/register.rb
+++ b/migrations/tooling/lib/migrations/tooling/register.rb
@@ -3,5 +3,11 @@
Migrations::CLI::Registry.register(
name: "schema",
command_class: "Migrations::Tooling::CLI::SchemaCommand",
- description: "Manage the IntermediateDB schema",
+ description: "Manage database schemas",
+)
+
+Migrations::CLI::Registry.register(
+ name: "check",
+ command_class: "Migrations::Tooling::CLI::CheckCommand",
+ description: "Run all schema and converter checks",
)
diff --git a/migrations/tooling/lib/migrations/tooling/schema.rb b/migrations/tooling/lib/migrations/tooling/schema.rb
index 9d102c33c7c..a0289ba48ac 100644
--- a/migrations/tooling/lib/migrations/tooling/schema.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema.rb
@@ -4,6 +4,7 @@ module Migrations
module Tooling
module Schema
Definition = Data.define(:tables, :enums)
+ GenerationResult = Data.define(:resolved, :deleted_files)
PreflightResult = Data.define(:resolved, :errors)
TableDefinition =
Data.define(
@@ -123,9 +124,11 @@ module Migrations
preflight(database:).errors
end
- def self.generate(database: :intermediate_db)
+ def self.generate(database: :intermediate_db, output_root: nil)
ensure_ready!(database:)
- DSL::Generator.new(self, database:).generate
+ generator = DSL::Generator.new(self, database:, output_root:)
+ resolved = generator.generate
+ GenerationResult.new(resolved:, deleted_files: generator.deleted_files)
end
def self.diff(database: :intermediate_db)
@@ -152,6 +155,13 @@ module Migrations
DSL::IgnoredFileEditor.new(config_path(database)).add_table(table_name, reason:)
end
+ # No database existence check here: removing entries of tables that no
+ # longer exist is the main use case.
+ def self.unignore_table(table_name, database: :intermediate_db)
+ ensure_ready!(database:)
+ DSL::IgnoredFileEditor.new(config_path(database)).remove_table(table_name)
+ end
+
# --- Lifecycle Methods ---
def self.ensure_ready!(database: :intermediate_db, refresh_manifest: true)
diff --git a/migrations/tooling/lib/migrations/tooling/schema/dsl/artifacts_checker.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/artifacts_checker.rb
new file mode 100644
index 00000000000..67bc7fb214a
--- /dev/null
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/artifacts_checker.rb
@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+
+module Migrations
+ module Tooling
+ module Schema
+ module DSL
+ # Compares the committed generated artifacts (SQL schema, models and
+ # enums) with what `disco schema generate` would produce right now.
+ # Generation happens in a temporary directory, so the working tree
+ # stays untouched.
+ class ArtifactsChecker
+ Result =
+ Data.define(:changed, :missing, :stale) do
+ def clean?
+ changed.empty? && missing.empty? && stale.empty?
+ end
+ end
+
+ def initialize(schema_module, database: :intermediate_db)
+ @schema = schema_module
+ @database = database
+ @output_config = schema_module.config.output_config
+ end
+
+ def check
+ Dir.mktmpdir("disco-schema-check-") do |tmp_root|
+ resolved = @schema.generate(database: @database, output_root: tmp_root).resolved
+ compare(resolved, tmp_root)
+ end
+ end
+
+ private
+
+ def compare(resolved, tmp_root)
+ changed = []
+ missing = []
+
+ # The same file list rooted at the two locations and in the same
+ # order, so each committed file lines up with its freshly generated
+ # counterpart.
+ committed = expected_paths(resolved, Migrations.root_path)
+ generated = expected_paths(resolved, tmp_root)
+
+ committed
+ .zip(generated)
+ .each do |committed_path, generated_path|
+ relative = relative_path(committed_path)
+
+ if !File.exist?(committed_path)
+ missing << relative
+ elsif File.read(committed_path) != File.read(generated_path)
+ changed << relative
+ end
+ end
+
+ Result.new(changed:, missing:, stale: stale_files(resolved))
+ end
+
+ # The files generation produces, rooted at `root`: the SQL schema plus
+ # one file per generated model and enum.
+ def expected_paths(resolved, root)
+ [File.expand_path(@output_config.schema_file, root)] +
+ GeneratedFiles.expected_paths(resolved, @output_config, root)
+ end
+
+ # Committed generated files that generation no longer produces,
+ # e.g. the model of a table that was removed from the config.
+ def stale_files(resolved)
+ root = Migrations.root_path
+ expected = GeneratedFiles.expected_paths(resolved, @output_config, root)
+
+ GeneratedFiles
+ .stale_paths(@output_config, root, expected)
+ .map { |path| relative_path(path) }
+ .sort
+ end
+
+ def relative_path(path)
+ Pathname.new(path).relative_path_from(Migrations.root_path).to_s
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/tooling/lib/migrations/tooling/schema/dsl/differ.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/differ.rb
index 2d31c670a98..24e567c8c65 100644
--- a/migrations/tooling/lib/migrations/tooling/schema/dsl/differ.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/differ.rb
@@ -13,9 +13,20 @@ module Migrations
:missing_columns,
:stale_ignored_columns,
:auto_ignored_columns,
- )
+ ) do
+ # Auto-ignored plugin columns are informational; everything else
+ # requires a config change.
+ def actionable?
+ unconfigured_columns.any? || missing_columns.any? || stale_ignored_columns.any?
+ end
+ end
DiffResult =
- Data.define(:unconfigured_tables, :missing_tables, :stale_ignored_tables, :table_diffs)
+ Data.define(:unconfigured_tables, :missing_tables, :stale_ignored_tables, :table_diffs) do
+ def actionable?
+ unconfigured_tables.any? || missing_tables.any? || stale_ignored_tables.any? ||
+ table_diffs.any?(&:actionable?)
+ end
+ end
class Differ
def initialize(schema_module)
diff --git a/migrations/tooling/lib/migrations/tooling/schema/dsl/generated_files.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/generated_files.rb
new file mode 100644
index 00000000000..27ff4d9ac51
--- /dev/null
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/generated_files.rb
@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Tooling
+ module Schema
+ module DSL
+ # Single source of truth for the set of files `disco schema generate`
+ # produces. Both the generator (which deletes files it no longer
+ # produces) and the artifacts checker (which detects drift between the
+ # committed files and a fresh generation) ask this module which files are
+ # expected and which committed files are now stale, so the two can't
+ # disagree about what "a generated file" is.
+ module GeneratedFiles
+ # The marker the generator writes into the header of every generated
+ # file (see Generator#file_header). Hand-written ("manual") models lack
+ # it, which is how generated files are told apart from them.
+ MARKER = "This file is auto-generated from the"
+
+ module_function
+
+ # Absolute paths of the model and enum files generation produces for
+ # `resolved`, rooted at `root`. Manual models are hand-written, so they
+ # are excluded. The SQL schema file is not included — callers that need
+ # it add it themselves.
+ def expected_paths(resolved, output_config, root)
+ models_dir = File.expand_path(output_config.models_directory, root)
+ enums_dir = File.expand_path(output_config.enums_directory, root)
+
+ model_paths =
+ resolved
+ .tables
+ .reject { |table| table.model_mode == :manual }
+ .map { |table| File.join(models_dir, ModelWriter.filename_for(table)) }
+
+ enum_paths =
+ resolved.enums.map { |enum| File.join(enums_dir, EnumWriter.filename_for(enum)) }
+
+ model_paths + enum_paths
+ end
+
+ # Absolute paths of generated `*.rb` files under `root`'s model and enum
+ # directories that the current definition no longer produces: they carry
+ # the auto-generated {MARKER} but are not in `expected` (the result of
+ # {expected_paths} for the same `root`). Hand-written models lack the
+ # marker and are never returned.
+ def stale_paths(output_config, root, expected)
+ expected = expected.to_set
+
+ [output_config.models_directory, output_config.enums_directory].uniq.flat_map do |dir|
+ Dir[File.join(File.expand_path(dir, root), "*.rb")].select do |path|
+ !expected.include?(path) && File.read(path).include?(MARKER)
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/tooling/lib/migrations/tooling/schema/dsl/generator.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/generator.rb
index 042b1b0d204..823007d1fef 100644
--- a/migrations/tooling/lib/migrations/tooling/schema/dsl/generator.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/generator.rb
@@ -10,10 +10,21 @@ module Migrations
CUSTOM_CODE_START = "# -- custom code --"
CUSTOM_CODE_END = "# -- end custom code --"
- def initialize(schema_module, database: :intermediate_db)
+ # Files deleted by the last `generate` run, relative to the output
+ # root.
+ attr_reader :deleted_files
+
+ # `output_root` overrides where the generated files are written
+ # (defaults to the repository root). Custom code of extended models
+ # is always read from the committed files, so generating into a
+ # temporary directory produces the same output as generating
+ # in place.
+ def initialize(schema_module, database: :intermediate_db, output_root: nil)
@schema = schema_module
@database = database
@output_config = schema_module.config.output_config
+ @output_root = output_root || Migrations.root_path
+ @deleted_files = []
end
def generate
@@ -23,6 +34,7 @@ module Migrations
generate_sql(resolved)
generate_enums(resolved)
generate_models(resolved)
+ delete_stale_files(resolved)
format_ruby_files!
resolved
end
@@ -81,7 +93,10 @@ module Migrations
when :manual
next
when :extended
- custom_code = extract_custom_code(path)
+ custom_code =
+ extract_custom_code(
+ File.join(source_path(@output_config.models_directory), filename),
+ )
File.open(path, "w") { |f| writer.output_table(table, f, custom_code:) }
else
File.open(path, "w") { |f| writer.output_table(table, f) }
@@ -89,6 +104,26 @@ module Migrations
end
end
+ # Deletes previously generated files that generation no longer
+ # produces, e.g. the model of a table that was removed from the
+ # config. Only files carrying the auto-generated header are
+ # touched; hand-written (manual) models don't have it.
+ def delete_stale_files(resolved)
+ expected = GeneratedFiles.expected_paths(resolved, @output_config, @output_root)
+
+ GeneratedFiles
+ .stale_paths(@output_config, @output_root, expected)
+ .each do |path|
+ File.delete(path)
+ @deleted_files << display_path(path)
+ end
+ end
+
+ def display_path(path)
+ relative = Pathname.new(path).relative_path_from(@output_root).to_s
+ relative.start_with?("..") ? path : relative
+ end
+
def extract_custom_code(path)
return nil unless File.exist?(path)
@@ -130,7 +165,13 @@ module Migrations
# formatting is best-effort; generation still succeeds
end
+ # Where generated files are written.
def expand_path(relative_path)
+ File.expand_path(relative_path, @output_root)
+ end
+
+ # Where the committed files live.
+ def source_path(relative_path)
File.expand_path(relative_path, Migrations.root_path)
end
@@ -139,7 +180,7 @@ module Migrations
begin
db_label = Helpers.db_label(@output_config.models_namespace)
<<~HEADER
- This file is auto-generated from the #{db_label} schema. To make changes,
+ #{GeneratedFiles::MARKER} #{db_label} schema. To make changes,
update the configuration files in "migrations/tooling/config/schema/" and then run
`migrations/bin/disco schema generate` to regenerate this file.
HEADER
diff --git a/migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_file_editor.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_file_editor.rb
index b408ade66a9..a4de81531ac 100644
--- a/migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_file_editor.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_file_editor.rb
@@ -43,6 +43,21 @@ module Migrations
format_file!
end
+ def remove_table(table_name)
+ table_name = table_name.to_s
+
+ if !File.exist?(@ignored_path)
+ raise ConfigError, "ignored.rb not found at #{@ignored_path}"
+ end
+
+ content = File.read(@ignored_path)
+ call = find_call_with_table(parsed_declaration(content).block, table_name)
+ raise ConfigError, "Table '#{table_name}' is not ignored" if call.nil?
+
+ File.write(@ignored_path, remove_table_from_call(content, call, table_name))
+ format_file!
+ end
+
private
def append_to_tables_group(content, tables_group, table_name)
@@ -67,7 +82,43 @@ module Migrations
content.byteslice(0, end_offset) + entry + content.byteslice(end_offset..)
end
- def parse_ignored_block(content)
+ def find_call_with_table(block_node, table_name)
+ body = block_node&.body
+ return nil unless body.is_a?(Prism::StatementsNode)
+
+ body.body.find do |statement|
+ statement.is_a?(Prism::CallNode) &&
+ %w[table tables].include?(statement.message.to_s) &&
+ table_names_from(statement).include?(table_name)
+ end
+ end
+
+ def remove_table_from_call(content, call_node, table_name)
+ remaining = table_names_from(call_node) - [table_name]
+ return delete_statement(content, call_node) if remaining.empty?
+
+ replacement = "tables " + remaining.map { |n| ":#{n}" }.join(", ")
+ keyword_source = trailing_keyword_source(call_node, content)
+ replacement += ", #{keyword_source}" if keyword_source
+
+ content.byteslice(0, call_node.location.start_offset) + replacement +
+ content.byteslice(call_node.location.end_offset..)
+ end
+
+ # Removes the statement including the lines it spans.
+ def delete_statement(content, node)
+ bytes = content.b
+ start_offset = node.location.start_offset
+ end_offset = node.location.end_offset
+
+ line_start = start_offset.zero? ? 0 : ((bytes.rindex("\n", start_offset - 1) || -1) + 1)
+ newline_after = bytes.index("\n", end_offset)
+ line_end = newline_after ? newline_after + 1 : bytes.bytesize
+
+ (bytes[0...line_start] + bytes[line_end..]).force_encoding(content.encoding)
+ end
+
+ def parsed_declaration(content)
result = Prism.parse(content)
if !result.success?
details = result.errors.map(&:message).join(", ")
@@ -80,6 +131,11 @@ module Migrations
"Could not find `Migrations::Tooling::Schema.ignored do ... end` in #{@ignored_path}"
end
+ declaration
+ end
+
+ def parse_ignored_block(content)
+ declaration = parsed_declaration(content)
last_tables = find_last_tables_group(declaration.block)
{
diff --git a/migrations/tooling/lib/migrations/tooling/schema/dsl/scaffolder.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/scaffolder.rb
index e82ed3c01dc..ff8104c9008 100644
--- a/migrations/tooling/lib/migrations/tooling/schema/dsl/scaffolder.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/scaffolder.rb
@@ -63,7 +63,7 @@ module Migrations
lines << " include_all"
lines << ""
- lines << " # TODO: Configure columns. Run `schema validate` to check."
+ lines << " # TODO: Configure columns. Run `#{Migrations::CLI::BIN} check schema` to check."
lines << ' # ignore :col1, :col2, reason: "..."'
lines << "end"
lines.join("\n") + "\n"
diff --git a/migrations/tooling/lib/migrations/tooling/schema/dsl/validator.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/validator.rb
index 83a6e446fbe..d8077d915fc 100644
--- a/migrations/tooling/lib/migrations/tooling/schema/dsl/validator.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/validator.rb
@@ -42,17 +42,9 @@ module Migrations
end
def validate_tables
- configured_table_names =
- @schema.tables.each_value.filter_map(&:source_table_name).to_set
- ignored_table_names = @scope.ignored_table_name_set
-
- # Check for unconfigured tables in database
- unconfigured = @db_table_names - configured_table_names - ignored_table_names
- if unconfigured.any?
- @errors << "Tables exist in database but are not configured or ignored: #{sort_and_join(unconfigured)}"
- end
-
- # Validate each configured table
+ # Unconfigured tables (present in the database but neither configured
+ # nor ignored) are reported as drift by the Differ — with
+ # `schema add`/`ignore` suggestions — not as a validity error here.
@schema.tables.each_value { |table_def| validate_table(table_def) }
end
@@ -81,7 +73,6 @@ module Migrations
validate_column_options
validate_added_columns
validate_ignored_columns
- validate_unconfigured_columns
validate_primary_key_columns
validate_index_columns
validate_enum_references
@@ -170,20 +161,6 @@ module Migrations
end
end
- def validate_unconfigured_columns
- return if @table_def.source_table_name.nil?
-
- configured_columns = @scope.effective_column_names(@table_def, @db_column_names)
- ignored_columns = @table_def.ignored_column_names.to_set
- unconfigured =
- @db_column_names - configured_columns - ignored_columns -
- @scope.globally_ignored_columns - auto_ignored_column_names
-
- if unconfigured.any?
- @errors << "Table '#{@table_def.name}': database columns are not configured or ignored: #{sort_and_join(unconfigured)}"
- end
- end
-
def validate_primary_key_columns
configured_primary_keys = @table_def.primary_key_columns || @db_primary_keys
return if configured_primary_keys.empty?
diff --git a/migrations/tooling/spec/lib/migrations/tooling/coverage/converter_analyzer_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/coverage/converter_analyzer_spec.rb
new file mode 100644
index 00000000000..142c1a3932e
--- /dev/null
+++ b/migrations/tooling/spec/lib/migrations/tooling/coverage/converter_analyzer_spec.rb
@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Tooling::Coverage::ConverterAnalyzer do
+ describe "#analyze" do
+ around do |example|
+ Dir.mktmpdir do |dir|
+ @converter_path = dir
+ example.run
+ end
+ end
+
+ def write_source(relative_path, source)
+ path = File.join(@converter_path, relative_path)
+ FileUtils.mkdir_p(File.dirname(path))
+ File.write(path, source)
+ end
+
+ it "unions columns for a model written across multiple files" do
+ write_source("steps/users.rb", "IntermediateDB::User.create(username: 'a')")
+ write_source("steps/more_users.rb", "IntermediateDB::User.create(name: 'n', trust_level: 1)")
+
+ result = described_class.new(@converter_path).analyze
+
+ expect(result.written_columns["User"]).to contain_exactly(:username, :name, :trust_level)
+ end
+
+ it "collects columns per model across the whole converter tree" do
+ write_source("steps/users.rb", "IntermediateDB::User.create(username: 'a')")
+ write_source("helpers/badges.rb", "IntermediateDB::Badge.create(name: 'b', original_id: 1)")
+
+ result = described_class.new(@converter_path).analyze
+
+ expect(result.written_columns.keys).to contain_exactly("User", "Badge")
+ end
+
+ it "collects unknown models with their call site locations" do
+ write_source("steps/old.rb", "IntermediateDB::RemovedModel.create(foo: 1)")
+
+ result = described_class.new(@converter_path).analyze
+
+ expect(result.written_columns).to be_empty
+ expect(result.unknown_models.keys).to contain_exactly("RemovedModel")
+ expect(result.unknown_models["RemovedModel"].first).to end_with("steps/old.rb:1")
+ end
+
+ it "returns an empty result when no .create calls are present" do
+ write_source("steps/noop.rb", "puts 'nothing here'")
+
+ result = described_class.new(@converter_path).analyze
+
+ expect(result.written_columns).to be_empty
+ expect(result.unknown_models).to be_empty
+ end
+
+ it "fails loudly when any source contains an unverifiable call site" do
+ write_source("steps/users.rb", "IntermediateDB::User.create(**attributes)")
+
+ expect { described_class.new(@converter_path).analyze }.to raise_error(
+ Migrations::Tooling::Coverage::AnalysisError,
+ )
+ end
+ end
+end
diff --git a/migrations/tooling/spec/lib/migrations/tooling/coverage/create_call_scanner_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/coverage/create_call_scanner_spec.rb
new file mode 100644
index 00000000000..f70adfc7e6b
--- /dev/null
+++ b/migrations/tooling/spec/lib/migrations/tooling/coverage/create_call_scanner_spec.rb
@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Tooling::Coverage::CreateCallScanner do
+ describe ".scan" do
+ def scan(source)
+ described_class.scan(source, path: "(spec)")
+ end
+
+ it "collects keyword names from a bare IntermediateDB receiver" do
+ result = scan("IntermediateDB::User.create(username: 'a', trust_level: 1)")
+
+ expect(result.columns["User"]).to contain_exactly(:username, :trust_level)
+ end
+
+ it "matches the trailing segment regardless of leading qualification" do
+ bare = scan("IntermediateDB::User.create(username: 'a')")
+ qualified = scan("Migrations::Database::IntermediateDB::User.create(name: 'n')")
+
+ expect(bare.columns.keys).to contain_exactly("User")
+ expect(qualified.columns.keys).to contain_exactly("User")
+ end
+
+ it "unions columns across all call sites of the same model" do
+ source = <<~RUBY
+ if condition
+ IntermediateDB::User.create(username: "a", name: "n")
+ else
+ IntermediateDB::User.create(username: "a", trust_level: 1)
+ end
+ RUBY
+
+ expect(scan(source).columns["User"]).to contain_exactly(:username, :name, :trust_level)
+ end
+
+ it "ignores .create calls on constants outside IntermediateDB" do
+ result = scan("Other::User.create(foo: 1)\nSomeClass.create(bar: 2)")
+
+ expect(result.columns).to be_empty
+ expect(result.unknown_models).to be_empty
+ end
+
+ it "records IntermediateDB constants that do not resolve to a model as unknown" do
+ result = scan("x = 1\nIntermediateDB::NotAModel.create(foo: 1)")
+
+ expect(result.columns).to be_empty
+ expect(result.unknown_models).to eq("NotAModel" => ["(spec):2"])
+ end
+
+ it "records every call site of an unknown model" do
+ source = <<~RUBY
+ IntermediateDB::NotAModel.create(foo: 1)
+ IntermediateDB::NotAModel.create(bar: 2)
+ RUBY
+
+ expect(scan(source).unknown_models).to eq("NotAModel" => %w[(spec):1 (spec):2])
+ end
+
+ it "ignores methods other than create, such as Upload.create_for_url" do
+ result = scan("IntermediateDB::Upload.create_for_url(url: 'x')")
+
+ expect(result.columns).to be_empty
+ expect(result.unknown_models).to be_empty
+ end
+
+ it "raises when a call site passes a ** splat" do
+ expect { scan("IntermediateDB::User.create(**attributes)") }.to raise_error(
+ Migrations::Tooling::Coverage::AnalysisError,
+ /splat/,
+ )
+ end
+
+ it "raises when a call site passes a non-literal keyword" do
+ expect { scan("IntermediateDB::User.create(dynamic => 1)") }.to raise_error(
+ Migrations::Tooling::Coverage::AnalysisError,
+ /non-literal keyword/,
+ )
+ end
+
+ it "raises when a call site passes an interpolated symbol keyword" do
+ expect { scan('IntermediateDB::User.create("col_#{x}": 1)') }.to raise_error(
+ Migrations::Tooling::Coverage::AnalysisError,
+ /non-literal keyword/,
+ )
+ end
+
+ it "raises with the source path when the source cannot be parsed" do
+ expect { described_class.scan("def broken(", path: "steps/users.rb") }.to raise_error(
+ Migrations::Tooling::Coverage::AnalysisError,
+ %r{steps/users\.rb},
+ )
+ end
+ end
+end
diff --git a/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb
new file mode 100644
index 00000000000..b9e30ee5e18
--- /dev/null
+++ b/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb
@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Tooling::Coverage::ReferenceCheck do
+ subject(:check) { described_class.new }
+
+ let(:schema_columns) { Migrations::Tooling::Coverage::SchemaColumns }
+ let(:analyzer_class) { Migrations::Tooling::Coverage::ConverterAnalyzer }
+
+ def model(name, required: [], optional: [])
+ Migrations::Tooling::Coverage::SchemaColumns::Model.new(name:, required:, optional:)
+ end
+
+ def analysis(written: {}, unknown: {})
+ Migrations::Tooling::Coverage::ConverterAnalyzer::Result.new(
+ written_columns: written.transform_values { |columns| Set.new(columns) },
+ unknown_models: unknown,
+ )
+ end
+
+ # `results` maps converter name => analysis(...). The reference converter is
+ # `discourse`, so include it unless a test is about its absence.
+ #
+ # `Migrations::Converters` lives in the converters gem, which isn't loaded in
+ # the tooling gem's isolated specs, so it is stubbed in rather than required.
+ def stub_coverage(results, schema:)
+ converters = Module.new
+ allow(converters).to receive(:names).and_return(results.keys)
+ allow(converters).to receive(:path_of) { |name| name }
+ stub_const("Migrations::Converters", converters)
+
+ allow(analyzer_class).to receive(:new) do |path|
+ instance_double(analyzer_class, analyze: results.fetch(path))
+ end
+ allow(schema_columns).to receive(:call).and_return(schema)
+ end
+
+ it "passes when the reference converter writes every column" do
+ stub_coverage(
+ { "discourse" => analysis(written: { "User" => %i[id name] }) },
+ schema: {
+ "User" => model("User", required: [:id], optional: [:name]),
+ },
+ )
+
+ passed = nil
+ expect { passed = check.run }.to output(/covers all/).to_stdout
+ expect(passed).to be true
+ end
+
+ it "fails when the reference converter is missing a column" do
+ stub_coverage(
+ { "discourse" => analysis(written: { "User" => [:id] }) },
+ schema: {
+ "User" => model("User", required: [:id], optional: [:name]),
+ },
+ )
+
+ passed = nil
+ expect { passed = check.run }.to output(/does not write every/).to_stdout
+ expect(passed).to be false
+ end
+
+ it "does not fail when a non-reference converter covers only a subset" do
+ stub_coverage(
+ {
+ "discourse" => analysis(written: { "User" => %i[id name] }),
+ "phpbb" => analysis(written: { "User" => [:id] }),
+ },
+ schema: {
+ "User" => model("User", optional: %i[id name]),
+ },
+ )
+
+ passed = nil
+ expect { passed = check.run }.to output.to_stdout
+ expect(passed).to be true
+ end
+
+ it "fails when any converter writes a column the schema doesn't have" do
+ stub_coverage(
+ {
+ "discourse" => analysis(written: { "User" => [:id] }),
+ "phpbb" => analysis(written: { "User" => %i[id bogus] }),
+ },
+ schema: {
+ "User" => model("User", optional: [:id]),
+ },
+ )
+
+ passed = nil
+ expect { passed = check.run }.to output(/columns that don't exist/).to_stdout
+ expect(passed).to be false
+ end
+
+ it "fails when any converter writes to a model the schema doesn't have" do
+ stub_coverage(
+ {
+ "discourse" => analysis(written: { "User" => [:id] }),
+ "phpbb" => analysis(written: { "User" => [:id] }, unknown: { "Ghost" => ["steps/x.rb:1"] }),
+ },
+ schema: {
+ "User" => model("User", optional: [:id]),
+ },
+ )
+
+ passed = nil
+ expect { passed = check.run }.to output(/models that don't exist/).to_stdout
+ expect(passed).to be false
+ end
+
+ it "raises when the reference converter is not among the discovered converters" do
+ stub_coverage(
+ { "phpbb" => analysis(written: { "User" => [:id] }) },
+ schema: {
+ "User" => model("User", optional: [:id]),
+ },
+ )
+
+ expect { check.run }.to raise_error(described_class::Error, /discourse/)
+ end
+end
diff --git a/migrations/tooling/spec/lib/migrations/tooling/coverage/schema_columns_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/coverage/schema_columns_spec.rb
new file mode 100644
index 00000000000..87eef3098b2
--- /dev/null
+++ b/migrations/tooling/spec/lib/migrations/tooling/coverage/schema_columns_spec.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Tooling::Coverage::SchemaColumns do
+ describe ".call" do
+ subject(:models) { described_class.call }
+
+ it "includes generated models keyed by their constant name" do
+ expect(models).to include("User", "Badge", "UserCustomField")
+ end
+
+ it "excludes manual (hand-written) models" do
+ expect(models).not_to include("Upload", "LogEntry")
+ end
+
+ it "splits required and optional columns from the create signature" do
+ user = models.fetch("User")
+
+ expect(user.required).to include(:original_id, :username, :created_at, :trust_level)
+ expect(user.optional).to include(:active, :name)
+ expect(user.required).not_to include(:active)
+ end
+
+ it "exposes all columns and the table name for a model" do
+ badge = models.fetch("Badge")
+
+ expect(badge.columns).to include(:original_id, :name)
+ expect(badge.table_name).to eq("badges")
+ end
+ end
+end
diff --git a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/diff_result_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/diff_result_spec.rb
new file mode 100644
index 00000000000..a27fb2e2a74
--- /dev/null
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/diff_result_spec.rb
@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Tooling::Schema::DSL::Differ do
+ # Specs for the diff result types defined alongside the Differ.
+
+ def table_diff(unconfigured: [], missing: [], stale: [], auto_ignored: [])
+ Migrations::Tooling::Schema::DSL::TableDiff.new(
+ table_name: "users",
+ unconfigured_columns: unconfigured,
+ missing_columns: missing,
+ stale_ignored_columns: stale,
+ auto_ignored_columns: auto_ignored,
+ )
+ end
+
+ def diff_result(unconfigured_tables: [], missing_tables: [], stale_tables: [], table_diffs: [])
+ Migrations::Tooling::Schema::DSL::DiffResult.new(
+ unconfigured_tables:,
+ missing_tables:,
+ stale_ignored_tables: stale_tables,
+ table_diffs:,
+ )
+ end
+
+ def column(name)
+ Migrations::Tooling::Schema::DSL::ColumnInfo.new(name:, plugin: nil)
+ end
+
+ def table(name)
+ Migrations::Tooling::Schema::DSL::TableInfo.new(name:, plugin: nil)
+ end
+
+ describe "TableDiff#actionable?" do
+ it "is not actionable when empty" do
+ expect(table_diff).not_to be_actionable
+ end
+
+ it "is actionable for unconfigured, missing or stale ignored columns" do
+ expect(table_diff(unconfigured: [column("locale")])).to be_actionable
+ expect(table_diff(missing: [column("locale")])).to be_actionable
+ expect(table_diff(stale: [column("locale")])).to be_actionable
+ end
+
+ it "is not actionable for auto-ignored plugin columns" do
+ expect(table_diff(auto_ignored: [column("assignable_level")])).not_to be_actionable
+ end
+ end
+
+ describe "DiffResult#actionable?" do
+ it "is not actionable when empty" do
+ expect(diff_result).not_to be_actionable
+ end
+
+ it "is actionable for table-level differences" do
+ expect(diff_result(unconfigured_tables: [table("users")])).to be_actionable
+ expect(diff_result(missing_tables: [table("users")])).to be_actionable
+ expect(diff_result(stale_tables: [table("users")])).to be_actionable
+ end
+
+ it "is actionable when a table diff is actionable" do
+ expect(diff_result(table_diffs: [table_diff(unconfigured: [column("locale")])])).to(
+ be_actionable,
+ )
+ end
+
+ it "is not actionable when table diffs only contain auto-ignored columns" do
+ expect(diff_result(table_diffs: [table_diff(auto_ignored: [column("foo")])])).not_to(
+ be_actionable,
+ )
+ end
+ end
+end
diff --git a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rb
index 77618577eb2..42d5d1a120d 100644
--- a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rb
@@ -101,6 +101,32 @@ RSpec.describe Migrations::Tooling::Schema::DSL::Differ, :rails do
expect(names).to contain_exactly("comments")
end
+ it "excludes plugin-ignored tables from unconfigured" do
+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)
+ allow(manifest).to receive(:available?).and_return(true)
+ allow(manifest).to receive(:tables_for_plugin).with("chat").and_return(
+ %w[chat_channels chat_messages],
+ )
+ allow(manifest).to receive(:columns_for_plugin).with("chat", table: "users").and_return([])
+ allow(manifest).to receive(:plugin_for_table).and_return(nil)
+ allow(Migrations::Tooling::Schema).to receive(:plugin_manifest).and_return(manifest)
+
+ Migrations::Tooling::Schema.table(:users) { include :id }
+ Migrations::Tooling::Schema.ignored { plugin :chat, "Not migrating" }
+
+ stub_database(
+ connection,
+ db_tables: %i[users chat_channels chat_messages],
+ table_columns: {
+ users: %i[id],
+ },
+ )
+
+ result = described_class.new(Migrations::Tooling::Schema).diff
+
+ expect(result.unconfigured_tables).to be_empty
+ end
+
it "detects missing tables" do
stub_plugin_manifest_unavailable
diff --git a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/generated_files_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/generated_files_spec.rb
new file mode 100644
index 00000000000..70ff4d0fe0d
--- /dev/null
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/generated_files_spec.rb
@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Tooling::Schema::DSL::GeneratedFiles do
+ # Minimal stand-ins: ModelWriter.filename_for only reads `name`/`model_mode`,
+ # EnumWriter.filename_for only reads `name`.
+ def table(name, model_mode = nil)
+ Struct.new(:name, :model_mode).new(name, model_mode)
+ end
+
+ def enum(name)
+ Struct.new(:name).new(name)
+ end
+
+ def resolved(tables: [], enums: [])
+ Struct.new(:tables, :enums).new(tables, enums)
+ end
+
+ # GeneratedFiles only reads the model/enum directories off the output config.
+ def output_config(models: "lib/models", enums: "lib/enums")
+ Struct.new(:models_directory, :enums_directory).new(models, enums)
+ end
+
+ let(:definition) do
+ resolved(
+ tables: [table("users"), table("uploads", :manual), table("badges", :extended)],
+ enums: [enum("visibility")],
+ )
+ end
+
+ describe ".expected_paths" do
+ it "returns absolute model and enum paths rooted at root, excluding manual models" do
+ paths = described_class.expected_paths(definition, output_config, "/repo")
+
+ expect(paths).to contain_exactly(
+ "/repo/lib/models/user.rb",
+ "/repo/lib/models/badge.rb",
+ "/repo/lib/enums/visibility.rb",
+ )
+ end
+
+ it "honours an absolute output directory by ignoring the root" do
+ config = output_config(models: "/abs/models", enums: "/abs/enums")
+
+ paths = described_class.expected_paths(definition, config, "/repo")
+
+ expect(paths).to include("/abs/models/user.rb", "/abs/enums/visibility.rb")
+ end
+ end
+
+ describe ".stale_paths" do
+ around do |example|
+ Dir.mktmpdir do |dir|
+ @root = dir
+ FileUtils.mkdir_p(File.join(dir, "lib/models"))
+ FileUtils.mkdir_p(File.join(dir, "lib/enums"))
+ example.run
+ end
+ end
+
+ def write(relative, contents)
+ File.write(File.join(@root, relative), contents)
+ end
+
+ def stale
+ config = output_config
+ expected = described_class.expected_paths(definition, config, @root)
+ described_class.stale_paths(config, @root, expected)
+ end
+
+ it "returns generated files that are no longer expected" do
+ write("lib/models/user.rb", "# #{described_class::MARKER} ...\n")
+ write("lib/models/old.rb", "# #{described_class::MARKER} ...\n")
+ write("lib/enums/gone.rb", "# #{described_class::MARKER} ...\n")
+
+ expect(stale).to contain_exactly(
+ File.join(@root, "lib/models/old.rb"),
+ File.join(@root, "lib/enums/gone.rb"),
+ )
+ end
+
+ it "ignores files that do not carry the generated marker" do
+ write("lib/models/manual.rb", "# hand-written, no marker\n")
+
+ expect(stale).to be_empty
+ end
+
+ it "does not report expected files as stale" do
+ write("lib/models/user.rb", "# #{described_class::MARKER} ...\n")
+ write("lib/models/badge.rb", "# #{described_class::MARKER} ...\n")
+
+ expect(stale).to be_empty
+ end
+ end
+end
diff --git a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/generator_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/generator_spec.rb
index 40644906060..b05f2b8f9fd 100644
--- a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/generator_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/generator_spec.rb
@@ -174,6 +174,41 @@ RSpec.describe Migrations::Tooling::Schema::DSL::Generator do
end
end
+ it "deletes generated files that are no longer produced" do
+ Dir.mktmpdir do |tmpdir|
+ paths = configure_output(tmpdir)
+ stub_validation_and_resolution(resolved_definition)
+
+ FileUtils.mkdir_p(paths[:models])
+ stale_path = File.join(paths[:models], "old_model.rb")
+ File.write(stale_path, "# This file is auto-generated from the Models schema.\n")
+
+ generator = described_class.new(Migrations::Tooling::Schema)
+ generator.generate
+
+ expect(File.exist?(stale_path)).to be false
+ expect(generator.deleted_files.size).to eq(1)
+ expect(generator.deleted_files.first).to end_with("old_model.rb")
+ end
+ end
+
+ it "keeps files without the auto-generated header" do
+ Dir.mktmpdir do |tmpdir|
+ paths = configure_output(tmpdir)
+ stub_validation_and_resolution(resolved_definition)
+
+ FileUtils.mkdir_p(paths[:models])
+ manual_path = File.join(paths[:models], "manual_model.rb")
+ File.write(manual_path, "# Hand-written model\n")
+
+ generator = described_class.new(Migrations::Tooling::Schema)
+ generator.generate
+
+ expect(File.exist?(manual_path)).to be true
+ expect(generator.deleted_files).to be_empty
+ end
+ end
+
it "generates model file with custom code markers for :extended mode" do
Dir.mktmpdir do |tmpdir|
paths = configure_output(tmpdir)
diff --git a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_file_editor_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_file_editor_spec.rb
index c5ca84be395..4f0f064dc41 100644
--- a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_file_editor_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_file_editor_spec.rb
@@ -75,4 +75,88 @@ RSpec.describe Migrations::Tooling::Schema::DSL::IgnoredFileEditor do
end
end
end
+
+ describe "#remove_table" do
+ it "removes a table from a tables group and preserves the reason" do
+ with_ignored_file(<<~RUBY) do |tmpdir, ignored_path|
+ Migrations::Tooling::Schema.ignored do
+ tables :a, :b, :c, reason: "legacy"
+ end
+ RUBY
+ allow(Migrations::Tooling::Schema::Helpers).to receive(:format_ruby_file)
+
+ described_class.new(tmpdir).remove_table(:b)
+
+ content = File.read(ignored_path)
+ expect(content).to include('tables :a, :c, reason: "legacy"')
+ expect(content).not_to include(":b")
+ end
+ end
+
+ it "removes a standalone table entry entirely" do
+ with_ignored_file(<<~RUBY) do |tmpdir, ignored_path|
+ Migrations::Tooling::Schema.ignored do
+ table :a, "Legacy table"
+ tables :b, :c
+ end
+ RUBY
+ allow(Migrations::Tooling::Schema::Helpers).to receive(:format_ruby_file)
+
+ described_class.new(tmpdir).remove_table(:a)
+
+ content = File.read(ignored_path)
+ expect(content).not_to include(":a,")
+ expect(content).not_to include("Legacy table")
+ expect(content).to include("tables :b, :c")
+ end
+ end
+
+ it "removes the whole group when the last table is removed" do
+ with_ignored_file(<<~RUBY) do |tmpdir, ignored_path|
+ Migrations::Tooling::Schema.ignored do
+ tables :a
+ tables :b, :c
+ end
+ RUBY
+ allow(Migrations::Tooling::Schema::Helpers).to receive(:format_ruby_file)
+
+ described_class.new(tmpdir).remove_table(:a)
+
+ content = File.read(ignored_path)
+ expect(content).not_to include("tables :a")
+ expect(content).to include("tables :b, :c")
+ end
+ end
+
+ it "removes a table from a multi-line tables group" do
+ with_ignored_file(<<~RUBY) do |tmpdir, ignored_path|
+ Migrations::Tooling::Schema.ignored do
+ tables :aaa,
+ :bbb,
+ :ccc
+ end
+ RUBY
+ allow(Migrations::Tooling::Schema::Helpers).to receive(:format_ruby_file)
+
+ described_class.new(tmpdir).remove_table(:aaa)
+
+ content = File.read(ignored_path)
+ expect(content).to include("tables :bbb, :ccc")
+ expect(content).not_to include(":aaa")
+ end
+ end
+
+ it "raises when the table is not ignored" do
+ with_ignored_file(<<~RUBY) do |tmpdir, _ignored_path|
+ Migrations::Tooling::Schema.ignored do
+ tables :a, :b
+ end
+ RUBY
+ expect { described_class.new(tmpdir).remove_table(:c) }.to raise_error(
+ Migrations::Tooling::Schema::ConfigError,
+ /is not ignored/,
+ )
+ end
+ end
+ end
end
diff --git a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/validator_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/validator_spec.rb
index cb6c21171e6..45faefa355b 100644
--- a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/validator_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/validator_spec.rb
@@ -72,50 +72,6 @@ RSpec.describe Migrations::Tooling::Schema::DSL::Validator, :rails do
expect(errors).to be_empty
end
- it "detects unconfigured tables in database" do
- schema = build_schema(tables: { users: proc { include_all } })
-
- stub_database(
- connection,
- db_tables: %i[users posts comments],
- table_columns: {
- users: {
- id: {
- type: :integer,
- },
- },
- },
- )
-
- errors = described_class.new(schema).validate
- expect(errors).to include(match(/not configured or ignored.*comments.*posts/m))
- end
-
- it "does not report ignored tables as unconfigured" do
- schema =
- build_schema(
- tables: {
- users: proc { include_all },
- },
- ignored: proc { table :posts, "not needed" },
- )
-
- stub_database(
- connection,
- db_tables: %i[users posts],
- table_columns: {
- users: {
- id: {
- type: :integer,
- },
- },
- },
- )
-
- errors = described_class.new(schema).validate
- expect(errors).to be_empty
- end
-
it "detects source table not existing in database" do
schema = build_schema(tables: { users: proc { include_all } })
@@ -144,31 +100,6 @@ RSpec.describe Migrations::Tooling::Schema::DSL::Validator, :rails do
expect(errors).to include(match(/included columns do not exist.*nonexistent/))
end
- it "detects database columns that are not configured or ignored" do
- schema = build_schema(tables: { users: proc { include :id, :username } })
-
- stub_database(
- connection,
- db_tables: %i[users],
- table_columns: {
- users: {
- id: {
- type: :integer,
- },
- username: {
- type: :text,
- },
- email: {
- type: :text,
- },
- },
- },
- )
-
- errors = described_class.new(schema).validate
- expect(errors).to include(match(/database columns are not configured or ignored.*email/))
- end
-
it "detects column options referencing missing columns" do
schema =
build_schema(
@@ -390,37 +321,6 @@ RSpec.describe Migrations::Tooling::Schema::DSL::Validator, :rails do
expect(errors).to contain_exactly(match(/source table 'users' does not exist in database/))
end
- it "does not report copy_structure_from source tables as unconfigured" do
- schema =
- build_schema(
- tables: {
- user_archive:
- proc do
- copy_structure_from :users
- include :id
- end,
- },
- )
-
- stub_database(
- connection,
- db_tables: %i[users],
- table_columns: {
- users: {
- id: {
- type: :integer,
- },
- },
- },
- primary_keys: {
- users: ["id"],
- },
- )
-
- errors = described_class.new(schema).validate
- expect(errors).to be_empty
- end
-
it "passes when index references added columns" do
schema =
build_schema(
@@ -477,40 +377,6 @@ RSpec.describe Migrations::Tooling::Schema::DSL::Validator, :rails do
expect(errors).to include(match(/primary key columns are not configured.*id/))
end
- it "does not report plugin-ignored tables as unconfigured" do
- manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)
- allow(manifest).to receive(:available?).and_return(true)
- allow(manifest).to receive(:tables_for_plugin).with("chat").and_return(
- %w[chat_channels chat_messages],
- )
- allow(manifest).to receive(:columns_for_plugin).with("chat", table: "users").and_return([])
- allow(manifest).to receive(:plugin_for_table).with("users").and_return(nil)
- allow(Migrations::Tooling::Schema).to receive(:plugin_manifest).and_return(manifest)
-
- schema =
- build_schema(
- tables: {
- users: proc { include_all },
- },
- ignored: proc { plugin :chat, "Not migrating" },
- )
-
- stub_database(
- connection,
- db_tables: %i[users chat_channels chat_messages],
- table_columns: {
- users: {
- id: {
- type: :integer,
- },
- },
- },
- )
-
- errors = described_class.new(schema).validate
- expect(errors).to be_empty
- end
-
it "detects tables that are both configured and ignored" do
schema =
build_schema(