The personal finance app for everyone
Stronger security unique index and data migration
Zach Gollwitzerfe24117c509b
diff --git a/db/migrate/20250522174753_stronger_uniqueness_constraint_on_securities.rb b/db/migrate/20250522174753_stronger_uniqueness_constraint_on_securities.rbnew file mode 100644index 00000000..ac49330f--- /dev/null+++ b/db/migrate/20250522174753_stronger_uniqueness_constraint_on_securities.rb@@ -0,0 +1,13 @@+class StrongerUniquenessConstraintOnSecurities < ActiveRecord::Migration[7.2]+ def change+ remove_index :securities, [ :ticker, :exchange_operating_mic ], unique: true++ # Matches our ActiveRecord validation:+ # - uppercase ticker+ # - either exchange_operating_mic or empty string (unique index doesn't work with NULL values)+ add_index :securities,+ "UPPER(ticker), COALESCE(UPPER(exchange_operating_mic), '')",+ unique: true,+ name: "index_securities_on_ticker_and_exchange_operating_mic_unique"+ end+enddiff --git a/db/schema.rb b/db/schema.rbindex 9f079956..1d03733d 100644--- a/db/schema.rb+++ b/db/schema.rb@@ -10,7 +10,7 @@## It's strongly recommended that you check this file into your version control system.-ActiveRecord::Schema[7.2].define(version: 2025_05_21_112347) do+ActiveRecord::Schema[7.2].define(version: 2025_05_22_174753) do# These are extensions that must be enabled in order to support this databaseenable_extension "pgcrypto"enable_extension "plpgsql"@@ -517,9 +517,9 @@ ActiveRecord::Schema[7.2].define(version: 2025_05_21_112347) dot.datetime "failed_fetch_at"t.integer "failed_fetch_count", default: 0, null: falset.datetime "last_health_check_at"+ t.index "upper((ticker)::text), COALESCE(upper((exchange_operating_mic)::text), ''::text)", name: "index_securities_on_ticker_and_exchange_operating_mic_unique", unique: truet.index ["country_code"], name: "index_securities_on_country_code"t.index ["exchange_operating_mic"], name: "index_securities_on_exchange_operating_mic"- t.index ["ticker", "exchange_operating_mic"], name: "index_securities_on_ticker_and_exchange_operating_mic", unique: trueendcreate_table "security_prices", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|diff --git a/lib/tasks/data_migration.rake b/lib/tasks/data_migration.rakeindex c84dbab0..3b6882fd 100644--- a/lib/tasks/data_migration.rake+++ b/lib/tasks/data_migration.rake@@ -20,4 +20,63 @@ namespace :data_migration doputs "Error updating webhook for Plaid item #{item.plaid_id}: #{error.message}"endend++ desc "Migrate duplicate securities"+ # 2025-05-22: older data allowed multiple rows with the same+ # ticker / exchange_operating_mic (case-insensitive, NULLs collapsed).+ # This task:+ # 1. Finds each duplicate group+ # 2. Chooses the earliest-created row as the keeper+ # 3. Re-points holdings and trades to the keeper+ # 4. Destroys the duplicate (which also removes its prices)+ task migrate_duplicate_securities: :environment do+ puts "==> Scanning for duplicate securities…"++ duplicate_sets = Security+ .select("UPPER(ticker) AS up_ticker,+ COALESCE(UPPER(exchange_operating_mic), '') AS up_mic,+ COUNT(*) AS dup_count")+ .group("up_ticker, up_mic")+ .having("COUNT(*) > 1")+ .to_a++ puts "Found #{duplicate_sets.size} duplicate groups."++ duplicate_sets.each_with_index do |set, idx|+ # Fetch duplicates ordered by creation; the first row becomes keeper+ duplicates_scope = Security+ .where("UPPER(ticker) = ? AND COALESCE(UPPER(exchange_operating_mic), '') = ?",+ set.up_ticker, set.up_mic)+ .order(:created_at)++ keeper = duplicates_scope.first+ next unless keeper++ duplicates = duplicates_scope.offset(1)++ dup_ids = duplicates.ids++ # Skip if nothing to merge (defensive; shouldn't occur)+ next if dup_ids.empty?++ begin+ ActiveRecord::Base.transaction do+ updated_holdings = Holding.where(security_id: dup_ids).update_all(security_id: keeper.id)+ updated_trades = Trade.where(security_id: dup_ids).update_all(security_id: keeper.id)++ # Ensure no rows remain pointing at duplicates before deletion+ raise "Leftover holdings detected" if Holding.where(security_id: dup_ids).exists?+ raise "Leftover trades detected" if Trade.where(security_id: dup_ids).exists?++ duplicates.each(&:destroy!) # destroys its security_prices via dependent: :destroy+ end++ puts "[#{idx + 1}/#{duplicate_sets.size}] Merged #{dup_ids.join(', ')} → #{keeper.id} (#{updated_holdings} holdings, #{updated_trades} trades)"+ rescue => e+ puts "ERROR migrating #{dup_ids.join(', ')}: #{e.message}"+ end+ end++ puts "✅ Duplicate security migration complete."+ endend