The personal finance app for everyone
Enhance security information retrieval and handling (#1826)
Josh Pigford68d7cb5de6c9
diff --git a/app/jobs/fetch_security_info_job.rb b/app/jobs/fetch_security_info_job.rbindex 5eaafa43..aa64c169 100644--- a/app/jobs/fetch_security_info_job.rb+++ b/app/jobs/fetch_security_info_job.rb@@ -6,10 +6,13 @@ class FetchSecurityInfoJob < ApplicationJobsecurity = Security.find(security_id)- security_info_response = Security.security_info_provider.fetch_security_info(- ticker: security.ticker,- mic_code: security.exchange_mic- )+ params = {+ ticker: security.ticker+ }+ params[:mic_code] = security.exchange_mic if security.exchange_mic.present?+ params[:operating_mic] = security.exchange_operating_mic if security.exchange_operating_mic.present?++ security_info_response = Security.security_info_provider.fetch_security_info(**params)security.update(name: security_info_response.info.dig("name")diff --git a/app/models/account/trade_builder.rb b/app/models/account/trade_builder.rbindex 8e0f9ad7..704d851f 100644--- a/app/models/account/trade_builder.rb+++ b/app/models/account/trade_builder.rb@@ -111,13 +111,10 @@ class Account::TradeBuilderenddef security- ticker_symbol, exchange_mic, exchange_acronym, exchange_country_code = ticker.split("|")+ ticker_symbol, exchange_operating_mic = ticker.split("|")- security = Security.find_or_create_by(ticker: ticker_symbol, exchange_mic: exchange_mic, country_code: exchange_country_code)- security.update(exchange_acronym: exchange_acronym)-- FetchSecurityInfoJob.perform_later(security.id)-- security+ Security.find_or_create_by(ticker: ticker_symbol, exchange_operating_mic: exchange_operating_mic) do |s|+ FetchSecurityInfoJob.perform_later(s.id)+ endendenddiff --git a/app/models/demo/generator.rb b/app/models/demo/generator.rbindex 22038857..283ae35b 100644--- a/app/models/demo/generator.rb+++ b/app/models/demo/generator.rb@@ -231,12 +231,12 @@ class Demo::Generatordef load_securities!# Create an unknown security to simulate edge cases- Security.create! ticker: "UNKNOWN", name: "Unknown Demo Stock", exchange_mic: "UNKNOWN"+ Security.create! ticker: "UNKNOWN", name: "Unknown Demo Stock"securities = [- { ticker: "AAPL", exchange_mic: "NASDAQ", name: "Apple Inc.", reference_price: 210 },- { ticker: "TM", exchange_mic: "NYSE", name: "Toyota Motor Corporation", reference_price: 202 },- { ticker: "MSFT", exchange_mic: "NASDAQ", name: "Microsoft Corporation", reference_price: 455 }+ { ticker: "AAPL", exchange_mic: "XNGS", exchange_operating_mic: "XNAS", name: "Apple Inc.", reference_price: 210 },+ { ticker: "TM", exchange_mic: "XNYS", exchange_operating_mic: "XNYS", name: "Toyota Motor Corporation", reference_price: 202 },+ { ticker: "MSFT", exchange_mic: "XNGS", exchange_operating_mic: "XNAS", name: "Microsoft Corporation", reference_price: 455 }]securities.each do |security_attributes|diff --git a/app/models/plaid_investment_sync.rb b/app/models/plaid_investment_sync.rbindex 8c1d9805..bcb1f330 100644--- a/app/models/plaid_investment_sync.rb+++ b/app/models/plaid_investment_sync.rb@@ -82,12 +82,15 @@ class PlaidInvestmentSyncendreturn [ nil, nil ] if plaid_security.nil? || plaid_security.ticker_symbol.blank?+ return [ nil, plaid_security ] if plaid_security.ticker_symbol == "CUR:USD" # internally, we do not consider cash a security and track it separately+ operating_mic = plaid_security.market_identifier_code++ # Find any matching securitysecurity = Security.find_or_create_by!(ticker: plaid_security.ticker_symbol,- exchange_mic: plaid_security.market_identifier_code || "XNAS",- country_code: "US"- ) unless plaid_security.ticker_symbol == "CUR:USD" # internally, we do not consider cash a security and track it separately+ exchange_operating_mic: operating_mic+ )[ security, plaid_security ]enddiff --git a/app/models/provider/synth.rb b/app/models/provider/synth.rbindex a5356a0f..65deaae5 100644--- a/app/models/provider/synth.rb+++ b/app/models/provider/synth.rb@@ -145,6 +145,7 @@ class Provider::Synthlogo_url: security.dig("logo_url"),exchange_acronym: security.dig("exchange", "acronym"),exchange_mic: security.dig("exchange", "mic_code"),+ exchange_operating_mic: security.dig("exchange", "operating_mic_code"),country_code: security.dig("exchange", "country_code")}end@@ -155,9 +156,10 @@ class Provider::Synthraw_response: responseend- def fetch_security_info(ticker:, mic_code:)+ def fetch_security_info(ticker:, mic_code: nil, operating_mic: nil)response = client.get("#{base_url}/tickers/#{ticker}") do |req|- req.params["mic_code"] = mic_code+ req.params["mic_code"] = mic_code if mic_code.present?+ req.params["operating_mic"] = operating_mic if operating_mic.present?endparsed = JSON.parse(response.body)diff --git a/app/models/security.rb b/app/models/security.rbindex d2ce6387..4a2bd1a7 100644--- a/app/models/security.rb+++ b/app/models/security.rb@@ -6,7 +6,7 @@ class Security < ApplicationRecordhas_many :prices, dependent: :destroyvalidates :ticker, presence: true- validates :ticker, uniqueness: { scope: :exchange_mic, case_sensitive: false }+ validates :ticker, uniqueness: { scope: :exchange_operating_mic, case_sensitive: false }class << selfdef search(query)@@ -30,11 +30,15 @@ class Security < ApplicationRecordname: name,logo_url: logo_url,exchange_acronym: exchange_acronym,- exchange_mic: exchange_mic,+ exchange_operating_mic: exchange_operating_mic,exchange_country_code: country_code)end+ def has_prices?+ exchange_operating_mic.present?+ end+privatedef upcase_tickerdiff --git a/app/models/security/price/provided.rb b/app/models/security/price/provided.rbindex 02e5ce14..e2a99774 100644--- a/app/models/security/price/provided.rb+++ b/app/models/security/price/provided.rb@@ -8,6 +8,7 @@ module Security::Price::Provideddef fetch_price_from_provider(security:, date:, cache: false)return nil unless security_prices_provider.present?+ return nil unless security.has_prices?response = security_prices_provider.fetch_security_prices \ticker: security.ticker,@@ -32,6 +33,7 @@ module Security::Price::Provideddef fetch_prices_from_provider(security:, start_date:, end_date:, cache: false)return [] unless security_prices_provider.present?return [] unless security+ return [] unless security.has_prices?response = security_prices_provider.fetch_security_prices \ticker: security.ticker,diff --git a/app/models/security/synth_combobox_option.rb b/app/models/security/synth_combobox_option.rbindex d3b4437d..9c2336df 100644--- a/app/models/security/synth_combobox_option.rb+++ b/app/models/security/synth_combobox_option.rb@@ -1,13 +1,14 @@class Security::SynthComboboxOptioninclude ActiveModel::Model- attr_accessor :symbol, :name, :logo_url, :exchange_acronym, :exchange_mic, :exchange_country_code+ attr_accessor :symbol, :name, :logo_url, :exchange_acronym, :exchange_country_code, :exchange_operating_micdef id- "#{symbol}|#{exchange_mic}|#{exchange_acronym}|#{exchange_country_code}" # submitted by combobox as value+ "#{symbol}|#{exchange_operating_mic}" # submitted by combobox as valueenddef to_combobox_display- "#{symbol} - #{name} (#{exchange_acronym})" # shown in combobox input when selected+ display_code = exchange_acronym.presence || exchange_operating_mic+ "#{symbol} - #{name} (#{display_code})" # shown in combobox input when selectedendenddiff --git a/app/views/securities/_combobox_security.turbo_stream.erb b/app/views/securities/_combobox_security.turbo_stream.erbindex cc0667a3..b14708c7 100644--- a/app/views/securities/_combobox_security.turbo_stream.erb+++ b/app/views/securities/_combobox_security.turbo_stream.erb@@ -5,7 +5,7 @@<%= combobox_security.name.presence || combobox_security.symbol %></span><span class="text-xs text-gray-500">- <%= "#{combobox_security.symbol} (#{combobox_security.exchange_acronym})" %>+ <%= "#{combobox_security.symbol} (#{combobox_security.exchange_acronym.presence || combobox_security.exchange_operating_mic})" %></span></div></div>diff --git a/db/migrate/20250207011850_add_exchange_operating_mic_to_securities.rb b/db/migrate/20250207011850_add_exchange_operating_mic_to_securities.rbnew file mode 100644index 00000000..cdd424f5--- /dev/null+++ b/db/migrate/20250207011850_add_exchange_operating_mic_to_securities.rb@@ -0,0 +1,6 @@+class AddExchangeOperatingMicToSecurities < ActiveRecord::Migration[7.2]+ def change+ add_column :securities, :exchange_operating_mic, :string+ add_index :securities, :exchange_operating_mic+ end+enddiff --git a/db/migrate/20250207194638_adjust_securities_indexes.rb b/db/migrate/20250207194638_adjust_securities_indexes.rbnew file mode 100644index 00000000..b04d8de1--- /dev/null+++ b/db/migrate/20250207194638_adjust_securities_indexes.rb@@ -0,0 +1,6 @@+class AdjustSecuritiesIndexes < ActiveRecord::Migration[7.2]+ def change+ remove_index :securities, name: "index_securities_on_ticker_and_exchange_mic"+ add_index :securities, [ :ticker, :exchange_operating_mic ], unique: true+ end+enddiff --git a/db/migrate/20250211161238_make_ticker_not_null.rb b/db/migrate/20250211161238_make_ticker_not_null.rbnew file mode 100644index 00000000..ee22e6da--- /dev/null+++ b/db/migrate/20250211161238_make_ticker_not_null.rb@@ -0,0 +1,5 @@+class MakeTickerNotNull < ActiveRecord::Migration[7.2]+ def change+ change_column_null :securities, :ticker, false+ end+enddiff --git a/db/schema.rb b/db/schema.rbindex f44329c6..c2d3815b 100644--- a/db/schema.rb+++ b/db/schema.rb@@ -10,7 +10,7 @@## It's strongly recommended that you check this file into your version control system.-ActiveRecord::Schema[7.2].define(version: 2025_02_07_014022) do+ActiveRecord::Schema[7.2].define(version: 2025_02_11_161238) do# These are extensions that must be enabled in order to support this databaseenable_extension "pgcrypto"enable_extension "plpgsql"@@ -542,7 +542,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_07_014022) doendcreate_table "securities", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|- t.string "ticker"+ t.string "ticker", null: falset.string "name"t.datetime "created_at", null: falset.datetime "updated_at", null: false@@ -550,8 +550,10 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_07_014022) dot.string "exchange_mic"t.string "exchange_acronym"t.string "logo_url"+ t.string "exchange_operating_mic"t.index ["country_code"], name: "index_securities_on_country_code"- t.index ["ticker", "exchange_mic"], name: "index_securities_on_ticker_and_exchange_mic", unique: true+ t.index ["exchange_operating_mic"], name: "index_securities_on_exchange_operating_mic"+ t.index ["ticker", "exchange_operating_mic"], name: "index_securities_on_ticker_and_exchange_operating_mic", unique: trueendcreate_table "security_prices", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|diff --git a/lib/tasks/securities.rake b/lib/tasks/securities.rakenew file mode 100644index 00000000..d1cbcb5a--- /dev/null+++ b/lib/tasks/securities.rake@@ -0,0 +1,141 @@+# frozen_string_literal: true++namespace :securities do+ desc "Backfill exchange_operating_mic for securities using Synth API"+ task backfill_exchange_mic: :environment do+ puts "Starting exchange_operating_mic backfill..."++ api_key = Rails.application.config.app_mode.self_hosted? ? Setting.synth_api_key : ENV["SYNTH_API_KEY"]+ unless api_key.present?+ puts "ERROR: No Synth API key found. Please set SYNTH_API_KEY env var or configure it in Settings for self-hosted mode."+ exit 1+ end++ securities = Security.where(exchange_operating_mic: nil).where.not(ticker: nil)+ total = securities.count+ processed = 0+ errors = []++ securities.find_each do |security|+ processed += 1+ print "\rProcessing #{processed}/#{total} (#{(processed.to_f/total * 100).round(1)}%)"++ begin+ response = Faraday.get("https://api.synthfinance.com/tickers/#{security.ticker}") do |req|+ req.params["country_code"] = security.country_code if security.country_code.present?+ req.headers["Authorization"] = "Bearer #{api_key}"+ end++ if response.success?+ data = JSON.parse(response.body).dig("data")+ exchange_data = data["exchange"]++ # Update security with exchange info and other metadata+ security.update!(+ exchange_operating_mic: exchange_data["operating_mic_code"],+ exchange_mic: exchange_data["mic_code"],+ exchange_acronym: exchange_data["acronym"],+ name: data["name"],+ logo_url: data["logo_url"],+ country_code: exchange_data["country_code"]+ )+ else+ errors << "#{security.ticker}: HTTP #{response.status} - #{response.body}"+ end+ rescue => e+ errors << "#{security.ticker}: #{e.message}"+ end++ # Add a small delay to not overwhelm the API+ sleep(0.1)+ end++ puts "\n\nBackfill complete!"+ puts "Processed #{processed} securities"++ if errors.any?+ puts "\nErrors encountered:"+ errors.each { |error| puts " - #{error}" }+ end+ end++ desc "De-duplicate securities based on ticker + exchange_operating_mic"+ task :deduplicate, [ :dry_run ] => :environment do |_t, args|+ # First check if we have any securities without exchange_operating_mic+ missing_mic_count = Security.where(exchange_operating_mic: nil).where.not(ticker: nil).count++ if missing_mic_count > 0+ puts "ERROR: Found #{missing_mic_count} securities without exchange_operating_mic."+ puts "Please run 'rails securities:backfill_exchange_mic' first to ensure all securities have exchange_operating_mic values."+ exit 1+ end++ dry_run = args[:dry_run].present?+ puts "Starting securities de-duplication... #{dry_run ? '(DRY RUN)' : ''}"++ # Find all duplicate securities (same ticker + exchange_operating_mic)+ duplicates = Security+ .where.not(ticker: nil)+ .where.not(exchange_operating_mic: nil)+ .group(:ticker, :exchange_operating_mic)+ .having("COUNT(*) > 1")+ .pluck(:ticker, :exchange_operating_mic)++ puts "Found #{duplicates.length} sets of duplicate securities"+ total_holdings = 0+ total_trades = 0+ total_prices = 0++ duplicates.each do |ticker, exchange_operating_mic|+ securities = Security.where(ticker: ticker, exchange_operating_mic: exchange_operating_mic)+ .order(created_at: :asc)++ canonical = securities.first+ duplicates = securities[1..]++ puts "\nProcessing #{ticker} (#{exchange_operating_mic}):"+ puts " Canonical: #{canonical.id} (created: #{canonical.created_at})"+ puts " Duplicates: #{duplicates.map(&:id).join(', ')}"++ # Count affected records+ holdings_count = Account::Holding.where(security_id: duplicates).count+ trades_count = Account::Trade.where(security_id: duplicates).count+ prices_count = Security::Price.where(security_id: duplicates).count++ total_holdings += holdings_count+ total_trades += trades_count+ total_prices += prices_count++ puts " Would update:"+ puts " - #{holdings_count} holdings"+ puts " - #{trades_count} trades"+ puts " - #{prices_count} prices"++ unless dry_run+ begin+ ActiveRecord::Base.transaction do+ # Update all references to point to the canonical security+ Account::Holding.where(security_id: duplicates).update_all(security_id: canonical.id)+ Account::Trade.where(security_id: duplicates).update_all(security_id: canonical.id)+ Security::Price.where(security_id: duplicates).update_all(security_id: canonical.id)++ # Delete the duplicates+ duplicates.each(&:destroy!)+ end+ puts " ✓ Successfully merged and removed duplicates"+ rescue => e+ puts " ✗ Error processing #{ticker}: #{e.message}"+ end+ end+ end++ puts "\nSummary:"+ puts " Total duplicate sets: #{duplicates.length}"+ puts " Total affected records:"+ puts " - #{total_holdings} holdings"+ puts " - #{total_trades} trades"+ puts " - #{total_prices} prices"+ puts " Mode: #{dry_run ? 'Dry run (no changes made)' : 'Live run (changes applied)'}"+ puts "\nDe-duplication complete!"+ end+enddiff --git a/test/fixtures/securities.yml b/test/fixtures/securities.ymlindex e5e38687..9133f008 100644--- a/test/fixtures/securities.yml+++ b/test/fixtures/securities.yml@@ -1,12 +1,12 @@aapl:ticker: AAPLname: Apple- exchange_mic: XNAS+ exchange_operating_mic: XNAScountry_code: USmsft:ticker: MSFTname: Microsoft- exchange_mic: XNAS+ exchange_operating_mic: XNAScountry_code: US