Skip to content

armgit / maybe

Public Imported from GitHub
0

The personal finance app for everyone

Enhance security information retrieval and handling (#1826)

Josh Pigford68d7cb5de6c9
diff --git a/app/jobs/fetch_security_info_job.rb b/app/jobs/fetch_security_info_job.rb
index 5eaafa43..aa64c169 100644
--- a/app/jobs/fetch_security_info_job.rb
+++ b/app/jobs/fetch_security_info_job.rb
@@ -6,10 +6,13 @@ class FetchSecurityInfoJob < ApplicationJob
security = Security.find(security_id)
- security_info_response = Security.security_info_provider.fetch_security_info(
- ticker: security.ticker,
- mic_code: security.exchange_mic
- )
+ params = {
+ ticker: security.ticker
+ }
+ params[:mic_code] = security.exchange_mic if security.exchange_mic.present?
+ params[:operating_mic] = security.exchange_operating_mic if security.exchange_operating_mic.present?
+
+ security_info_response = Security.security_info_provider.fetch_security_info(**params)
security.update(
name: security_info_response.info.dig("name")
diff --git a/app/models/account/trade_builder.rb b/app/models/account/trade_builder.rb
index 8e0f9ad7..704d851f 100644
--- a/app/models/account/trade_builder.rb
+++ b/app/models/account/trade_builder.rb
@@ -111,13 +111,10 @@ class Account::TradeBuilder
end
def security
- ticker_symbol, exchange_mic, exchange_acronym, exchange_country_code = ticker.split("|")
+ ticker_symbol, exchange_operating_mic = ticker.split("|")
- security = Security.find_or_create_by(ticker: ticker_symbol, exchange_mic: exchange_mic, country_code: exchange_country_code)
- security.update(exchange_acronym: exchange_acronym)
-
- FetchSecurityInfoJob.perform_later(security.id)
-
- security
+ Security.find_or_create_by(ticker: ticker_symbol, exchange_operating_mic: exchange_operating_mic) do |s|
+ FetchSecurityInfoJob.perform_later(s.id)
+ end
end
end
diff --git a/app/models/demo/generator.rb b/app/models/demo/generator.rb
index 22038857..283ae35b 100644
--- a/app/models/demo/generator.rb
+++ b/app/models/demo/generator.rb
@@ -231,12 +231,12 @@ class Demo::Generator
def load_securities!
# Create an unknown security to simulate edge cases
- Security.create! ticker: "UNKNOWN", name: "Unknown Demo Stock", exchange_mic: "UNKNOWN"
+ Security.create! ticker: "UNKNOWN", name: "Unknown Demo Stock"
securities = [
- { ticker: "AAPL", exchange_mic: "NASDAQ", name: "Apple Inc.", reference_price: 210 },
- { ticker: "TM", exchange_mic: "NYSE", name: "Toyota Motor Corporation", reference_price: 202 },
- { ticker: "MSFT", exchange_mic: "NASDAQ", name: "Microsoft Corporation", reference_price: 455 }
+ { ticker: "AAPL", exchange_mic: "XNGS", exchange_operating_mic: "XNAS", name: "Apple Inc.", reference_price: 210 },
+ { ticker: "TM", exchange_mic: "XNYS", exchange_operating_mic: "XNYS", name: "Toyota Motor Corporation", reference_price: 202 },
+ { ticker: "MSFT", exchange_mic: "XNGS", exchange_operating_mic: "XNAS", name: "Microsoft Corporation", reference_price: 455 }
]
securities.each do |security_attributes|
diff --git a/app/models/plaid_investment_sync.rb b/app/models/plaid_investment_sync.rb
index 8c1d9805..bcb1f330 100644
--- a/app/models/plaid_investment_sync.rb
+++ b/app/models/plaid_investment_sync.rb
@@ -82,12 +82,15 @@ class PlaidInvestmentSync
end
return [ nil, nil ] if plaid_security.nil? || plaid_security.ticker_symbol.blank?
+ return [ nil, plaid_security ] if plaid_security.ticker_symbol == "CUR:USD" # internally, we do not consider cash a security and track it separately
+ operating_mic = plaid_security.market_identifier_code
+
+ # Find any matching security
security = Security.find_or_create_by!(
ticker: plaid_security.ticker_symbol,
- exchange_mic: plaid_security.market_identifier_code || "XNAS",
- country_code: "US"
- ) unless plaid_security.ticker_symbol == "CUR:USD" # internally, we do not consider cash a security and track it separately
+ exchange_operating_mic: operating_mic
+ )
[ security, plaid_security ]
end
diff --git a/app/models/provider/synth.rb b/app/models/provider/synth.rb
index a5356a0f..65deaae5 100644
--- a/app/models/provider/synth.rb
+++ b/app/models/provider/synth.rb
@@ -145,6 +145,7 @@ class Provider::Synth
logo_url: security.dig("logo_url"),
exchange_acronym: security.dig("exchange", "acronym"),
exchange_mic: security.dig("exchange", "mic_code"),
+ exchange_operating_mic: security.dig("exchange", "operating_mic_code"),
country_code: security.dig("exchange", "country_code")
}
end
@@ -155,9 +156,10 @@ class Provider::Synth
raw_response: response
end
- def fetch_security_info(ticker:, mic_code:)
+ def fetch_security_info(ticker:, mic_code: nil, operating_mic: nil)
response = client.get("#{base_url}/tickers/#{ticker}") do |req|
- req.params["mic_code"] = mic_code
+ req.params["mic_code"] = mic_code if mic_code.present?
+ req.params["operating_mic"] = operating_mic if operating_mic.present?
end
parsed = JSON.parse(response.body)
diff --git a/app/models/security.rb b/app/models/security.rb
index d2ce6387..4a2bd1a7 100644
--- a/app/models/security.rb
+++ b/app/models/security.rb
@@ -6,7 +6,7 @@ class Security < ApplicationRecord
has_many :prices, dependent: :destroy
validates :ticker, presence: true
- validates :ticker, uniqueness: { scope: :exchange_mic, case_sensitive: false }
+ validates :ticker, uniqueness: { scope: :exchange_operating_mic, case_sensitive: false }
class << self
def search(query)
@@ -30,11 +30,15 @@ class Security < ApplicationRecord
name: name,
logo_url: logo_url,
exchange_acronym: exchange_acronym,
- exchange_mic: exchange_mic,
+ exchange_operating_mic: exchange_operating_mic,
exchange_country_code: country_code
)
end
+ def has_prices?
+ exchange_operating_mic.present?
+ end
+
private
def upcase_ticker
diff --git a/app/models/security/price/provided.rb b/app/models/security/price/provided.rb
index 02e5ce14..e2a99774 100644
--- a/app/models/security/price/provided.rb
+++ b/app/models/security/price/provided.rb
@@ -8,6 +8,7 @@ module Security::Price::Provided
def fetch_price_from_provider(security:, date:, cache: false)
return nil unless security_prices_provider.present?
+ return nil unless security.has_prices?
response = security_prices_provider.fetch_security_prices \
ticker: security.ticker,
@@ -32,6 +33,7 @@ module Security::Price::Provided
def fetch_prices_from_provider(security:, start_date:, end_date:, cache: false)
return [] unless security_prices_provider.present?
return [] unless security
+ return [] unless security.has_prices?
response = security_prices_provider.fetch_security_prices \
ticker: security.ticker,
diff --git a/app/models/security/synth_combobox_option.rb b/app/models/security/synth_combobox_option.rb
index d3b4437d..9c2336df 100644
--- a/app/models/security/synth_combobox_option.rb
+++ b/app/models/security/synth_combobox_option.rb
@@ -1,13 +1,14 @@
class Security::SynthComboboxOption
include ActiveModel::Model
- attr_accessor :symbol, :name, :logo_url, :exchange_acronym, :exchange_mic, :exchange_country_code
+ attr_accessor :symbol, :name, :logo_url, :exchange_acronym, :exchange_country_code, :exchange_operating_mic
def id
- "#{symbol}|#{exchange_mic}|#{exchange_acronym}|#{exchange_country_code}" # submitted by combobox as value
+ "#{symbol}|#{exchange_operating_mic}" # submitted by combobox as value
end
def to_combobox_display
- "#{symbol} - #{name} (#{exchange_acronym})" # shown in combobox input when selected
+ display_code = exchange_acronym.presence || exchange_operating_mic
+ "#{symbol} - #{name} (#{display_code})" # shown in combobox input when selected
end
end
diff --git a/app/views/securities/_combobox_security.turbo_stream.erb b/app/views/securities/_combobox_security.turbo_stream.erb
index cc0667a3..b14708c7 100644
--- a/app/views/securities/_combobox_security.turbo_stream.erb
+++ b/app/views/securities/_combobox_security.turbo_stream.erb
@@ -5,7 +5,7 @@
<%= combobox_security.name.presence || combobox_security.symbol %>
</span>
<span class="text-xs text-gray-500">
- <%= "#{combobox_security.symbol} (#{combobox_security.exchange_acronym})" %>
+ <%= "#{combobox_security.symbol} (#{combobox_security.exchange_acronym.presence || combobox_security.exchange_operating_mic})" %>
</span>
</div>
</div>
diff --git a/db/migrate/20250207011850_add_exchange_operating_mic_to_securities.rb b/db/migrate/20250207011850_add_exchange_operating_mic_to_securities.rb
new file mode 100644
index 00000000..cdd424f5
--- /dev/null
+++ b/db/migrate/20250207011850_add_exchange_operating_mic_to_securities.rb
@@ -0,0 +1,6 @@
+class AddExchangeOperatingMicToSecurities < ActiveRecord::Migration[7.2]
+ def change
+ add_column :securities, :exchange_operating_mic, :string
+ add_index :securities, :exchange_operating_mic
+ end
+end
diff --git a/db/migrate/20250207194638_adjust_securities_indexes.rb b/db/migrate/20250207194638_adjust_securities_indexes.rb
new file mode 100644
index 00000000..b04d8de1
--- /dev/null
+++ b/db/migrate/20250207194638_adjust_securities_indexes.rb
@@ -0,0 +1,6 @@
+class AdjustSecuritiesIndexes < ActiveRecord::Migration[7.2]
+ def change
+ remove_index :securities, name: "index_securities_on_ticker_and_exchange_mic"
+ add_index :securities, [ :ticker, :exchange_operating_mic ], unique: true
+ end
+end
diff --git a/db/migrate/20250211161238_make_ticker_not_null.rb b/db/migrate/20250211161238_make_ticker_not_null.rb
new file mode 100644
index 00000000..ee22e6da
--- /dev/null
+++ b/db/migrate/20250211161238_make_ticker_not_null.rb
@@ -0,0 +1,5 @@
+class MakeTickerNotNull < ActiveRecord::Migration[7.2]
+ def change
+ change_column_null :securities, :ticker, false
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index f44329c6..c2d3815b 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema[7.2].define(version: 2025_02_07_014022) do
+ActiveRecord::Schema[7.2].define(version: 2025_02_11_161238) do
# These are extensions that must be enabled in order to support this database
enable_extension "pgcrypto"
enable_extension "plpgsql"
@@ -542,7 +542,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_07_014022) do
end
create_table "securities", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
- t.string "ticker"
+ t.string "ticker", null: false
t.string "name"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
@@ -550,8 +550,10 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_07_014022) do
t.string "exchange_mic"
t.string "exchange_acronym"
t.string "logo_url"
+ t.string "exchange_operating_mic"
t.index ["country_code"], name: "index_securities_on_country_code"
- t.index ["ticker", "exchange_mic"], name: "index_securities_on_ticker_and_exchange_mic", unique: true
+ t.index ["exchange_operating_mic"], name: "index_securities_on_exchange_operating_mic"
+ t.index ["ticker", "exchange_operating_mic"], name: "index_securities_on_ticker_and_exchange_operating_mic", unique: true
end
create_table "security_prices", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
diff --git a/lib/tasks/securities.rake b/lib/tasks/securities.rake
new file mode 100644
index 00000000..d1cbcb5a
--- /dev/null
+++ b/lib/tasks/securities.rake
@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+namespace :securities do
+ desc "Backfill exchange_operating_mic for securities using Synth API"
+ task backfill_exchange_mic: :environment do
+ puts "Starting exchange_operating_mic backfill..."
+
+ api_key = Rails.application.config.app_mode.self_hosted? ? Setting.synth_api_key : ENV["SYNTH_API_KEY"]
+ unless api_key.present?
+ puts "ERROR: No Synth API key found. Please set SYNTH_API_KEY env var or configure it in Settings for self-hosted mode."
+ exit 1
+ end
+
+ securities = Security.where(exchange_operating_mic: nil).where.not(ticker: nil)
+ total = securities.count
+ processed = 0
+ errors = []
+
+ securities.find_each do |security|
+ processed += 1
+ print "\rProcessing #{processed}/#{total} (#{(processed.to_f/total * 100).round(1)}%)"
+
+ begin
+ response = Faraday.get("https://api.synthfinance.com/tickers/#{security.ticker}") do |req|
+ req.params["country_code"] = security.country_code if security.country_code.present?
+ req.headers["Authorization"] = "Bearer #{api_key}"
+ end
+
+ if response.success?
+ data = JSON.parse(response.body).dig("data")
+ exchange_data = data["exchange"]
+
+ # Update security with exchange info and other metadata
+ security.update!(
+ exchange_operating_mic: exchange_data["operating_mic_code"],
+ exchange_mic: exchange_data["mic_code"],
+ exchange_acronym: exchange_data["acronym"],
+ name: data["name"],
+ logo_url: data["logo_url"],
+ country_code: exchange_data["country_code"]
+ )
+ else
+ errors << "#{security.ticker}: HTTP #{response.status} - #{response.body}"
+ end
+ rescue => e
+ errors << "#{security.ticker}: #{e.message}"
+ end
+
+ # Add a small delay to not overwhelm the API
+ sleep(0.1)
+ end
+
+ puts "\n\nBackfill complete!"
+ puts "Processed #{processed} securities"
+
+ if errors.any?
+ puts "\nErrors encountered:"
+ errors.each { |error| puts " - #{error}" }
+ end
+ end
+
+ desc "De-duplicate securities based on ticker + exchange_operating_mic"
+ task :deduplicate, [ :dry_run ] => :environment do |_t, args|
+ # First check if we have any securities without exchange_operating_mic
+ missing_mic_count = Security.where(exchange_operating_mic: nil).where.not(ticker: nil).count
+
+ if missing_mic_count > 0
+ puts "ERROR: Found #{missing_mic_count} securities without exchange_operating_mic."
+ puts "Please run 'rails securities:backfill_exchange_mic' first to ensure all securities have exchange_operating_mic values."
+ exit 1
+ end
+
+ dry_run = args[:dry_run].present?
+ puts "Starting securities de-duplication... #{dry_run ? '(DRY RUN)' : ''}"
+
+ # Find all duplicate securities (same ticker + exchange_operating_mic)
+ duplicates = Security
+ .where.not(ticker: nil)
+ .where.not(exchange_operating_mic: nil)
+ .group(:ticker, :exchange_operating_mic)
+ .having("COUNT(*) > 1")
+ .pluck(:ticker, :exchange_operating_mic)
+
+ puts "Found #{duplicates.length} sets of duplicate securities"
+ total_holdings = 0
+ total_trades = 0
+ total_prices = 0
+
+ duplicates.each do |ticker, exchange_operating_mic|
+ securities = Security.where(ticker: ticker, exchange_operating_mic: exchange_operating_mic)
+ .order(created_at: :asc)
+
+ canonical = securities.first
+ duplicates = securities[1..]
+
+ puts "\nProcessing #{ticker} (#{exchange_operating_mic}):"
+ puts " Canonical: #{canonical.id} (created: #{canonical.created_at})"
+ puts " Duplicates: #{duplicates.map(&:id).join(', ')}"
+
+ # Count affected records
+ holdings_count = Account::Holding.where(security_id: duplicates).count
+ trades_count = Account::Trade.where(security_id: duplicates).count
+ prices_count = Security::Price.where(security_id: duplicates).count
+
+ total_holdings += holdings_count
+ total_trades += trades_count
+ total_prices += prices_count
+
+ puts " Would update:"
+ puts " - #{holdings_count} holdings"
+ puts " - #{trades_count} trades"
+ puts " - #{prices_count} prices"
+
+ unless dry_run
+ begin
+ ActiveRecord::Base.transaction do
+ # Update all references to point to the canonical security
+ Account::Holding.where(security_id: duplicates).update_all(security_id: canonical.id)
+ Account::Trade.where(security_id: duplicates).update_all(security_id: canonical.id)
+ Security::Price.where(security_id: duplicates).update_all(security_id: canonical.id)
+
+ # Delete the duplicates
+ duplicates.each(&:destroy!)
+ end
+ puts " ✓ Successfully merged and removed duplicates"
+ rescue => e
+ puts " ✗ Error processing #{ticker}: #{e.message}"
+ end
+ end
+ end
+
+ puts "\nSummary:"
+ puts " Total duplicate sets: #{duplicates.length}"
+ puts " Total affected records:"
+ puts " - #{total_holdings} holdings"
+ puts " - #{total_trades} trades"
+ puts " - #{total_prices} prices"
+ puts " Mode: #{dry_run ? 'Dry run (no changes made)' : 'Live run (changes applied)'}"
+ puts "\nDe-duplication complete!"
+ end
+end
diff --git a/test/fixtures/securities.yml b/test/fixtures/securities.yml
index e5e38687..9133f008 100644
--- a/test/fixtures/securities.yml
+++ b/test/fixtures/securities.yml
@@ -1,12 +1,12 @@
aapl:
ticker: AAPL
name: Apple
- exchange_mic: XNAS
+ exchange_operating_mic: XNAS
country_code: US
msft:
ticker: MSFT
name: Microsoft
- exchange_mic: XNAS
+ exchange_operating_mic: XNAS
country_code: US