The personal finance app for everyone
Initial pass at Synth-based ticker selection (#1392)
Josh Pigfordcd91e6661885
diff --git a/.env.example b/.env.exampleindex 02865a5a..1169e684 100644--- a/.env.example+++ b/.env.example@@ -11,14 +11,10 @@# For users who have other applications listening at 3000, this allows them to set a value puma will listen to.PORT=3000-# Exchange Rate & US Stock Pricing API-# This is used to convert between different currencies in the app. In addition, it fetches US stock prices. We use Synth, which is a Maybe product. You can sign up for a free account at synthfinance.com.+# Exchange Rate & Stock Pricing API+# This is used to convert between different currencies in the app. In addition, it fetches global stock prices. We use Synth, which is a Maybe product. You can sign up for a free account at synthfinance.com.SYNTH_API_KEY=-# Non-US Stock Pricing API-# This is used to fetch non-US stock prices. We use Marketstack.com for this and while they offer a free tier, it is quite limited. You'll almost certainly need their Basic plan, which is $9.99 per month.-MARKETSTACK_API_KEY=-# SMTP Configuration# This is only needed if you intend on sending emails from your Maybe instance (such as for password resets or email financial reports).# Resend.com is a good option that offers a free tier for sending emails.diff --git a/.env.local.example b/.env.local.exampleindex cd38f220..d393f623 100644--- a/.env.local.example+++ b/.env.local.example@@ -3,6 +3,3 @@ SELF_HOSTED=false# Enable Synth market data (careful, this will use your API credits)SYNTH_API_KEY=yourapikeyhere--# Enable Marketstack market data (careful, this will use your API credits)-MARKETSTACK_API_KEY=yourapikeyherediff --git a/app/controllers/account/trades_controller.rb b/app/controllers/account/trades_controller.rbindex 8b5a878c..0b8bc99d 100644--- a/app/controllers/account/trades_controller.rb+++ b/app/controllers/account/trades_controller.rb@@ -34,7 +34,10 @@ class Account::TradesController < ApplicationControllerenddef securities- @pagy, @securities = pagy(Security.order(:name).search(params[:q]), limit: 20)+ query = params[:q]+ return render json: [] if query.blank? || query.length < 2 || query.length > 100++ @securities = Security::SynthComboboxOption.find_in_synth(query)endprivatediff --git a/app/jobs/securities_import_job.rb b/app/jobs/securities_import_job.rbdeleted file mode 100644index 85fef904..00000000--- a/app/jobs/securities_import_job.rb+++ /dev/null@@ -1,13 +0,0 @@-class SecuritiesImportJob < ApplicationJob- queue_as :default-- def perform(country_code = nil)- exchanges = StockExchange.in_country(country_code)- market_stack_client = Provider::Marketstack.new(ENV["MARKETSTACK_API_KEY"])-- exchanges.each do |exchange|- importer = Security::Importer.new(market_stack_client, exchange.mic)- importer.import- end- end-enddiff --git a/app/models/account/trade_builder.rb b/app/models/account/trade_builder.rbindex 4610407f..3219fb8a 100644--- a/app/models/account/trade_builder.rb+++ b/app/models/account/trade_builder.rb@@ -31,8 +31,9 @@ class Account::TradeBuilder < Account::EntryBuilderenddef security- return Security.find(ticker) if ticker.match?(/\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/i)- Security.find_or_create_by(ticker: ticker)+ ticker_symbol, exchange_mic = ticker.split("|")++ Security.find_or_create_by(ticker: ticker_symbol, exchange_mic: exchange_mic)enddef amountdiff --git a/app/models/provider/marketstack.rb b/app/models/provider/marketstack.rbdeleted file mode 100644index 10e2aa5d..00000000--- a/app/models/provider/marketstack.rb+++ /dev/null@@ -1,120 +0,0 @@-class Provider::Marketstack- include Retryable-- def initialize(api_key)- @api_key = api_key- end-- def fetch_security_prices(ticker:, start_date:, end_date:)- prices = paginate("#{base_url}/eod", {- symbols: ticker,- date_from: start_date.to_s,- date_to: end_date.to_s- }) do |body|- body.dig("data").map do |price|- {- date: price["date"],- price: price["close"]&.to_f,- currency: "USD"- }- end- end-- SecurityPriceResponse.new(- prices: prices,- success?: true,- raw_response: prices.to_json- )- rescue StandardError => error- SecurityPriceResponse.new(- success?: false,- error: error,- raw_response: error- )- end-- def fetch_tickers(exchange_mic: nil)- url = exchange_mic ? "#{base_url}/tickers?exchange=#{exchange_mic}" : "#{base_url}/tickers"- tickers = paginate(url) do |body|- body.dig("data").map do |ticker|- {- name: ticker["name"],- symbol: ticker["symbol"],- exchange_mic: exchange_mic || ticker.dig("stock_exchange", "mic"),- exchange_acronym: ticker.dig("stock_exchange", "acronym"),- country_code: ticker.dig("stock_exchange", "country_code")- }- end- end-- TickerResponse.new(- tickers: tickers,- success?: true,- raw_response: tickers.to_json- )- rescue StandardError => error- TickerResponse.new(- success?: false,- error: error,- raw_response: error- )- end-- private-- attr_reader :api_key-- SecurityPriceResponse = Struct.new(:prices, :success?, :error, :raw_response, keyword_init: true)- TickerResponse = Struct.new(:tickers, :success?, :error, :raw_response, keyword_init: true)-- def base_url- "https://api.marketstack.com/v1"- end-- def client- @client ||= Faraday.new(url: base_url) do |faraday|- faraday.params["access_key"] = api_key- end- end-- def build_error(response)- Provider::Base::ProviderError.new(<<~ERROR)- Failed to fetch data from #{self.class}- Status: #{response.status}- Body: #{response.body.inspect}- ERROR- end-- def fetch_page(url, page, params = {})- client.get(url) do |req|- params.each { |k, v| req.params[k.to_s] = v.to_s }- req.params["offset"] = (page - 1) * 100 # Marketstack uses offset-based pagination- req.params["limit"] = 10000 # Maximum allowed by Marketstack- end- end-- def paginate(url, params = {})- results = []- page = 1- total_results = Float::INFINITY-- while results.length < total_results- response = fetch_page(url, page, params)-- if response.success?- body = JSON.parse(response.body)- page_results = yield(body)- results.concat(page_results)-- total_results = body.dig("pagination", "total")- page += 1- else- raise build_error(response)- end-- break if results.length >= total_results- end-- results- end-enddiff --git a/app/models/provider/synth.rb b/app/models/provider/synth.rbindex d5f28e50..6dabd941 100644--- a/app/models/provider/synth.rb+++ b/app/models/provider/synth.rb@@ -122,6 +122,31 @@ class Provider::Synthraw_response: errorend+ def search_securities(query:, dataset: "limited", country_code:)+ response = client.get("#{base_url}/tickers/search") do |req|+ req.params["name"] = query+ req.params["dataset"] = dataset+ req.params["country_code"] = country_code+ end++ parsed = JSON.parse(response.body)++ securities = parsed.dig("data").map do |security|+ {+ symbol: security.dig("symbol"),+ name: security.dig("name"),+ logo_url: security.dig("logo_url"),+ exchange_acronym: security.dig("exchange", "acronym"),+ exchange_mic: security.dig("exchange", "mic_code")+ }+ end++ SearchSecuritiesResponse.new \+ securities: securities,+ success?: true,+ raw_response: response+ end+privateattr_reader :api_key@@ -130,6 +155,7 @@ class Provider::SynthSecurityPriceResponse = Struct.new :prices, :success?, :error, :raw_response, keyword_init: trueExchangeRatesResponse = Struct.new :rates, :success?, :error, :raw_response, keyword_init: trueUsageResponse = Struct.new :used, :limit, :utilization, :plan, :success?, :error, :raw_response, keyword_init: true+ SearchSecuritiesResponse = Struct.new :securities, :success?, :error, :raw_response, keyword_init: truedef base_url"https://api.synthfinance.com"diff --git a/app/models/security.rb b/app/models/security.rbindex d1a160b8..732599ce 100644--- a/app/models/security.rb+++ b/app/models/security.rb@@ -1,4 +1,5 @@class Security < ApplicationRecord+ include Providablebefore_save :upcase_tickerhas_many :trades, dependent: :nullify, class_name: "Account::Trade"@@ -7,25 +8,6 @@ class Security < ApplicationRecordvalidates :ticker, presence: truevalidates :ticker, uniqueness: { scope: :exchange_mic, case_sensitive: false }- scope :search, ->(query) {- return none if query.blank? || query.length < 2-- # Clean and normalize the search terms- sanitized_query = query.split.map do |term|- cleaned_term = term.gsub(/[^a-zA-Z0-9]/, " ").strip- next if cleaned_term.blank?- cleaned_term- end.compact.join(" | ")-- return none if sanitized_query.blank?-- sanitized_query = ActiveRecord::Base.connection.quote(sanitized_query)-- where("search_vector @@ to_tsquery('simple', #{sanitized_query}) AND exchange_mic IS NOT NULL")- .select("securities.*, ts_rank_cd(search_vector, to_tsquery('simple', #{sanitized_query})) AS rank")- .reorder("rank DESC")- }-def current_price@current_price ||= Security::Price.find_price(security: self, date: Date.current)return nil if @current_price.nil?@@ -33,7 +15,7 @@ class Security < ApplicationRecordenddef to_combobox_display- "#{ticker} - #{name} (#{exchange_acronym})"+ "#{ticker} (#{exchange_acronym})"enddiff --git a/app/models/security/importer.rb b/app/models/security/importer.rbdeleted file mode 100644index 4146fea2..00000000--- a/app/models/security/importer.rb+++ /dev/null@@ -1,45 +0,0 @@-class Security::Importer- def initialize(provider, stock_exchange = nil)- @provider = provider- @stock_exchange = stock_exchange- end-- def import- securities = @provider.fetch_tickers(exchange_mic: @stock_exchange)&.tickers-- # Deduplicate securities based on ticker and exchange_mic- securities_to_create = securities- .map do |security|- {- name: security[:name],- ticker: security[:symbol],- country_code: security[:country_code],- exchange_mic: security[:exchange_mic],- exchange_acronym: security[:exchange_acronym]- }- end- .compact- .uniq { |security| [ security[:ticker], security[:exchange_mic] ] }-- # First update any existing securities that only have a ticker- Security.where(exchange_mic: nil)- .where(ticker: securities_to_create.map { |s| s[:ticker] })- .update_all(- securities_to_create.map do |security|- {- name: security[:name],- country_code: security[:country_code],- exchange_mic: security[:exchange_mic],- exchange_acronym: security[:exchange_acronym]- }- end.first- )-- # Then create/update any remaining securities- Security.upsert_all(- securities_to_create,- unique_by: [ :ticker, :exchange_mic ],- update_only: [ :name, :country_code, :exchange_acronym ]- ) unless securities_to_create.empty?- end-enddiff --git a/app/models/security/synth_combobox_option.rb b/app/models/security/synth_combobox_option.rbnew file mode 100644index 00000000..a4746ee7--- /dev/null+++ b/app/models/security/synth_combobox_option.rb@@ -0,0 +1,20 @@+class Security::SynthComboboxOption+ include ActiveModel::Model+ include Providable++ attr_accessor :symbol, :name, :logo_url, :exchange_acronym, :exchange_mic++ class << self+ def find_in_synth(query)+ security_prices_provider.search_securities(query:, dataset: "limited", country_code: Current.family.country).securities.map { |attrs| new(**attrs) }+ end+ end++ def id+ "#{symbol}|#{exchange_mic}" # submitted by combobox as value+ end++ def to_combobox_display+ "#{symbol} - #{name} (#{exchange_acronym})" # shown in combobox input when selected+ end+enddiff --git a/app/views/account/trades/_form.html.erb b/app/views/account/trades/_form.html.erbindex 283a7725..f8c4033a 100644--- a/app/views/account/trades/_form.html.erb+++ b/app/views/account/trades/_form.html.erb@@ -8,7 +8,7 @@<%= form.select :type, options_for_select([%w[Buy buy], %w[Sell sell], %w[Deposit transfer_in], %w[Withdrawal transfer_out], %w[Interest interest]], "buy"), { label: t(".type") }, { data: { "trade-form-target": "typeInput" } } %><div data-trade-form-target="tickerInput"><div class="form-field combobox">- <%= form.combobox :ticker, securities_account_trades_path(entry.account), label: t(".holding"), placeholder: t(".ticker_placeholder"), autocomplete: :list, free_text: true %>+ <%= form.combobox :ticker, securities_account_trades_path(entry.account), label: t(".holding"), placeholder: t(".ticker_placeholder") %></div></div>diff --git a/app/views/account/trades/_security.turbo_stream.erb b/app/views/account/trades/_security.turbo_stream.erbnew file mode 100644index 00000000..f270b028--- /dev/null+++ b/app/views/account/trades/_security.turbo_stream.erb@@ -0,0 +1,11 @@+<div class="flex items-center">+ <%= image_tag(security.logo_url, class: "rounded-full h-8 w-8 inline-block mr-2" ) %>+ <div class="flex flex-col">+ <span class="text-sm font-medium">+ <%= security.name.presence || security.symbol %>+ </span>+ <span class="text-xs text-gray-500">+ <%= "#{security.symbol} (#{security.exchange_acronym})" %>+ </span>+ </div>+</div>\ No newline at end of filediff --git a/app/views/account/trades/_tickers.turbo_stream.erb b/app/views/account/trades/_tickers.turbo_stream.erbdeleted file mode 100644index 56436ab8..00000000--- a/app/views/account/trades/_tickers.turbo_stream.erb+++ /dev/null@@ -1,7 +0,0 @@-<div class="flex items-center">- <%= image_tag("https://logo.synthfinance.com/ticker/#{tickers&.ticker}", class: "rounded-full h-8 w-8 inline-block mr-2") %>- <div class="flex flex-col">- <span class="text-sm font-medium"><%= tickers&.name.presence || tickers&.ticker %></span>- <span class="text-xs text-gray-500"><%= "#{tickers&.ticker} (#{tickers&.exchange_acronym})" %></span>- </div>-</div>diff --git a/app/views/account/trades/securities.turbo_stream.erb b/app/views/account/trades/securities.turbo_stream.erbindex 7ed8e90e..a3128c77 100644--- a/app/views/account/trades/securities.turbo_stream.erb+++ b/app/views/account/trades/securities.turbo_stream.erb@@ -1,3 +1,2 @@<%= async_combobox_options @securities,- render_in: { partial: "account/trades/tickers" },- next_page: @pagy.next %>\ No newline at end of file+ render_in: { partial: "account/trades/security" } %>\ No newline at end of filediff --git a/db/migrate/20241029234028_remove_search_vector.rb b/db/migrate/20241029234028_remove_search_vector.rbnew file mode 100644index 00000000..99d94602--- /dev/null+++ b/db/migrate/20241029234028_remove_search_vector.rb@@ -0,0 +1,5 @@+class RemoveSearchVector < ActiveRecord::Migration[7.2]+ def change+ remove_column :securities, :search_vector+ end+enddiff --git a/db/migrate/20241030121302_fix_not_null_stock_exchange_data.rb b/db/migrate/20241030121302_fix_not_null_stock_exchange_data.rbnew file mode 100644index 00000000..9a1f1012--- /dev/null+++ b/db/migrate/20241030121302_fix_not_null_stock_exchange_data.rb@@ -0,0 +1,10 @@+class FixNotNullStockExchangeData < ActiveRecord::Migration[7.2]+ def change+ change_column_null :stock_exchanges, :currency_code, true+ change_column_null :stock_exchanges, :currency_symbol, true+ change_column_null :stock_exchanges, :currency_name, true+ change_column_null :stock_exchanges, :city, true+ change_column_null :stock_exchanges, :timezone_name, true+ change_column_null :stock_exchanges, :timezone_abbr, true+ end+enddiff --git a/db/schema.rb b/db/schema.rbindex 507fd0f9..d9d597d7 100644--- a/db/schema.rb+++ b/db/schema.rb@@ -10,7 +10,7 @@## It's strongly recommended that you check this file into your version control system.-ActiveRecord::Schema[7.2].define(version: 2024_10_29_184115) do+ActiveRecord::Schema[7.2].define(version: 2024_10_30_121302) do# These are extensions that must be enabled in order to support this databaseenable_extension "pgcrypto"enable_extension "plpgsql"@@ -119,7 +119,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_10_29_184115) dot.boolean "is_active", default: true, null: falset.date "last_sync_date"t.uuid "institution_id"- t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY ((ARRAY['Loan'::character varying, 'CreditCard'::character varying, 'OtherLiability'::character varying])::text[])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: true+ t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY (ARRAY[('Loan'::character varying)::text, ('CreditCard'::character varying)::text, ('OtherLiability'::character varying)::text])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: truet.uuid "import_id"t.string "mode"t.index ["accountable_id", "accountable_type"], name: "index_accounts_on_accountable_id_and_accountable_type"@@ -481,9 +481,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_10_29_184115) dot.string "country_code"t.string "exchange_mic"t.string "exchange_acronym"- t.virtual "search_vector", type: :tsvector, as: "(setweight(to_tsvector('simple'::regconfig, (COALESCE(ticker, ''::character varying))::text), 'B'::\"char\") || to_tsvector('simple'::regconfig, (COALESCE(name, ''::character varying))::text))", stored: truet.index ["country_code"], name: "index_securities_on_country_code"- t.index ["search_vector"], name: "index_securities_on_search_vector", using: :gint.index ["ticker", "exchange_mic"], name: "index_securities_on_ticker_and_exchange_mic", unique: trueend@@ -524,14 +522,14 @@ ActiveRecord::Schema[7.2].define(version: 2024_10_29_184115) dot.string "mic", null: falset.string "country", null: falset.string "country_code", null: false- t.string "city", null: false+ t.string "city"t.string "website"- t.string "timezone_name", null: false- t.string "timezone_abbr", null: false+ t.string "timezone_name"+ t.string "timezone_abbr"t.string "timezone_abbr_dst"- t.string "currency_code", null: false- t.string "currency_symbol", null: false- t.string "currency_name", null: false+ t.string "currency_code"+ t.string "currency_symbol"+ t.string "currency_name"t.datetime "created_at", null: falset.datetime "updated_at", null: falset.index ["country"], name: "index_stock_exchanges_on_country"diff --git a/test/controllers/account/trades_controller_test.rb b/test/controllers/account/trades_controller_test.rbindex 46f53046..b9c72852 100644--- a/test/controllers/account/trades_controller_test.rb+++ b/test/controllers/account/trades_controller_test.rb@@ -97,7 +97,7 @@ class Account::TradesControllerTest < ActionDispatch::IntegrationTestaccount_entry: {type: "buy",date: Date.current,- ticker: "NVDA",+ ticker: "NVDA (NASDAQ)",qty: 10,price: 10}@@ -118,7 +118,7 @@ class Account::TradesControllerTest < ActionDispatch::IntegrationTestpost account_trades_url(@entry.account), params: {account_entry: {type: "sell",- ticker: "AAPL",+ ticker: "AAPL (NYSE)",date: Date.current,currency: "USD",qty: 10,diff --git a/test/fixtures/stock_exchanges.yml b/test/fixtures/stock_exchanges.ymlnew file mode 100644index 00000000..872e5b27--- /dev/null+++ b/test/fixtures/stock_exchanges.yml@@ -0,0 +1,13 @@+nasdaq:+ name: NASDAQ+ mic: XNAS+ acronym: NASDAQ+ country: USA+ country_code: US++nyse:+ name: New York Stock Exchange+ mic: XNYS+ acronym: NYSE+ country: USA+ country_code: USdiff --git a/test/system/trades_test.rb b/test/system/trades_test.rbindex c79c3507..c9539276 100644--- a/test/system/trades_test.rb+++ b/test/system/trades_test.rb@@ -9,6 +9,16 @@ class TradesTest < ApplicationSystemTestCase@account = accounts(:investment)visit_account_trades++ Security::SynthComboboxOption.stubs(:find_in_synth).returns([+ Security::SynthComboboxOption.new(+ symbol: "AAPL",+ name: "Apple Inc.",+ logo_url: "https://logo.synthfinance.com/ticker/AAPL",+ exchange_acronym: "NASDAQ",+ exchange_mic: "XNAS"+ )+ ])endtest "can create buy transaction" do