Skip to content

armgit / maybe

Public Imported from GitHub
0

The personal finance app for everyone

Initial pass at Synth-based ticker selection (#1392)

Josh Pigfordcd91e6661885
diff --git a/.env.example b/.env.example
index 02865a5a..1169e684 100644
--- a/.env.example
+++ b/.env.example
@@ -11,14 +11,10 @@
# For users who have other applications listening at 3000, this allows them to set a value puma will listen to.
PORT=3000
-# Exchange Rate & US Stock Pricing API
-# This is used to convert between different currencies in the app. In addition, it fetches US stock prices. We use Synth, which is a Maybe product. You can sign up for a free account at synthfinance.com.
+# Exchange Rate & Stock Pricing API
+# This is used to convert between different currencies in the app. In addition, it fetches global stock prices. We use Synth, which is a Maybe product. You can sign up for a free account at synthfinance.com.
SYNTH_API_KEY=
-# Non-US Stock Pricing API
-# This is used to fetch non-US stock prices. We use Marketstack.com for this and while they offer a free tier, it is quite limited. You'll almost certainly need their Basic plan, which is $9.99 per month.
-MARKETSTACK_API_KEY=
-
# SMTP Configuration
# This is only needed if you intend on sending emails from your Maybe instance (such as for password resets or email financial reports).
# Resend.com is a good option that offers a free tier for sending emails.
diff --git a/.env.local.example b/.env.local.example
index cd38f220..d393f623 100644
--- a/.env.local.example
+++ b/.env.local.example
@@ -3,6 +3,3 @@ SELF_HOSTED=false
# Enable Synth market data (careful, this will use your API credits)
SYNTH_API_KEY=yourapikeyhere
-
-# Enable Marketstack market data (careful, this will use your API credits)
-MARKETSTACK_API_KEY=yourapikeyhere
diff --git a/app/controllers/account/trades_controller.rb b/app/controllers/account/trades_controller.rb
index 8b5a878c..0b8bc99d 100644
--- a/app/controllers/account/trades_controller.rb
+++ b/app/controllers/account/trades_controller.rb
@@ -34,7 +34,10 @@ class Account::TradesController < ApplicationController
end
def securities
- @pagy, @securities = pagy(Security.order(:name).search(params[:q]), limit: 20)
+ query = params[:q]
+ return render json: [] if query.blank? || query.length < 2 || query.length > 100
+
+ @securities = Security::SynthComboboxOption.find_in_synth(query)
end
private
diff --git a/app/jobs/securities_import_job.rb b/app/jobs/securities_import_job.rb
deleted file mode 100644
index 85fef904..00000000
--- a/app/jobs/securities_import_job.rb
+++ /dev/null
@@ -1,13 +0,0 @@
-class SecuritiesImportJob < ApplicationJob
- queue_as :default
-
- def perform(country_code = nil)
- exchanges = StockExchange.in_country(country_code)
- market_stack_client = Provider::Marketstack.new(ENV["MARKETSTACK_API_KEY"])
-
- exchanges.each do |exchange|
- importer = Security::Importer.new(market_stack_client, exchange.mic)
- importer.import
- end
- end
-end
diff --git a/app/models/account/trade_builder.rb b/app/models/account/trade_builder.rb
index 4610407f..3219fb8a 100644
--- a/app/models/account/trade_builder.rb
+++ b/app/models/account/trade_builder.rb
@@ -31,8 +31,9 @@ class Account::TradeBuilder < Account::EntryBuilder
end
def security
- return Security.find(ticker) if ticker.match?(/\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/i)
- Security.find_or_create_by(ticker: ticker)
+ ticker_symbol, exchange_mic = ticker.split("|")
+
+ Security.find_or_create_by(ticker: ticker_symbol, exchange_mic: exchange_mic)
end
def amount
diff --git a/app/models/provider/marketstack.rb b/app/models/provider/marketstack.rb
deleted file mode 100644
index 10e2aa5d..00000000
--- a/app/models/provider/marketstack.rb
+++ /dev/null
@@ -1,120 +0,0 @@
-class Provider::Marketstack
- include Retryable
-
- def initialize(api_key)
- @api_key = api_key
- end
-
- def fetch_security_prices(ticker:, start_date:, end_date:)
- prices = paginate("#{base_url}/eod", {
- symbols: ticker,
- date_from: start_date.to_s,
- date_to: end_date.to_s
- }) do |body|
- body.dig("data").map do |price|
- {
- date: price["date"],
- price: price["close"]&.to_f,
- currency: "USD"
- }
- end
- end
-
- SecurityPriceResponse.new(
- prices: prices,
- success?: true,
- raw_response: prices.to_json
- )
- rescue StandardError => error
- SecurityPriceResponse.new(
- success?: false,
- error: error,
- raw_response: error
- )
- end
-
- def fetch_tickers(exchange_mic: nil)
- url = exchange_mic ? "#{base_url}/tickers?exchange=#{exchange_mic}" : "#{base_url}/tickers"
- tickers = paginate(url) do |body|
- body.dig("data").map do |ticker|
- {
- name: ticker["name"],
- symbol: ticker["symbol"],
- exchange_mic: exchange_mic || ticker.dig("stock_exchange", "mic"),
- exchange_acronym: ticker.dig("stock_exchange", "acronym"),
- country_code: ticker.dig("stock_exchange", "country_code")
- }
- end
- end
-
- TickerResponse.new(
- tickers: tickers,
- success?: true,
- raw_response: tickers.to_json
- )
- rescue StandardError => error
- TickerResponse.new(
- success?: false,
- error: error,
- raw_response: error
- )
- end
-
- private
-
- attr_reader :api_key
-
- SecurityPriceResponse = Struct.new(:prices, :success?, :error, :raw_response, keyword_init: true)
- TickerResponse = Struct.new(:tickers, :success?, :error, :raw_response, keyword_init: true)
-
- def base_url
- "https://api.marketstack.com/v1"
- end
-
- def client
- @client ||= Faraday.new(url: base_url) do |faraday|
- faraday.params["access_key"] = api_key
- end
- end
-
- def build_error(response)
- Provider::Base::ProviderError.new(<<~ERROR)
- Failed to fetch data from #{self.class}
- Status: #{response.status}
- Body: #{response.body.inspect}
- ERROR
- end
-
- def fetch_page(url, page, params = {})
- client.get(url) do |req|
- params.each { |k, v| req.params[k.to_s] = v.to_s }
- req.params["offset"] = (page - 1) * 100 # Marketstack uses offset-based pagination
- req.params["limit"] = 10000 # Maximum allowed by Marketstack
- end
- end
-
- def paginate(url, params = {})
- results = []
- page = 1
- total_results = Float::INFINITY
-
- while results.length < total_results
- response = fetch_page(url, page, params)
-
- if response.success?
- body = JSON.parse(response.body)
- page_results = yield(body)
- results.concat(page_results)
-
- total_results = body.dig("pagination", "total")
- page += 1
- else
- raise build_error(response)
- end
-
- break if results.length >= total_results
- end
-
- results
- end
-end
diff --git a/app/models/provider/synth.rb b/app/models/provider/synth.rb
index d5f28e50..6dabd941 100644
--- a/app/models/provider/synth.rb
+++ b/app/models/provider/synth.rb
@@ -122,6 +122,31 @@ class Provider::Synth
raw_response: error
end
+ def search_securities(query:, dataset: "limited", country_code:)
+ response = client.get("#{base_url}/tickers/search") do |req|
+ req.params["name"] = query
+ req.params["dataset"] = dataset
+ req.params["country_code"] = country_code
+ end
+
+ parsed = JSON.parse(response.body)
+
+ securities = parsed.dig("data").map do |security|
+ {
+ symbol: security.dig("symbol"),
+ name: security.dig("name"),
+ logo_url: security.dig("logo_url"),
+ exchange_acronym: security.dig("exchange", "acronym"),
+ exchange_mic: security.dig("exchange", "mic_code")
+ }
+ end
+
+ SearchSecuritiesResponse.new \
+ securities: securities,
+ success?: true,
+ raw_response: response
+ end
+
private
attr_reader :api_key
@@ -130,6 +155,7 @@ class Provider::Synth
SecurityPriceResponse = Struct.new :prices, :success?, :error, :raw_response, keyword_init: true
ExchangeRatesResponse = Struct.new :rates, :success?, :error, :raw_response, keyword_init: true
UsageResponse = Struct.new :used, :limit, :utilization, :plan, :success?, :error, :raw_response, keyword_init: true
+ SearchSecuritiesResponse = Struct.new :securities, :success?, :error, :raw_response, keyword_init: true
def base_url
"https://api.synthfinance.com"
diff --git a/app/models/security.rb b/app/models/security.rb
index d1a160b8..732599ce 100644
--- a/app/models/security.rb
+++ b/app/models/security.rb
@@ -1,4 +1,5 @@
class Security < ApplicationRecord
+ include Providable
before_save :upcase_ticker
has_many :trades, dependent: :nullify, class_name: "Account::Trade"
@@ -7,25 +8,6 @@ class Security < ApplicationRecord
validates :ticker, presence: true
validates :ticker, uniqueness: { scope: :exchange_mic, case_sensitive: false }
- scope :search, ->(query) {
- return none if query.blank? || query.length < 2
-
- # Clean and normalize the search terms
- sanitized_query = query.split.map do |term|
- cleaned_term = term.gsub(/[^a-zA-Z0-9]/, " ").strip
- next if cleaned_term.blank?
- cleaned_term
- end.compact.join(" | ")
-
- return none if sanitized_query.blank?
-
- sanitized_query = ActiveRecord::Base.connection.quote(sanitized_query)
-
- where("search_vector @@ to_tsquery('simple', #{sanitized_query}) AND exchange_mic IS NOT NULL")
- .select("securities.*, ts_rank_cd(search_vector, to_tsquery('simple', #{sanitized_query})) AS rank")
- .reorder("rank DESC")
- }
-
def current_price
@current_price ||= Security::Price.find_price(security: self, date: Date.current)
return nil if @current_price.nil?
@@ -33,7 +15,7 @@ class Security < ApplicationRecord
end
def to_combobox_display
- "#{ticker} - #{name} (#{exchange_acronym})"
+ "#{ticker} (#{exchange_acronym})"
end
diff --git a/app/models/security/importer.rb b/app/models/security/importer.rb
deleted file mode 100644
index 4146fea2..00000000
--- a/app/models/security/importer.rb
+++ /dev/null
@@ -1,45 +0,0 @@
-class Security::Importer
- def initialize(provider, stock_exchange = nil)
- @provider = provider
- @stock_exchange = stock_exchange
- end
-
- def import
- securities = @provider.fetch_tickers(exchange_mic: @stock_exchange)&.tickers
-
- # Deduplicate securities based on ticker and exchange_mic
- securities_to_create = securities
- .map do |security|
- {
- name: security[:name],
- ticker: security[:symbol],
- country_code: security[:country_code],
- exchange_mic: security[:exchange_mic],
- exchange_acronym: security[:exchange_acronym]
- }
- end
- .compact
- .uniq { |security| [ security[:ticker], security[:exchange_mic] ] }
-
- # First update any existing securities that only have a ticker
- Security.where(exchange_mic: nil)
- .where(ticker: securities_to_create.map { |s| s[:ticker] })
- .update_all(
- securities_to_create.map do |security|
- {
- name: security[:name],
- country_code: security[:country_code],
- exchange_mic: security[:exchange_mic],
- exchange_acronym: security[:exchange_acronym]
- }
- end.first
- )
-
- # Then create/update any remaining securities
- Security.upsert_all(
- securities_to_create,
- unique_by: [ :ticker, :exchange_mic ],
- update_only: [ :name, :country_code, :exchange_acronym ]
- ) unless securities_to_create.empty?
- end
-end
diff --git a/app/models/security/synth_combobox_option.rb b/app/models/security/synth_combobox_option.rb
new file mode 100644
index 00000000..a4746ee7
--- /dev/null
+++ b/app/models/security/synth_combobox_option.rb
@@ -0,0 +1,20 @@
+class Security::SynthComboboxOption
+ include ActiveModel::Model
+ include Providable
+
+ attr_accessor :symbol, :name, :logo_url, :exchange_acronym, :exchange_mic
+
+ class << self
+ def find_in_synth(query)
+ security_prices_provider.search_securities(query:, dataset: "limited", country_code: Current.family.country).securities.map { |attrs| new(**attrs) }
+ end
+ end
+
+ def id
+ "#{symbol}|#{exchange_mic}" # submitted by combobox as value
+ end
+
+ def to_combobox_display
+ "#{symbol} - #{name} (#{exchange_acronym})" # shown in combobox input when selected
+ end
+end
diff --git a/app/views/account/trades/_form.html.erb b/app/views/account/trades/_form.html.erb
index 283a7725..f8c4033a 100644
--- a/app/views/account/trades/_form.html.erb
+++ b/app/views/account/trades/_form.html.erb
@@ -8,7 +8,7 @@
<%= form.select :type, options_for_select([%w[Buy buy], %w[Sell sell], %w[Deposit transfer_in], %w[Withdrawal transfer_out], %w[Interest interest]], "buy"), { label: t(".type") }, { data: { "trade-form-target": "typeInput" } } %>
<div data-trade-form-target="tickerInput">
<div class="form-field combobox">
- <%= form.combobox :ticker, securities_account_trades_path(entry.account), label: t(".holding"), placeholder: t(".ticker_placeholder"), autocomplete: :list, free_text: true %>
+ <%= form.combobox :ticker, securities_account_trades_path(entry.account), label: t(".holding"), placeholder: t(".ticker_placeholder") %>
</div>
</div>
diff --git a/app/views/account/trades/_security.turbo_stream.erb b/app/views/account/trades/_security.turbo_stream.erb
new file mode 100644
index 00000000..f270b028
--- /dev/null
+++ b/app/views/account/trades/_security.turbo_stream.erb
@@ -0,0 +1,11 @@
+<div class="flex items-center">
+ <%= image_tag(security.logo_url, class: "rounded-full h-8 w-8 inline-block mr-2" ) %>
+ <div class="flex flex-col">
+ <span class="text-sm font-medium">
+ <%= security.name.presence || security.symbol %>
+ </span>
+ <span class="text-xs text-gray-500">
+ <%= "#{security.symbol} (#{security.exchange_acronym})" %>
+ </span>
+ </div>
+</div>
\ No newline at end of file
diff --git a/app/views/account/trades/_tickers.turbo_stream.erb b/app/views/account/trades/_tickers.turbo_stream.erb
deleted file mode 100644
index 56436ab8..00000000
--- a/app/views/account/trades/_tickers.turbo_stream.erb
+++ /dev/null
@@ -1,7 +0,0 @@
-<div class="flex items-center">
- <%= image_tag("https://logo.synthfinance.com/ticker/#{tickers&.ticker}", class: "rounded-full h-8 w-8 inline-block mr-2") %>
- <div class="flex flex-col">
- <span class="text-sm font-medium"><%= tickers&.name.presence || tickers&.ticker %></span>
- <span class="text-xs text-gray-500"><%= "#{tickers&.ticker} (#{tickers&.exchange_acronym})" %></span>
- </div>
-</div>
diff --git a/app/views/account/trades/securities.turbo_stream.erb b/app/views/account/trades/securities.turbo_stream.erb
index 7ed8e90e..a3128c77 100644
--- a/app/views/account/trades/securities.turbo_stream.erb
+++ b/app/views/account/trades/securities.turbo_stream.erb
@@ -1,3 +1,2 @@
<%= async_combobox_options @securities,
- render_in: { partial: "account/trades/tickers" },
- next_page: @pagy.next %>
\ No newline at end of file
+ render_in: { partial: "account/trades/security" } %>
\ No newline at end of file
diff --git a/db/migrate/20241029234028_remove_search_vector.rb b/db/migrate/20241029234028_remove_search_vector.rb
new file mode 100644
index 00000000..99d94602
--- /dev/null
+++ b/db/migrate/20241029234028_remove_search_vector.rb
@@ -0,0 +1,5 @@
+class RemoveSearchVector < ActiveRecord::Migration[7.2]
+ def change
+ remove_column :securities, :search_vector
+ end
+end
diff --git a/db/migrate/20241030121302_fix_not_null_stock_exchange_data.rb b/db/migrate/20241030121302_fix_not_null_stock_exchange_data.rb
new file mode 100644
index 00000000..9a1f1012
--- /dev/null
+++ b/db/migrate/20241030121302_fix_not_null_stock_exchange_data.rb
@@ -0,0 +1,10 @@
+class FixNotNullStockExchangeData < ActiveRecord::Migration[7.2]
+ def change
+ change_column_null :stock_exchanges, :currency_code, true
+ change_column_null :stock_exchanges, :currency_symbol, true
+ change_column_null :stock_exchanges, :currency_name, true
+ change_column_null :stock_exchanges, :city, true
+ change_column_null :stock_exchanges, :timezone_name, true
+ change_column_null :stock_exchanges, :timezone_abbr, true
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 507fd0f9..d9d597d7 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema[7.2].define(version: 2024_10_29_184115) do
+ActiveRecord::Schema[7.2].define(version: 2024_10_30_121302) do
# These are extensions that must be enabled in order to support this database
enable_extension "pgcrypto"
enable_extension "plpgsql"
@@ -119,7 +119,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_10_29_184115) do
t.boolean "is_active", default: true, null: false
t.date "last_sync_date"
t.uuid "institution_id"
- t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY ((ARRAY['Loan'::character varying, 'CreditCard'::character varying, 'OtherLiability'::character varying])::text[])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: true
+ t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY (ARRAY[('Loan'::character varying)::text, ('CreditCard'::character varying)::text, ('OtherLiability'::character varying)::text])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: true
t.uuid "import_id"
t.string "mode"
t.index ["accountable_id", "accountable_type"], name: "index_accounts_on_accountable_id_and_accountable_type"
@@ -481,9 +481,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_10_29_184115) do
t.string "country_code"
t.string "exchange_mic"
t.string "exchange_acronym"
- t.virtual "search_vector", type: :tsvector, as: "(setweight(to_tsvector('simple'::regconfig, (COALESCE(ticker, ''::character varying))::text), 'B'::\"char\") || to_tsvector('simple'::regconfig, (COALESCE(name, ''::character varying))::text))", stored: true
t.index ["country_code"], name: "index_securities_on_country_code"
- t.index ["search_vector"], name: "index_securities_on_search_vector", using: :gin
t.index ["ticker", "exchange_mic"], name: "index_securities_on_ticker_and_exchange_mic", unique: true
end
@@ -524,14 +522,14 @@ ActiveRecord::Schema[7.2].define(version: 2024_10_29_184115) do
t.string "mic", null: false
t.string "country", null: false
t.string "country_code", null: false
- t.string "city", null: false
+ t.string "city"
t.string "website"
- t.string "timezone_name", null: false
- t.string "timezone_abbr", null: false
+ t.string "timezone_name"
+ t.string "timezone_abbr"
t.string "timezone_abbr_dst"
- t.string "currency_code", null: false
- t.string "currency_symbol", null: false
- t.string "currency_name", null: false
+ t.string "currency_code"
+ t.string "currency_symbol"
+ t.string "currency_name"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["country"], name: "index_stock_exchanges_on_country"
diff --git a/test/controllers/account/trades_controller_test.rb b/test/controllers/account/trades_controller_test.rb
index 46f53046..b9c72852 100644
--- a/test/controllers/account/trades_controller_test.rb
+++ b/test/controllers/account/trades_controller_test.rb
@@ -97,7 +97,7 @@ class Account::TradesControllerTest < ActionDispatch::IntegrationTest
account_entry: {
type: "buy",
date: Date.current,
- ticker: "NVDA",
+ ticker: "NVDA (NASDAQ)",
qty: 10,
price: 10
}
@@ -118,7 +118,7 @@ class Account::TradesControllerTest < ActionDispatch::IntegrationTest
post account_trades_url(@entry.account), params: {
account_entry: {
type: "sell",
- ticker: "AAPL",
+ ticker: "AAPL (NYSE)",
date: Date.current,
currency: "USD",
qty: 10,
diff --git a/test/fixtures/stock_exchanges.yml b/test/fixtures/stock_exchanges.yml
new file mode 100644
index 00000000..872e5b27
--- /dev/null
+++ b/test/fixtures/stock_exchanges.yml
@@ -0,0 +1,13 @@
+nasdaq:
+ name: NASDAQ
+ mic: XNAS
+ acronym: NASDAQ
+ country: USA
+ country_code: US
+
+nyse:
+ name: New York Stock Exchange
+ mic: XNYS
+ acronym: NYSE
+ country: USA
+ country_code: US
diff --git a/test/system/trades_test.rb b/test/system/trades_test.rb
index c79c3507..c9539276 100644
--- a/test/system/trades_test.rb
+++ b/test/system/trades_test.rb
@@ -9,6 +9,16 @@ class TradesTest < ApplicationSystemTestCase
@account = accounts(:investment)
visit_account_trades
+
+ Security::SynthComboboxOption.stubs(:find_in_synth).returns([
+ Security::SynthComboboxOption.new(
+ symbol: "AAPL",
+ name: "Apple Inc.",
+ logo_url: "https://logo.synthfinance.com/ticker/AAPL",
+ exchange_acronym: "NASDAQ",
+ exchange_mic: "XNAS"
+ )
+ ])
end
test "can create buy transaction" do