Skip to content

armgit / maybe

Public Imported from GitHub
0

The personal finance app for everyone

Stock imports (#1363)

Josh Pigfordaa3342b0dcbf
diff --git a/.env.example b/.env.example
index fd3e93c4..02865a5a 100644
--- a/.env.example
+++ b/.env.example
@@ -15,6 +15,10 @@ PORT=3000
# This is used to convert between different currencies in the app. In addition, it fetches US stock prices. We use Synth, which is a Maybe product. You can sign up for a free account at synthfinance.com.
SYNTH_API_KEY=
+# Non-US Stock Pricing API
+# This is used to fetch non-US stock prices. We use Marketstack.com for this and while they offer a free tier, it is quite limited. You'll almost certainly need their Basic plan, which is $9.99 per month.
+MARKETSTACK_API_KEY=
+
# SMTP Configuration
# This is only needed if you intend on sending emails from your Maybe instance (such as for password resets or email financial reports).
# Resend.com is a good option that offers a free tier for sending emails.
diff --git a/.env.local.example b/.env.local.example
index d393f623..cd38f220 100644
--- a/.env.local.example
+++ b/.env.local.example
@@ -3,3 +3,6 @@ SELF_HOSTED=false
# Enable Synth market data (careful, this will use your API credits)
SYNTH_API_KEY=yourapikeyhere
+
+# Enable Marketstack market data (careful, this will use your API credits)
+MARKETSTACK_API_KEY=yourapikeyhere
diff --git a/app/controllers/securities_controller.rb b/app/controllers/securities_controller.rb
new file mode 100644
index 00000000..24356118
--- /dev/null
+++ b/app/controllers/securities_controller.rb
@@ -0,0 +1,5 @@
+class SecuritiesController < ApplicationController
+ def import
+ SecuritiesImportJob.perform_later(params[:exchange_mic])
+ end
+end
diff --git a/app/helpers/securities_helper.rb b/app/helpers/securities_helper.rb
new file mode 100644
index 00000000..05762cb3
--- /dev/null
+++ b/app/helpers/securities_helper.rb
@@ -0,0 +1,2 @@
+module SecuritiesHelper
+end
diff --git a/app/jobs/securities_import_job.rb b/app/jobs/securities_import_job.rb
new file mode 100644
index 00000000..85fef904
--- /dev/null
+++ b/app/jobs/securities_import_job.rb
@@ -0,0 +1,13 @@
+class SecuritiesImportJob < ApplicationJob
+ queue_as :default
+
+ def perform(country_code = nil)
+ exchanges = StockExchange.in_country(country_code)
+ market_stack_client = Provider::Marketstack.new(ENV["MARKETSTACK_API_KEY"])
+
+ exchanges.each do |exchange|
+ importer = Security::Importer.new(market_stack_client, exchange.mic)
+ importer.import
+ end
+ end
+end
diff --git a/app/models/provider/marketstack.rb b/app/models/provider/marketstack.rb
new file mode 100644
index 00000000..5a1b8de9
--- /dev/null
+++ b/app/models/provider/marketstack.rb
@@ -0,0 +1,119 @@
+class Provider::Marketstack
+ include Retryable
+
+ def initialize(api_key)
+ @api_key = api_key
+ end
+
+ def fetch_security_prices(ticker:, start_date:, end_date:)
+ prices = paginate("#{base_url}/eod", {
+ symbols: ticker,
+ date_from: start_date.to_s,
+ date_to: end_date.to_s
+ }) do |body|
+ body.dig("data").map do |price|
+ {
+ date: price["date"],
+ price: price["close"]&.to_f,
+ currency: "USD"
+ }
+ end
+ end
+
+ SecurityPriceResponse.new(
+ prices: prices,
+ success?: true,
+ raw_response: prices.to_json
+ )
+ rescue StandardError => error
+ SecurityPriceResponse.new(
+ success?: false,
+ error: error,
+ raw_response: error
+ )
+ end
+
+ def fetch_tickers(exchange_mic: nil)
+ url = exchange_mic ? "#{base_url}/tickers?exchange=#{exchange_mic}" : "#{base_url}/tickers"
+ tickers = paginate(url) do |body|
+ body.dig("data").map do |ticker|
+ {
+ name: ticker["name"],
+ symbol: ticker["symbol"],
+ exchange: exchange_mic || ticker.dig("stock_exchange", "mic"),
+ country_code: ticker.dig("stock_exchange", "country_code")
+ }
+ end
+ end
+
+ TickerResponse.new(
+ tickers: tickers,
+ success?: true,
+ raw_response: tickers.to_json
+ )
+ rescue StandardError => error
+ TickerResponse.new(
+ success?: false,
+ error: error,
+ raw_response: error
+ )
+ end
+
+ private
+
+ attr_reader :api_key
+
+ SecurityPriceResponse = Struct.new(:prices, :success?, :error, :raw_response, keyword_init: true)
+ TickerResponse = Struct.new(:tickers, :success?, :error, :raw_response, keyword_init: true)
+
+ def base_url
+ "https://api.marketstack.com/v1"
+ end
+
+ def client
+ @client ||= Faraday.new(url: base_url) do |faraday|
+ faraday.params["access_key"] = api_key
+ end
+ end
+
+ def build_error(response)
+ Provider::Base::ProviderError.new(<<~ERROR)
+ Failed to fetch data from #{self.class}
+ Status: #{response.status}
+ Body: #{response.body.inspect}
+ ERROR
+ end
+
+ def fetch_page(url, page, params = {})
+ client.get(url) do |req|
+ params.each { |k, v| req.params[k.to_s] = v.to_s }
+ req.params["offset"] = (page - 1) * 100 # Marketstack uses offset-based pagination
+ req.params["limit"] = 10000 # Maximum allowed by Marketstack
+ end
+ end
+
+ def paginate(url, params = {})
+ results = []
+ page = 1
+ total_results = Float::INFINITY
+
+ while results.length < total_results
+ response = fetch_page(url, page, params)
+
+ if response.success?
+ body = JSON.parse(response.body)
+ page_results = yield(body)
+ results.concat(page_results)
+
+ total_results = body.dig("pagination", "total")
+ page += 1
+ else
+ raise build_error(response)
+ end
+
+ break if results.length >= total_results
+ end
+
+ results
+ end
+end
diff --git a/app/models/security/importer.rb b/app/models/security/importer.rb
new file mode 100644
index 00000000..4970170b
--- /dev/null
+++ b/app/models/security/importer.rb
@@ -0,0 +1,27 @@
+class Security::Importer
+ def initialize(provider, stock_exchange = nil)
+ @provider = provider
+ @stock_exchange = stock_exchange
+ end
+
+ def import
+ securities = @provider.fetch_tickers(exchange_mic: @stock_exchange)&.tickers
+
+ stock_exchanges = StockExchange.where(mic: securities.map { |s| s[:exchange] }).index_by(&:mic)
+ existing_securities = Security.where(ticker: securities.map { |s| s[:symbol] }, stock_exchange_id: stock_exchanges.values.map(&:id)).pluck(:ticker, :stock_exchange_id).to_set
+
+ securities_to_create = securities.map do |security|
+ stock_exchange_id = stock_exchanges[security[:exchange]]&.id
+ next if existing_securities.include?([ security[:symbol], stock_exchange_id ])
+
+ {
+ name: security[:name],
+ ticker: security[:symbol],
+ stock_exchange_id: stock_exchange_id,
+ country_code: security[:country_code]
+ }
+ end.compact
+
+ Security.insert_all(securities_to_create) unless securities_to_create.empty?
+ end
+end
diff --git a/app/models/stock_exchange.rb b/app/models/stock_exchange.rb
index 7141198e..93631426 100644
--- a/app/models/stock_exchange.rb
+++ b/app/models/stock_exchange.rb
@@ -1,2 +1,3 @@
class StockExchange < ApplicationRecord
+ scope :in_country, ->(country_code) { where(country_code: country_code) }
end
diff --git a/db/migrate/20241023195438_add_stock_exchange_reference.rb b/db/migrate/20241023195438_add_stock_exchange_reference.rb
new file mode 100644
index 00000000..a07588b2
--- /dev/null
+++ b/db/migrate/20241023195438_add_stock_exchange_reference.rb
@@ -0,0 +1,7 @@
+class AddStockExchangeReference < ActiveRecord::Migration[7.2]
+ def change
+ add_column :securities, :country_code, :string
+ add_reference :securities, :stock_exchange, type: :uuid, foreign_key: true
+ add_index :securities, :country_code
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 7fe73452..f008ff6f 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -119,7 +119,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_10_24_142537) do
t.boolean "is_active", default: true, null: false
t.date "last_sync_date"
t.uuid "institution_id"
- t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY ((ARRAY['Loan'::character varying, 'CreditCard'::character varying, 'OtherLiability'::character varying])::text[])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: true
+ t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY (ARRAY[('Loan'::character varying)::text, ('CreditCard'::character varying)::text, ('OtherLiability'::character varying)::text])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: true
t.uuid "import_id"
t.string "mode"
t.index ["accountable_id", "accountable_type"], name: "index_accounts_on_accountable_id_and_accountable_type"
@@ -478,6 +478,10 @@ ActiveRecord::Schema[7.2].define(version: 2024_10_24_142537) do
t.string "name"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
+ t.string "country_code"
+ t.uuid "stock_exchange_id"
+ t.index ["country_code"], name: "index_securities_on_country_code"
+ t.index ["stock_exchange_id"], name: "index_securities_on_stock_exchange_id"
end
create_table "security_prices", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
@@ -599,6 +603,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_10_24_142537) do
add_foreign_key "imports", "families"
add_foreign_key "institutions", "families"
add_foreign_key "merchants", "families"
+ add_foreign_key "securities", "stock_exchanges"
add_foreign_key "sessions", "impersonation_sessions", column: "active_impersonator_session_id"
add_foreign_key "sessions", "users"
add_foreign_key "taggings", "tags"