Skip to content

armgit / maybe

Public Imported from GitHub
0

The personal finance app for everyone

Add account data enrichment (#1532)

Zach Gollwitzerfe199f235784
diff --git a/app/controllers/settings/hostings_controller.rb b/app/controllers/settings/hostings_controller.rb
index 222ae018..97b8de92 100644
--- a/app/controllers/settings/hostings_controller.rb
+++ b/app/controllers/settings/hostings_controller.rb
@@ -26,6 +26,10 @@ class Settings::HostingsController < SettingsController
Setting.synth_api_key = hosting_params[:synth_api_key]
end
+ if hosting_params.key?(:data_enrichment_enabled)
+ Setting.data_enrichment_enabled = hosting_params[:data_enrichment_enabled]
+ end
+
redirect_to settings_hosting_path, notice: t(".success")
rescue ActiveRecord::RecordInvalid => error
flash.now[:alert] = t(".failure")
@@ -34,7 +38,7 @@ class Settings::HostingsController < SettingsController
private
def hosting_params
- params.require(:setting).permit(:render_deploy_hook, :upgrades_setting, :require_invite_for_signup, :synth_api_key)
+ params.require(:setting).permit(:render_deploy_hook, :upgrades_setting, :require_invite_for_signup, :synth_api_key, :data_enrichment_enabled)
end
def raise_if_not_self_hosted
diff --git a/app/jobs/enrich_data_job.rb b/app/jobs/enrich_data_job.rb
new file mode 100644
index 00000000..97286b82
--- /dev/null
+++ b/app/jobs/enrich_data_job.rb
@@ -0,0 +1,7 @@
+class EnrichDataJob < ApplicationJob
+ queue_as :default
+
+ def perform(account)
+ account.enrich_data
+ end
+end
diff --git a/app/models/account.rb b/app/models/account.rb
index 05931b7b..400e8bea 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -126,6 +126,14 @@ class Account < ApplicationRecord
classification == "asset" ? "up" : "down"
end
+ def enrich_data
+ DataEnricher.new(self).run
+ end
+
+ def enrich_data_later
+ EnrichDataJob.perform_later(self)
+ end
+
def update_with_sync!(attributes)
transaction do
update!(attributes)
diff --git a/app/models/account/data_enricher.rb b/app/models/account/data_enricher.rb
new file mode 100644
index 00000000..4beb0b92
--- /dev/null
+++ b/app/models/account/data_enricher.rb
@@ -0,0 +1,61 @@
+class Account::DataEnricher
+ include Providable
+
+ attr_reader :account
+
+ def initialize(account)
+ @account = account
+ end
+
+ def run
+ enrich_transactions
+ end
+
+ private
+ def enrich_transactions
+ candidates = account.entries.account_transactions.includes(entryable: [ :merchant, :category ])
+
+ Rails.logger.info("Enriching #{candidates.count} transactions for account #{account.id}")
+
+ merchants = {}
+ categories = {}
+
+ candidates.each do |entry|
+ if entry.enriched_at.nil? || entry.entryable.merchant_id.nil? || entry.entryable.category_id.nil?
+ begin
+ info = self.class.synth_provider.enrich_transaction(entry.name).info
+
+ next unless info.present?
+
+ if info.name.present?
+ merchant = merchants[info.name] ||= account.family.merchants.find_or_create_by(name: info.name)
+
+ if info.icon_url.present?
+ merchant.icon_url = info.icon_url
+ end
+ end
+
+ if info.category.present?
+ category = categories[info.category] ||= account.family.categories.find_or_create_by(name: info.category)
+ end
+
+ entryable_attributes = { id: entry.entryable_id }
+ entryable_attributes[:merchant_id] = merchant.id if merchant.present? && entry.entryable.merchant_id.nil?
+ entryable_attributes[:category_id] = category.id if category.present? && entry.entryable.category_id.nil?
+
+ Account.transaction do
+ merchant.save! if merchant.present?
+ category.save! if category.present?
+ entry.update!(
+ enriched_at: Time.current,
+ name: entry.enriched_at.nil? ? info.name : entry.name,
+ entryable_attributes: entryable_attributes
+ )
+ end
+ rescue => e
+ Rails.logger.warn("Error enriching transaction #{entry.id}: #{e.message}")
+ end
+ end
+ end
+ end
+end
diff --git a/app/models/account/syncer.rb b/app/models/account/syncer.rb
index d42ff431..9160e64f 100644
--- a/app/models/account/syncer.rb
+++ b/app/models/account/syncer.rb
@@ -10,6 +10,12 @@ class Account::Syncer
account.reload
update_account_info(balances, holdings) unless account.plaid_account_id.present?
convert_records_to_family_currency(balances, holdings) unless account.currency == account.family.currency
+
+ if Setting.data_enrichment_enabled || Rails.configuration.app_mode.managed?
+ account.enrich_data_later
+ else
+ Rails.logger.info("Data enrichment is disabled, skipping enrichment for account #{account.id}")
+ end
end
private
diff --git a/app/models/concerns/providable.rb b/app/models/concerns/providable.rb
index 996efff8..4a8de8c0 100644
--- a/app/models/concerns/providable.rb
+++ b/app/models/concerns/providable.rb
@@ -23,8 +23,10 @@ module Providable
end
def synth_provider
- api_key = self_hosted? ? Setting.synth_api_key : ENV["SYNTH_API_KEY"]
- api_key.present? ? Provider::Synth.new(api_key) : nil
+ @synth_provider ||= begin
+ api_key = self_hosted? ? Setting.synth_api_key : ENV["SYNTH_API_KEY"]
+ api_key.present? ? Provider::Synth.new(api_key) : nil
+ end
end
private
diff --git a/app/models/provider/synth.rb b/app/models/provider/synth.rb
index c212e992..b7735575 100644
--- a/app/models/provider/synth.rb
+++ b/app/models/provider/synth.rb
@@ -167,6 +167,35 @@ class Provider::Synth
raw_response: response
end
+ def enrich_transaction(description, amount: nil, date: nil, city: nil, state: nil, country: nil)
+ params = {
+ description: description,
+ amount: amount,
+ date: date,
+ city: city,
+ state: state,
+ country: country
+ }.compact
+
+ response = client.get("#{base_url}/enrich", params)
+
+ parsed = JSON.parse(response.body)
+
+ EnrichTransactionResponse.new \
+ info: EnrichTransactionInfo.new(
+ name: parsed.dig("merchant"),
+ icon_url: parsed.dig("icon"),
+ category: parsed.dig("category")
+ ),
+ success?: true,
+ raw_response: response
+ rescue StandardError => error
+ EnrichTransactionResponse.new \
+ success?: false,
+ error: error,
+ raw_response: error
+ end
+
private
attr_reader :api_key
@@ -177,6 +206,8 @@ class Provider::Synth
UsageResponse = Struct.new :used, :limit, :utilization, :plan, :success?, :error, :raw_response, keyword_init: true
SearchSecuritiesResponse = Struct.new :securities, :success?, :error, :raw_response, keyword_init: true
SecurityInfoResponse = Struct.new :info, :success?, :error, :raw_response, keyword_init: true
+ EnrichTransactionResponse = Struct.new :info, :success?, :error, :raw_response, keyword_init: true
+ EnrichTransactionInfo = Struct.new :name, :icon_url, :category, keyword_init: true
def base_url
ENV["SYNTH_URL"] || "https://api.synthfinance.com"
diff --git a/app/models/setting.rb b/app/models/setting.rb
index d576fbea..eb1a9369 100644
--- a/app/models/setting.rb
+++ b/app/models/setting.rb
@@ -17,6 +17,10 @@ class Setting < RailsSettings::Base
default: ENV.fetch("UPGRADES_TARGET", "release"),
validates: { inclusion: { in: %w[release commit] } }
+ field :data_enrichment_enabled,
+ type: :boolean,
+ default: true
+
field :synth_api_key, type: :string, default: ENV["SYNTH_API_KEY"]
field :require_invite_for_signup, type: :boolean, default: false
diff --git a/app/views/account/transactions/_transaction.html.erb b/app/views/account/transactions/_transaction.html.erb
index dc2a026a..9a37a958 100644
--- a/app/views/account/transactions/_transaction.html.erb
+++ b/app/views/account/transactions/_transaction.html.erb
@@ -11,15 +11,17 @@
<div class="max-w-full">
<%= content_tag :div, class: ["flex items-center gap-2"] do %>
- <div class="flex h-8 w-8 shrink-0 items-center justify-center rounded-full bg-gray-600/5 text-gray-600">
- <%= transaction.name.first.upcase %>
- </div>
+ <% if entry.account_transaction.merchant&.icon_url %>
+ <%= image_tag entry.account_transaction.merchant.icon_url, class: "w-6 h-6 rounded-full" %>
+ <% else %>
+ <%= render "shared/circle_logo", name: entry.name, size: "sm" %>
+ <% end %>
<div class="truncate">
<% if entry.new_record? %>
- <%= content_tag :p, transaction.name %>
+ <%= content_tag :p, entry.name %>
<% else %>
- <%= link_to transaction.name,
+ <%= link_to entry.name,
entry.transfer.present? ? account_transfer_path(entry.transfer) : account_entry_path(entry),
data: { turbo_frame: "drawer", turbo_prefetch: false },
class: "hover:underline hover:text-gray-800" %>
diff --git a/app/views/merchants/_merchant.html.erb b/app/views/merchants/_merchant.html.erb
index 2b964140..a454d41a 100644
--- a/app/views/merchants/_merchant.html.erb
+++ b/app/views/merchants/_merchant.html.erb
@@ -2,7 +2,14 @@
<div class="flex justify-between items-center p-4 bg-white">
<div class="flex w-full items-center gap-2.5">
- <%= render partial: "shared/color_avatar", locals: { name: merchant.name, color: merchant.color } %>
+ <% if merchant.icon_url %>
+ <div class="w-8 h-8 rounded-full flex justify-center items-center">
+ <%= image_tag merchant.icon_url, class: "w-8 h-8 rounded-full" %>
+ </div>
+ <% else %>
+ <%= render partial: "shared/color_avatar", locals: { name: merchant.name, color: merchant.color } %>
+ <% end %>
+
<p class="text-gray-900 text-sm truncate">
<%= merchant.name %>
</p>
diff --git a/app/views/settings/hostings/_data_enrichment_settings.html.erb b/app/views/settings/hostings/_data_enrichment_settings.html.erb
new file mode 100644
index 00000000..6d409923
--- /dev/null
+++ b/app/views/settings/hostings/_data_enrichment_settings.html.erb
@@ -0,0 +1,18 @@
+<div class="space-y-4">
+ <div class="flex items-center justify-between">
+ <div class="space-y-1">
+ <p class="text-sm"><%= t(".title") %></p>
+ <p class="text-gray-500 text-sm"><%= t(".description") %></p>
+ </div>
+
+ <%= styled_form_with model: Setting.new,
+ url: settings_hosting_path,
+ method: :patch,
+ data: { controller: "auto-submit-form", "auto-submit-form-trigger-event-value": "blur" } do |form| %>
+ <div class="relative inline-block select-none">
+ <%= form.check_box :data_enrichment_enabled, class: "sr-only peer", "data-auto-submit-form-target": "auto", "data-autosubmit-trigger-event": "input" %>
+ <%= form.label :data_enrichment_enabled, "&nbsp;".html_safe, class: "maybe-switch" %>
+ </div>
+ <% end %>
+ </div>
+</div>
diff --git a/app/views/settings/hostings/show.html.erb b/app/views/settings/hostings/show.html.erb
index ba4b7d5d..a2af0bed 100644
--- a/app/views/settings/hostings/show.html.erb
+++ b/app/views/settings/hostings/show.html.erb
@@ -10,6 +10,7 @@
<%= render "settings/hostings/upgrade_settings" %>
<%= render "settings/hostings/provider_settings" %>
<%= render "settings/hostings/synth_settings" %>
+ <%= render "settings/hostings/data_enrichment_settings" %>
</div>
<% end %>
diff --git a/config/locales/views/settings/hostings/en.yml b/config/locales/views/settings/hostings/en.yml
index 90a89fd7..6b34a6cd 100644
--- a/config/locales/views/settings/hostings/en.yml
+++ b/config/locales/views/settings/hostings/en.yml
@@ -2,6 +2,9 @@
en:
settings:
hostings:
+ data_enrichment_settings:
+ description: Enable data enrichment for your accounts such as merchant info, transaction description cleanup, and more
+ title: Data Enrichment
invite_code_settings:
description: Every new user that joins your instance of Maybe can only do
so via an invite code
diff --git a/db/migrate/20241212141453_add_merchant_logo.rb b/db/migrate/20241212141453_add_merchant_logo.rb
new file mode 100644
index 00000000..81bd198c
--- /dev/null
+++ b/db/migrate/20241212141453_add_merchant_logo.rb
@@ -0,0 +1,8 @@
+class AddMerchantLogo < ActiveRecord::Migration[7.2]
+ def change
+ add_column :merchants, :icon_url, :string
+ add_column :merchants, :enriched_at, :datetime
+
+ add_column :account_entries, :enriched_at, :datetime
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 31183733..5fd3f26d 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema[7.2].define(version: 2024_12_07_002408) do
+ActiveRecord::Schema[7.2].define(version: 2024_12_12_141453) do
# These are extensions that must be enabled in order to support this database
enable_extension "pgcrypto"
enable_extension "plpgsql"
@@ -48,6 +48,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_12_07_002408) do
t.text "notes"
t.boolean "excluded", default: false
t.string "plaid_id"
+ t.datetime "enriched_at"
t.index ["account_id"], name: "index_account_entries_on_account_id"
t.index ["import_id"], name: "index_account_entries_on_import_id"
t.index ["transfer_id"], name: "index_account_entries_on_transfer_id"
@@ -452,6 +453,8 @@ ActiveRecord::Schema[7.2].define(version: 2024_12_07_002408) do
t.uuid "family_id", null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
+ t.string "icon_url"
+ t.datetime "enriched_at"
t.index ["family_id"], name: "index_merchants_on_family_id"
end
diff --git a/test/jobs/enrich_data_job_test.rb b/test/jobs/enrich_data_job_test.rb
new file mode 100644
index 00000000..067767f6
--- /dev/null
+++ b/test/jobs/enrich_data_job_test.rb
@@ -0,0 +1,7 @@
+require "test_helper"
+
+class EnrichDataJobTest < ActiveJob::TestCase
+ # test "the truth" do
+ # assert true
+ # end
+end