The personal finance app for everyone
Add account data enrichment (#1532)
Zach Gollwitzerfe199f235784
diff --git a/app/controllers/settings/hostings_controller.rb b/app/controllers/settings/hostings_controller.rbindex 222ae018..97b8de92 100644--- a/app/controllers/settings/hostings_controller.rb+++ b/app/controllers/settings/hostings_controller.rb@@ -26,6 +26,10 @@ class Settings::HostingsController < SettingsControllerSetting.synth_api_key = hosting_params[:synth_api_key]end+ if hosting_params.key?(:data_enrichment_enabled)+ Setting.data_enrichment_enabled = hosting_params[:data_enrichment_enabled]+ end+redirect_to settings_hosting_path, notice: t(".success")rescue ActiveRecord::RecordInvalid => errorflash.now[:alert] = t(".failure")@@ -34,7 +38,7 @@ class Settings::HostingsController < SettingsControllerprivatedef hosting_params- params.require(:setting).permit(:render_deploy_hook, :upgrades_setting, :require_invite_for_signup, :synth_api_key)+ params.require(:setting).permit(:render_deploy_hook, :upgrades_setting, :require_invite_for_signup, :synth_api_key, :data_enrichment_enabled)enddef raise_if_not_self_hosteddiff --git a/app/jobs/enrich_data_job.rb b/app/jobs/enrich_data_job.rbnew file mode 100644index 00000000..97286b82--- /dev/null+++ b/app/jobs/enrich_data_job.rb@@ -0,0 +1,7 @@+class EnrichDataJob < ApplicationJob+ queue_as :default++ def perform(account)+ account.enrich_data+ end+enddiff --git a/app/models/account.rb b/app/models/account.rbindex 05931b7b..400e8bea 100644--- a/app/models/account.rb+++ b/app/models/account.rb@@ -126,6 +126,14 @@ class Account < ApplicationRecordclassification == "asset" ? "up" : "down"end+ def enrich_data+ DataEnricher.new(self).run+ end++ def enrich_data_later+ EnrichDataJob.perform_later(self)+ end+def update_with_sync!(attributes)transaction doupdate!(attributes)diff --git a/app/models/account/data_enricher.rb b/app/models/account/data_enricher.rbnew file mode 100644index 00000000..4beb0b92--- /dev/null+++ b/app/models/account/data_enricher.rb@@ -0,0 +1,61 @@+class Account::DataEnricher+ include Providable++ attr_reader :account++ def initialize(account)+ @account = account+ end++ def run+ enrich_transactions+ end++ private+ def enrich_transactions+ candidates = account.entries.account_transactions.includes(entryable: [ :merchant, :category ])++ Rails.logger.info("Enriching #{candidates.count} transactions for account #{account.id}")++ merchants = {}+ categories = {}++ candidates.each do |entry|+ if entry.enriched_at.nil? || entry.entryable.merchant_id.nil? || entry.entryable.category_id.nil?+ begin+ info = self.class.synth_provider.enrich_transaction(entry.name).info++ next unless info.present?++ if info.name.present?+ merchant = merchants[info.name] ||= account.family.merchants.find_or_create_by(name: info.name)++ if info.icon_url.present?+ merchant.icon_url = info.icon_url+ end+ end++ if info.category.present?+ category = categories[info.category] ||= account.family.categories.find_or_create_by(name: info.category)+ end++ entryable_attributes = { id: entry.entryable_id }+ entryable_attributes[:merchant_id] = merchant.id if merchant.present? && entry.entryable.merchant_id.nil?+ entryable_attributes[:category_id] = category.id if category.present? && entry.entryable.category_id.nil?++ Account.transaction do+ merchant.save! if merchant.present?+ category.save! if category.present?+ entry.update!(+ enriched_at: Time.current,+ name: entry.enriched_at.nil? ? info.name : entry.name,+ entryable_attributes: entryable_attributes+ )+ end+ rescue => e+ Rails.logger.warn("Error enriching transaction #{entry.id}: #{e.message}")+ end+ end+ end+ end+enddiff --git a/app/models/account/syncer.rb b/app/models/account/syncer.rbindex d42ff431..9160e64f 100644--- a/app/models/account/syncer.rb+++ b/app/models/account/syncer.rb@@ -10,6 +10,12 @@ class Account::Synceraccount.reloadupdate_account_info(balances, holdings) unless account.plaid_account_id.present?convert_records_to_family_currency(balances, holdings) unless account.currency == account.family.currency++ if Setting.data_enrichment_enabled || Rails.configuration.app_mode.managed?+ account.enrich_data_later+ else+ Rails.logger.info("Data enrichment is disabled, skipping enrichment for account #{account.id}")+ endendprivatediff --git a/app/models/concerns/providable.rb b/app/models/concerns/providable.rbindex 996efff8..4a8de8c0 100644--- a/app/models/concerns/providable.rb+++ b/app/models/concerns/providable.rb@@ -23,8 +23,10 @@ module Providableenddef synth_provider- api_key = self_hosted? ? Setting.synth_api_key : ENV["SYNTH_API_KEY"]- api_key.present? ? Provider::Synth.new(api_key) : nil+ @synth_provider ||= begin+ api_key = self_hosted? ? Setting.synth_api_key : ENV["SYNTH_API_KEY"]+ api_key.present? ? Provider::Synth.new(api_key) : nil+ endendprivatediff --git a/app/models/provider/synth.rb b/app/models/provider/synth.rbindex c212e992..b7735575 100644--- a/app/models/provider/synth.rb+++ b/app/models/provider/synth.rb@@ -167,6 +167,35 @@ class Provider::Synthraw_response: responseend+ def enrich_transaction(description, amount: nil, date: nil, city: nil, state: nil, country: nil)+ params = {+ description: description,+ amount: amount,+ date: date,+ city: city,+ state: state,+ country: country+ }.compact++ response = client.get("#{base_url}/enrich", params)++ parsed = JSON.parse(response.body)++ EnrichTransactionResponse.new \+ info: EnrichTransactionInfo.new(+ name: parsed.dig("merchant"),+ icon_url: parsed.dig("icon"),+ category: parsed.dig("category")+ ),+ success?: true,+ raw_response: response+ rescue StandardError => error+ EnrichTransactionResponse.new \+ success?: false,+ error: error,+ raw_response: error+ end+privateattr_reader :api_key@@ -177,6 +206,8 @@ class Provider::SynthUsageResponse = Struct.new :used, :limit, :utilization, :plan, :success?, :error, :raw_response, keyword_init: trueSearchSecuritiesResponse = Struct.new :securities, :success?, :error, :raw_response, keyword_init: trueSecurityInfoResponse = Struct.new :info, :success?, :error, :raw_response, keyword_init: true+ EnrichTransactionResponse = Struct.new :info, :success?, :error, :raw_response, keyword_init: true+ EnrichTransactionInfo = Struct.new :name, :icon_url, :category, keyword_init: truedef base_urlENV["SYNTH_URL"] || "https://api.synthfinance.com"diff --git a/app/models/setting.rb b/app/models/setting.rbindex d576fbea..eb1a9369 100644--- a/app/models/setting.rb+++ b/app/models/setting.rb@@ -17,6 +17,10 @@ class Setting < RailsSettings::Basedefault: ENV.fetch("UPGRADES_TARGET", "release"),validates: { inclusion: { in: %w[release commit] } }+ field :data_enrichment_enabled,+ type: :boolean,+ default: true+field :synth_api_key, type: :string, default: ENV["SYNTH_API_KEY"]field :require_invite_for_signup, type: :boolean, default: falsediff --git a/app/views/account/transactions/_transaction.html.erb b/app/views/account/transactions/_transaction.html.erbindex dc2a026a..9a37a958 100644--- a/app/views/account/transactions/_transaction.html.erb+++ b/app/views/account/transactions/_transaction.html.erb@@ -11,15 +11,17 @@<div class="max-w-full"><%= content_tag :div, class: ["flex items-center gap-2"] do %>- <div class="flex h-8 w-8 shrink-0 items-center justify-center rounded-full bg-gray-600/5 text-gray-600">- <%= transaction.name.first.upcase %>- </div>+ <% if entry.account_transaction.merchant&.icon_url %>+ <%= image_tag entry.account_transaction.merchant.icon_url, class: "w-6 h-6 rounded-full" %>+ <% else %>+ <%= render "shared/circle_logo", name: entry.name, size: "sm" %>+ <% end %><div class="truncate"><% if entry.new_record? %>- <%= content_tag :p, transaction.name %>+ <%= content_tag :p, entry.name %><% else %>- <%= link_to transaction.name,+ <%= link_to entry.name,entry.transfer.present? ? account_transfer_path(entry.transfer) : account_entry_path(entry),data: { turbo_frame: "drawer", turbo_prefetch: false },class: "hover:underline hover:text-gray-800" %>diff --git a/app/views/merchants/_merchant.html.erb b/app/views/merchants/_merchant.html.erbindex 2b964140..a454d41a 100644--- a/app/views/merchants/_merchant.html.erb+++ b/app/views/merchants/_merchant.html.erb@@ -2,7 +2,14 @@<div class="flex justify-between items-center p-4 bg-white"><div class="flex w-full items-center gap-2.5">- <%= render partial: "shared/color_avatar", locals: { name: merchant.name, color: merchant.color } %>+ <% if merchant.icon_url %>+ <div class="w-8 h-8 rounded-full flex justify-center items-center">+ <%= image_tag merchant.icon_url, class: "w-8 h-8 rounded-full" %>+ </div>+ <% else %>+ <%= render partial: "shared/color_avatar", locals: { name: merchant.name, color: merchant.color } %>+ <% end %>+<p class="text-gray-900 text-sm truncate"><%= merchant.name %></p>diff --git a/app/views/settings/hostings/_data_enrichment_settings.html.erb b/app/views/settings/hostings/_data_enrichment_settings.html.erbnew file mode 100644index 00000000..6d409923--- /dev/null+++ b/app/views/settings/hostings/_data_enrichment_settings.html.erb@@ -0,0 +1,18 @@+<div class="space-y-4">+ <div class="flex items-center justify-between">+ <div class="space-y-1">+ <p class="text-sm"><%= t(".title") %></p>+ <p class="text-gray-500 text-sm"><%= t(".description") %></p>+ </div>++ <%= styled_form_with model: Setting.new,+ url: settings_hosting_path,+ method: :patch,+ data: { controller: "auto-submit-form", "auto-submit-form-trigger-event-value": "blur" } do |form| %>+ <div class="relative inline-block select-none">+ <%= form.check_box :data_enrichment_enabled, class: "sr-only peer", "data-auto-submit-form-target": "auto", "data-autosubmit-trigger-event": "input" %>+ <%= form.label :data_enrichment_enabled, " ".html_safe, class: "maybe-switch" %>+ </div>+ <% end %>+ </div>+</div>diff --git a/app/views/settings/hostings/show.html.erb b/app/views/settings/hostings/show.html.erbindex ba4b7d5d..a2af0bed 100644--- a/app/views/settings/hostings/show.html.erb+++ b/app/views/settings/hostings/show.html.erb@@ -10,6 +10,7 @@<%= render "settings/hostings/upgrade_settings" %><%= render "settings/hostings/provider_settings" %><%= render "settings/hostings/synth_settings" %>+ <%= render "settings/hostings/data_enrichment_settings" %></div><% end %>diff --git a/config/locales/views/settings/hostings/en.yml b/config/locales/views/settings/hostings/en.ymlindex 90a89fd7..6b34a6cd 100644--- a/config/locales/views/settings/hostings/en.yml+++ b/config/locales/views/settings/hostings/en.yml@@ -2,6 +2,9 @@en:settings:hostings:+ data_enrichment_settings:+ description: Enable data enrichment for your accounts such as merchant info, transaction description cleanup, and more+ title: Data Enrichmentinvite_code_settings:description: Every new user that joins your instance of Maybe can only doso via an invite codediff --git a/db/migrate/20241212141453_add_merchant_logo.rb b/db/migrate/20241212141453_add_merchant_logo.rbnew file mode 100644index 00000000..81bd198c--- /dev/null+++ b/db/migrate/20241212141453_add_merchant_logo.rb@@ -0,0 +1,8 @@+class AddMerchantLogo < ActiveRecord::Migration[7.2]+ def change+ add_column :merchants, :icon_url, :string+ add_column :merchants, :enriched_at, :datetime++ add_column :account_entries, :enriched_at, :datetime+ end+enddiff --git a/db/schema.rb b/db/schema.rbindex 31183733..5fd3f26d 100644--- a/db/schema.rb+++ b/db/schema.rb@@ -10,7 +10,7 @@## It's strongly recommended that you check this file into your version control system.-ActiveRecord::Schema[7.2].define(version: 2024_12_07_002408) do+ActiveRecord::Schema[7.2].define(version: 2024_12_12_141453) do# These are extensions that must be enabled in order to support this databaseenable_extension "pgcrypto"enable_extension "plpgsql"@@ -48,6 +48,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_12_07_002408) dot.text "notes"t.boolean "excluded", default: falset.string "plaid_id"+ t.datetime "enriched_at"t.index ["account_id"], name: "index_account_entries_on_account_id"t.index ["import_id"], name: "index_account_entries_on_import_id"t.index ["transfer_id"], name: "index_account_entries_on_transfer_id"@@ -452,6 +453,8 @@ ActiveRecord::Schema[7.2].define(version: 2024_12_07_002408) dot.uuid "family_id", null: falset.datetime "created_at", null: falset.datetime "updated_at", null: false+ t.string "icon_url"+ t.datetime "enriched_at"t.index ["family_id"], name: "index_merchants_on_family_id"enddiff --git a/test/jobs/enrich_data_job_test.rb b/test/jobs/enrich_data_job_test.rbnew file mode 100644index 00000000..067767f6--- /dev/null+++ b/test/jobs/enrich_data_job_test.rb@@ -0,0 +1,7 @@+require "test_helper"++class EnrichDataJobTest < ActiveJob::TestCase+ # test "the truth" do+ # assert true+ # end+end