The personal finance app for everyone
Allow CSV imports to be configured with single or multi-account mode (#1943)
Zach Gollwitzerc5da8ea5502f
diff --git a/app/controllers/import/confirms_controller.rb b/app/controllers/import/confirms_controller.rbindex 0c1a8872..1a687d4b 100644--- a/app/controllers/import/confirms_controller.rb+++ b/app/controllers/import/confirms_controller.rb@@ -4,6 +4,10 @@ class Import::ConfirmsController < ApplicationControllerbefore_action :set_importdef show+ if @import.mapping_steps.empty?+ return redirect_to import_path(@import)+ end+redirect_to import_clean_path(@import), alert: "You have invalid data, please edit until all errors are resolved" unless @import.cleaned?enddiff --git a/app/controllers/import/uploads_controller.rb b/app/controllers/import/uploads_controller.rbindex 8efc7c75..42e6c975 100644--- a/app/controllers/import/uploads_controller.rb+++ b/app/controllers/import/uploads_controller.rb@@ -8,6 +8,7 @@ class Import::UploadsController < ApplicationControllerdef updateif csv_valid?(csv_str)+ @import.account = Current.family.accounts.find_by(id: params.dig(:import, :account_id))@import.assign_attributes(raw_file_str: csv_str, col_sep: upload_params[:col_sep])@import.save!(validate: false)diff --git a/app/controllers/imports_controller.rb b/app/controllers/imports_controller.rbindex 3fb704fe..e95e6e11 100644--- a/app/controllers/imports_controller.rb+++ b/app/controllers/imports_controller.rb@@ -18,7 +18,8 @@ class ImportsController < ApplicationControllerenddef create- import = Current.family.imports.create! import_params+ account = Current.family.accounts.find_by(id: params.dig(:import, :account_id))+ import = Current.family.imports.create!(type: import_params[:type], account: account)redirect_to import_upload_path(import)enddiff --git a/app/models/family.rb b/app/models/family.rbindex ffb14a7d..0f71731f 100644--- a/app/models/family.rb+++ b/app/models/family.rb@@ -106,7 +106,7 @@ class Family < ApplicationRecord# If family has any entries in different currencies, they need a provider for historical exchange ratesuniq_currencies = entries.pluck(:currency).uniqreturn true if uniq_currencies.count > 1- return true if uniq_currencies.first != self.currency+ return true if uniq_currencies.count > 0 && uniq_currencies.first != self.currencyfalseenddiff --git a/app/models/import.rb b/app/models/import.rbindex 9a21d6bd..600f30a7 100644--- a/app/models/import.rb+++ b/app/models/import.rb@@ -1,6 +1,7 @@class Import < ApplicationRecordTYPES = %w[TransactionImport TradeImport AccountImport MintImport].freezeSIGNAGE_CONVENTIONS = %w[inflows_positive inflows_negative]+ SEPARATORS = [ [ "Comma (,)", "," ], [ "Semicolon (;)", ";" ] ].freezeNUMBER_FORMATS = {"1,234.56" => { separator: ".", delimiter: "," }, # US/UK/Asia@@ -10,6 +11,7 @@ class Import < ApplicationRecord}.freezebelongs_to :family+ belongs_to :account, optional: truebefore_validation :set_default_number_format@@ -25,7 +27,7 @@ class Import < ApplicationRecord}, validate: true, default: "pending"validates :type, inclusion: { in: TYPES }- validates :col_sep, inclusion: { in: [ ",", ";" ] }+ validates :col_sep, inclusion: { in: SEPARATORS.map(&:last) }validates :signage_convention, inclusion: { in: SIGNAGE_CONVENTIONS }validates :number_format, presence: true, inclusion: { in: NUMBER_FORMATS.keys }@@ -98,12 +100,17 @@ class Import < ApplicationRecordenddef dry_run- {+ mappings = {transactions: rows.count,- accounts: Import::AccountMapping.for_import(self).creational.count,categories: Import::CategoryMapping.for_import(self).creational.count,tags: Import::TagMapping.for_import(self).creational.count}++ mappings.merge(+ accounts: Import::AccountMapping.for_import(self).creational.count,+ ) if account.nil?++ mappingsenddef required_column_keysdiff --git a/app/models/import/account_mapping.rb b/app/models/import/account_mapping.rbindex c4c00414..9cf43f7a 100644--- a/app/models/import/account_mapping.rb+++ b/app/models/import/account_mapping.rb@@ -1,5 +1,5 @@class Import::AccountMapping < Import::Mapping- validates :mappable, presence: true, if: -> { key.blank? || !create_when_empty }+ validates :mappable, presence: true, if: :requires_mapping?class << selfdef mapping_values(import)@@ -42,4 +42,9 @@ class Import::AccountMapping < Import::Mappingself.mappable = accountsave!end++ private+ def requires_mapping?+ (key.blank? || !create_when_empty) && import.account.nil?+ endenddiff --git a/app/models/trade_import.rb b/app/models/trade_import.rbindex 549c9093..4bbffdaa 100644--- a/app/models/trade_import.rb+++ b/app/models/trade_import.rb@@ -4,7 +4,11 @@ class TradeImport < Importmappings.each(&:create_mappable!)rows.each do |row|- account = mappings.accounts.mappable_for(row.account)+ mapped_account = if account+ account+ else+ mappings.accounts.mappable_for(row.account)+ end# Try to find or create security with ticker onlysecurity = find_or_create_security(@@ -12,15 +16,15 @@ class TradeImport < Importexchange_operating_mic: row.exchange_operating_mic)- entry = account.entries.build \+ entry = mapped_account.entries.build \date: row.date_iso,amount: row.signed_amount,name: row.name,- currency: row.currency.presence || account.currency,+ currency: row.currency.presence || mapped_account.currency,entryable: Account::Trade.new(security: security,qty: row.qty,- currency: row.currency.presence || account.currency,+ currency: row.currency.presence || mapped_account.currency,price: row.price),import: self@@ -31,7 +35,9 @@ class TradeImport < Importenddef mapping_steps- [ Import::AccountMapping ]+ base = []+ base << Import::AccountMapping if account.nil?+ baseenddef required_column_keys@@ -39,14 +45,19 @@ class TradeImport < Importenddef column_keys- %i[date ticker exchange_operating_mic currency qty price account name]+ base = %i[date ticker exchange_operating_mic currency qty price name]+ base.unshift(:account) if account.nil?+ baseenddef dry_run- {- transactions: rows.count,+ mappings = { transactions: rows.count }++ mappings.merge(accounts: Import::AccountMapping.for_import(self).creational.count- }+ ) if account.nil?++ mappingsenddef csv_template@@ -57,7 +68,9 @@ class TradeImport < Import05/17/2024,TSLA,XNAS,USD,2,700.50,Retirement Account,Tesla Inc. PurchaseCSV- CSV.parse(template, headers: true)+ csv = CSV.parse(template, headers: true)+ csv.delete("account") if account.present?+ csvendprivatediff --git a/app/models/transaction_import.rb b/app/models/transaction_import.rbindex e2b1c3d6..2bb9d4d5 100644--- a/app/models/transaction_import.rb+++ b/app/models/transaction_import.rb@@ -4,11 +4,16 @@ class TransactionImport < Importmappings.each(&:create_mappable!)rows.each do |row|- account = mappings.accounts.mappable_for(row.account)+ mapped_account = if account+ account+ else+ mappings.accounts.mappable_for(row.account)+ end+category = mappings.categories.mappable_for(row.category)tags = row.tags_list.map { |tag| mappings.tags.mappable_for(tag) }.compact- entry = account.entries.build \+ entry = mapped_account.entries.build \date: row.date_iso,amount: row.signed_amount,name: row.name,@@ -27,11 +32,15 @@ class TransactionImport < Importenddef column_keys- %i[date amount name currency category tags account notes]+ base = %i[date amount name currency category tags notes]+ base.unshift(:account) if account.nil?+ baseenddef mapping_steps- [ Import::CategoryMapping, Import::TagMapping, Import::AccountMapping ]+ base = [ Import::CategoryMapping, Import::TagMapping ]+ base << Import::AccountMapping if account.nil?+ baseenddef csv_template@@ -42,6 +51,8 @@ class TransactionImport < Import05/17/2024,-12.50,Coffee Shop,,,coffee,,CSV- CSV.parse(template, headers: true)+ csv = CSV.parse(template, headers: true)+ csv.delete("account") if account.present?+ csvendenddiff --git a/app/views/account/trades/_form.html.erb b/app/views/account/trades/_form.html.erbindex a74d401f..89c8c151 100644--- a/app/views/account/trades/_form.html.erb+++ b/app/views/account/trades/_form.html.erb@@ -29,11 +29,11 @@<% if %w[buy sell].include?(type) %><% if Security.provider.present? %><div class="form-field combobox">- <%= form.combobox :ticker,- securities_path(country_code: Current.family.country),+ <%= form.combobox :ticker,+ securities_path(country_code: Current.family.country),name_when_new: "account_entry[manual_ticker]",- label: t(".holding"),- placeholder: t(".ticker_placeholder"),+ label: t(".holding"),+ placeholder: t(".ticker_placeholder"),required: true %></div><% else %>diff --git a/app/views/accounts/show/_menu.html.erb b/app/views/accounts/show/_menu.html.erbindex f41723a6..57241528 100644--- a/app/views/accounts/show/_menu.html.erb+++ b/app/views/accounts/show/_menu.html.erb@@ -20,8 +20,8 @@<% end %><% unless account.crypto? %>- <%= link_to new_import_path,- data: { turbo_frame: :modal },+ <%= button_to imports_path({ import: { type: account.investment? ? "TradeImport" : "TransactionImport", account_id: account.id } }),+ data: { turbo_frame: :_top },class: "block w-full py-2 px-3 space-x-2 text-primary hover:bg-gray-50 flex items-center rounded-lg" do %><%= lucide_icon "download", class: "w-5 h-5 text-secondary" %>diff --git a/app/views/import/configurations/_mint_import.html.erb b/app/views/import/configurations/_mint_import.html.erbindex 1aabce75..987f2fed 100644--- a/app/views/import/configurations/_mint_import.html.erb+++ b/app/views/import/configurations/_mint_import.html.erb@@ -1,6 +1,6 @@<%# locals: (import:) %>-<div class="flex items-center justify-between border border-secondary rounded-lg bg-green-500/5 p-5 gap-4">+<div class="flex items-center justify-between border border-secondary rounded-lg bg-green-500/5 p-5 gap-4 mb-4"><%= lucide_icon("check-circle", class: "w-5 h-5 shrink-0 text-green-500") %><p class="text-sm text-primary italic">We have pre-configured your Mint import for you. Please proceed to the next step.</p></div>@@ -21,7 +21,10 @@<%= form.select :number_format, Import::NUMBER_FORMATS.keys, { label: "Format", prompt: "Select format" }, required: true %></div>- <%= form.select :account_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Account (optional)" }, disabled: import.complete? %>+ <% unless import.account.present? %>+ <%= form.select :account_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Account (optional)" }, disabled: import.complete? %>+ <% end %>+<%= form.select :name_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Name (optional)" }, disabled: import.complete? %><%= form.select :category_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Category (optional)" }, disabled: import.complete? %><%= form.select :tags_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Tags (optional)" }, disabled: import.complete? %>diff --git a/app/views/import/configurations/_trade_import.html.erb b/app/views/import/configurations/_trade_import.html.erbindex f923940c..7db04ba9 100644--- a/app/views/import/configurations/_trade_import.html.erb+++ b/app/views/import/configurations/_trade_import.html.erb@@ -20,7 +20,11 @@<%= form.select :ticker_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Ticker" } %><%= form.select :exchange_operating_mic_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Exchange Operating MIC" } %><%= form.select :price_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Price" } %>- <%= form.select :account_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Account (optional)" } %>++ <% unless import.account.present? %>+ <%= form.select :account_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Account (optional)" } %>+ <% end %>+<%= form.select :name_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Name (optional)" } %><% unless Security.provider %>diff --git a/app/views/import/configurations/_transaction_import.html.erb b/app/views/import/configurations/_transaction_import.html.erbindex 65c9f5f4..a75b1681 100644--- a/app/views/import/configurations/_transaction_import.html.erb+++ b/app/views/import/configurations/_transaction_import.html.erb@@ -16,7 +16,10 @@<%= form.select :number_format, Import::NUMBER_FORMATS.keys, { label: "Format", prompt: "Select format" }, required: true %></div>- <%= form.select :account_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Account (optional)" } %>+ <% unless import.account.present? %>+ <%= form.select :account_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Account (optional)" } %>+ <% end %>+<%= form.select :name_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Name (optional)" } %><%= form.select :category_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Category (optional)" } %><%= form.select :tags_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Tags (optional)" } %>diff --git a/app/views/import/confirms/_mappings.html.erb b/app/views/import/confirms/_mappings.html.erbindex 2880ec6d..0f07eb2f 100644--- a/app/views/import/confirms/_mappings.html.erb+++ b/app/views/import/confirms/_mappings.html.erb@@ -3,7 +3,7 @@<% mappings = mapping_class.for_import(import) %><% is_last_step = step_idx == import.mapping_steps.count - 1 %>-<% if mapping_class == Import::AccountMapping %>+<% if mapping_class == Import::AccountMapping && import.account.nil? %><% if import.requires_account? %><div class="flex items-center justify-between p-4 mb-4 gap-4 text-secondary bg-red-100 border border-red-200 rounded-lg w-[650px]"><%= tag.p t(".no_accounts"), class: "text-sm" %>diff --git a/app/views/import/uploads/show.html.erb b/app/views/import/uploads/show.html.erbindex 2f126a1c..ab7e4bb2 100644--- a/app/views/import/uploads/show.html.erb+++ b/app/views/import/uploads/show.html.erb@@ -19,33 +19,33 @@</div></div>- <div data-tabs-target="tab" id="csv-paste-tab">- <%= styled_form_with model: @import, scope: :import, url: import_upload_path(@import), multipart: true, class: "space-y-2" do |form| %>- <%= form.select :col_sep, [["Comma (,)", ","], ["Semicolon (;)", ";"]], label: true %>- <%= form.text_area :raw_file_str,+ <% ["csv-paste-tab", "csv-upload-tab"].each do |tab| %>+ <%= tag.div id: tab, data: { tabs_target: "tab" }, class: tab == "csv-upload-tab" ? "hidden" : "" do %>+ <%= styled_form_with model: @import, scope: :import, url: import_upload_path(@import), multipart: true, class: "space-y-2" do |form| %>+ <%= form.select :col_sep, Import::SEPARATORS, label: true %>++ <% unless @import.type == "MintImport" %>+ <%= form.select :account_id, @import.family.accounts.pluck(:name, :id), { label: "Account (optional)", include_blank: "Multi-account import", selected: @import.account_id } %>+ <% end %>++ <% if tab == "csv-paste-tab" %>+ <%= form.text_area :raw_file_str,rows: 10,required: true,placeholder: "Paste your CSV file contents here","data-auto-submit-form-target": "auto" %>+ <% else %>+ <label for="import_csv_file" class="flex flex-col items-center justify-center w-full h-56 border-2 border-gray-300 border-dashed rounded-lg cursor-pointer bg-gray-50">+ <div class="flex flex-col items-center justify-center pt-5 pb-6">+ <%= form.file_field :csv_file, class: "ml-32", "data-auto-submit-form-target": "auto" %>+ </div>+ </label>+ <% end %>- <%= form.submit "Upload CSV", disabled: @import.complete? %>+ <%= form.submit "Upload CSV", disabled: @import.complete? %>+ <% end %><% end %>-- </div>-- <div data-tabs-target="tab" id="csv-upload-tab" class="hidden">- <%= styled_form_with model: @import, scope: :import, url: import_upload_path(@import), multipart: true, class: "space-y-2" do |form| %>- <%= form.select :col_sep, [["Comma (,)", ","], ["Semicolon (;)", ";"]], label: true %>-- <label for="import_csv_file" class="flex flex-col items-center justify-center w-full h-56 border-2 border-gray-300 border-dashed rounded-lg cursor-pointer bg-gray-50">- <div class="flex flex-col items-center justify-center pt-5 pb-6">- <%= form.file_field :csv_file, class: "ml-32", "data-auto-submit-form-target": "auto" %>- </div>- </label>-- <%= form.submit "Upload CSV", disabled: @import.complete? %>- <% end %>- </div>+ <% end %></div></div>diff --git a/app/views/imports/_import.html.erb b/app/views/imports/_import.html.erbindex 3ee7d68c..26c77ee9 100644--- a/app/views/imports/_import.html.erb+++ b/app/views/imports/_import.html.erb@@ -2,6 +2,10 @@<div class="flex items-center gap-2 mb-1"><%= link_to import_path(import), class: "text-sm text-primary hover:underline" do %>+ <% if import.account.present? %>+ <%= import.account.name + " " %>+ <% end %>+<%= t(".label", type: import.type.titleize, datetime: import.updated_at.strftime("%b %-d, %Y at %l:%M %p")) %><% end %>diff --git a/app/views/imports/_nav.html.erb b/app/views/imports/_nav.html.erbindex b68b6ce1..0b9bd817 100644--- a/app/views/imports/_nav.html.erb+++ b/app/views/imports/_nav.html.erb@@ -6,7 +6,7 @@{ name: "Clean", path: import_clean_path(import), is_complete: import.cleaned?, step_number: 3 },{ name: "Map", path: import_confirm_path(import), is_complete: import.publishable?, step_number: 4 },{ name: "Confirm", path: import_path(import), is_complete: import.complete?, step_number: 5 }-] %>+].reject { |step| step[:name] == "Map" && import.mapping_steps.empty? } %><ul class="flex items-center gap-2"><% steps.each_with_index do |step, idx| %>diff --git a/db/migrate/20250303141007_add_optional_account_for_import.rb b/db/migrate/20250303141007_add_optional_account_for_import.rbnew file mode 100644index 00000000..52da03d9--- /dev/null+++ b/db/migrate/20250303141007_add_optional_account_for_import.rb@@ -0,0 +1,5 @@+class AddOptionalAccountForImport < ActiveRecord::Migration[7.2]+ def change+ rename_column :imports, :original_account_id, :account_id+ end+enddiff --git a/db/schema.rb b/db/schema.rbindex d78271ac..492edb1e 100644--- a/db/schema.rb+++ b/db/schema.rb@@ -10,7 +10,7 @@## It's strongly recommended that you check this file into your version control system.-ActiveRecord::Schema[7.2].define(version: 2025_02_20_200735) do+ActiveRecord::Schema[7.2].define(version: 2025_03_03_141007) do# These are extensions that must be enabled in order to support this databaseenable_extension "pgcrypto"enable_extension "plpgsql"@@ -398,7 +398,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_20_200735) dot.datetime "updated_at", null: falset.string "col_sep", default: ","t.uuid "family_id", null: false- t.uuid "original_account_id"+ t.uuid "account_id"t.string "type", null: falset.string "date_col_label", default: "date"t.string "amount_col_label", default: "amount"diff --git a/test/controllers/import/rows_controller_test.rb b/test/controllers/import/rows_controller_test.rbindex d16fd143..e6394d94 100644--- a/test/controllers/import/rows_controller_test.rb+++ b/test/controllers/import/rows_controller_test.rb@@ -22,7 +22,7 @@ class Import::RowsControllerTest < ActionDispatch::IntegrationTestget import_row_path(import, row)- assert_row_fields(row, [ :date, :ticker, :qty, :price, :currency, :account, :name ])+ assert_row_fields(row, [ :date, :ticker, :qty, :price, :currency, :account, :name, :account ])assert_response :successend