The personal finance app for everyone
feat(import): add currency and number format support for CSV imports (#1819)
Daniel Esteves077694bbde44
diff --git a/app/models/import.rb b/app/models/import.rbindex 472b5e12..1fd85d02 100644--- a/app/models/import.rb+++ b/app/models/import.rb@@ -2,8 +2,17 @@ class Import < ApplicationRecordTYPES = %w[TransactionImport TradeImport AccountImport MintImport].freezeSIGNAGE_CONVENTIONS = %w[inflows_positive inflows_negative]+ NUMBER_FORMATS = {+ "1,234.56" => { separator: ".", delimiter: "," }, # US/UK/Asia+ "1.234,56" => { separator: ",", delimiter: "." }, # Most of Europe+ "1 234,56" => { separator: ",", delimiter: " " }, # French/Scandinavian+ "1,234" => { separator: "", delimiter: "," } # Zero-decimal currencies like JPY+ }.freeze+belongs_to :family+ before_validation :set_default_number_format+scope :ordered, -> { order(created_at: :desc) }enum :status, {@@ -18,6 +27,7 @@ class Import < ApplicationRecordvalidates :type, inclusion: { in: TYPES }validates :col_sep, inclusion: { in: [ ",", ";" ] }validates :signage_convention, inclusion: { in: SIGNAGE_CONVENTIONS }+ validates :number_format, presence: true, inclusion: { in: NUMBER_FORMATS.keys }has_many :rows, dependent: :destroyhas_many :mappings, dependent: :destroy@@ -95,8 +105,8 @@ class Import < ApplicationRecorddef generate_rows_from_csvrows.destroy_all- mapped_rows = csv_rows.map do |row|- {+ csv_rows.each do |row|+ rows.create!(account: row[account_col_label].to_s,date: row[date_col_label].to_s,qty: sanitize_number(row[qty_col_label]).to_s,@@ -109,10 +119,8 @@ class Import < ApplicationRecordtags: row[tags_col_label].to_s,entity_type: row[entity_type_col_label].to_s,notes: row[notes_col_label].to_s- }+ )end-- rows.insert_all!(mapped_rows)enddef sync_mappings@@ -177,6 +185,39 @@ class Import < ApplicationRecorddef sanitize_number(value)return "" if value.nil?- value.gsub(/[^\d.\-]/, "")++ format = NUMBER_FORMATS[number_format]+ return "" unless format++ # First, normalize spaces and remove any characters that aren't numbers, delimiters, separators, or minus signs+ sanitized = value.to_s.strip++ # Handle French/Scandinavian format specially+ if format[:delimiter] == " "+ sanitized = sanitized.gsub(/\s+/, "") # Remove all spaces first+ else+ sanitized = sanitized.gsub(/[^\d#{Regexp.escape(format[:delimiter])}#{Regexp.escape(format[:separator])}\-]/, "")++ # Replace delimiter with empty string+ if format[:delimiter].present?+ sanitized = sanitized.gsub(format[:delimiter], "")+ end+ end++ # Replace separator with period for proper float parsing+ if format[:separator].present?+ sanitized = sanitized.gsub(format[:separator], ".")+ end++ # Return empty string if not a valid number+ unless sanitized =~ /\A-?\d+\.?\d*\z/+ return ""+ end++ sanitized+ end++ def set_default_number_format+ self.number_format ||= "1,234.56" # Default to US/UK formatendenddiff --git a/app/views/import/configurations/_mint_import.html.erb b/app/views/import/configurations/_mint_import.html.erbindex d0eeced4..6e07168a 100644--- a/app/views/import/configurations/_mint_import.html.erb+++ b/app/views/import/configurations/_mint_import.html.erb@@ -16,7 +16,10 @@<%= form.select :signage_convention, [["Incomes are negative", "inflows_negative"], ["Incomes are positive", "inflows_positive"]], { label: true }, disabled: import.complete? %></div>- <%= form.select :currency_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Currency" }, disabled: import.complete? %>+ <div class="flex items-center gap-2">+ <%= form.select :currency_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Currency" }, disabled: import.complete? %>+ <%= form.select :number_format, Import::NUMBER_FORMATS.keys, { label: "Format", prompt: "Select format" }, required: true %>+ </div><%= form.select :account_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Account (optional)" }, disabled: import.complete? %><%= form.select :name_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Name (optional)" }, disabled: import.complete? %>diff --git a/app/views/import/configurations/_trade_import.html.erb b/app/views/import/configurations/_trade_import.html.erbindex b8dab627..e49d29c3 100644--- a/app/views/import/configurations/_trade_import.html.erb+++ b/app/views/import/configurations/_trade_import.html.erb@@ -11,7 +11,10 @@<%= form.select :signage_convention, [["Buys are positive qty", "inflows_positive"], ["Buys are negative qty", "inflows_negative"]], label: true %></div>- <%= form.select :currency_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Currency" } %>+ <div class="flex items-center gap-2">+ <%= form.select :currency_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Currency" } %>+ <%= form.select :number_format, Import::NUMBER_FORMATS.keys, { label: "Format", prompt: "Select format" }, required: true %>+ </div><%= form.select :ticker_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Ticker" } %><%= form.select :price_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Price" } %>diff --git a/app/views/import/configurations/_transaction_import.html.erb b/app/views/import/configurations/_transaction_import.html.erbindex 82abb1c5..65c9f5f4 100644--- a/app/views/import/configurations/_transaction_import.html.erb+++ b/app/views/import/configurations/_transaction_import.html.erb@@ -11,7 +11,10 @@<%= form.select :signage_convention, [["Incomes are positive", "inflows_positive"], ["Incomes are negative", "inflows_negative"]], label: true %></div>- <%= form.select :currency_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Currency" } %>+ <div class="flex items-center gap-2">+ <%= form.select :currency_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Currency" } %>+ <%= form.select :number_format, Import::NUMBER_FORMATS.keys, { label: "Format", prompt: "Select format" }, required: true %>+ </div><%= form.select :account_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Account (optional)" } %><%= form.select :name_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Name (optional)" } %>diff --git a/config/locales/models/import/en.yml b/config/locales/models/import/en.ymlindex 0019291c..713ad905 100644--- a/config/locales/models/import/en.yml+++ b/config/locales/models/import/en.yml@@ -7,3 +7,7 @@ en:attributes:raw_file_str:invalid_csv_format: is not a valid CSV format+ attributes:+ import:+ currency: Currency+ number_format: Number Formatdiff --git a/db/migrate/20250207014022_add_number_format_to_imports.rb b/db/migrate/20250207014022_add_number_format_to_imports.rbnew file mode 100644index 00000000..265fbc6b--- /dev/null+++ b/db/migrate/20250207014022_add_number_format_to_imports.rb@@ -0,0 +1,5 @@+class AddNumberFormatToImports < ActiveRecord::Migration[7.2]+ def change+ add_column :imports, :number_format, :string+ end+enddiff --git a/db/schema.rb b/db/schema.rbindex 5d6a8e12..f44329c6 100644--- a/db/schema.rb+++ b/db/schema.rb@@ -10,7 +10,7 @@## It's strongly recommended that you check this file into your version control system.-ActiveRecord::Schema[7.2].define(version: 2025_02_06_204404) do+ActiveRecord::Schema[7.2].define(version: 2025_02_07_014022) do# These are extensions that must be enabled in order to support this databaseenable_extension "pgcrypto"enable_extension "plpgsql"@@ -101,7 +101,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_06_204404) dot.decimal "balance", precision: 19, scale: 4t.string "currency"t.boolean "is_active", default: true, null: false- t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY ((ARRAY['Loan'::character varying, 'CreditCard'::character varying, 'OtherLiability'::character varying])::text[])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: true+ t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY (ARRAY[('Loan'::character varying)::text, ('CreditCard'::character varying)::text, ('OtherLiability'::character varying)::text])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: truet.uuid "import_id"t.uuid "plaid_account_id"t.boolean "scheduled_for_deletion", default: false@@ -414,6 +414,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_06_204404) dot.string "date_format", default: "%m/%d/%Y"t.string "signage_convention", default: "inflows_positive"t.string "error"+ t.string "number_format"t.index ["family_id"], name: "index_imports_on_family_id"enddiff --git a/test/interfaces/import_interface_test.rb b/test/interfaces/import_interface_test.rbindex 68b09306..9ee56b37 100644--- a/test/interfaces/import_interface_test.rb+++ b/test/interfaces/import_interface_test.rb@@ -54,4 +54,233 @@ module ImportInterfaceTestassert_equal "Failed to publish", import.errorassert_equal "failed", import.statusend++ test "parses US/UK number format correctly" do+ import = imports(:transaction)+ import.update!(+ number_format: "1,234.56",+ amount_col_label: "amount",+ date_col_label: "date",+ name_col_label: "name",+ date_format: "%m/%d/%Y"+ )++ csv_data = "date,amount,name\n01/01/2024,\"1,234.56\",Test"+ import.update!(raw_file_str: csv_data)+ import.generate_rows_from_csv++ row = import.rows.first+ assert_equal "1234.56", row.amount+ end++ test "parses European number format correctly" do+ import = imports(:transaction)+ import.update!(+ number_format: "1.234,56",+ amount_col_label: "amount",+ date_col_label: "date",+ name_col_label: "name",+ date_format: "%m/%d/%Y"+ )++ csv_data = "date,amount,name\n01/01/2024,\"1.234,56\",Test"+ import.update!(raw_file_str: csv_data)+ import.generate_rows_from_csv++ row = import.rows.first+ assert_equal "1234.56", row.amount+ end++ test "parses French/Scandinavian number format correctly" do+ import = imports(:transaction)+ import.update!(+ number_format: "1 234,56",+ amount_col_label: "amount",+ date_col_label: "date",+ name_col_label: "name",+ date_format: "%m/%d/%Y"+ )++ # Quote the amount field to ensure proper CSV parsing+ csv_data = "date,amount,name\n01/01/2024,\"1 234,56\",Test"+ import.update!(raw_file_str: csv_data)+ import.generate_rows_from_csv++ row = import.rows.first+ assert_equal "1234.56", row.amount+ end++ test "parses zero-decimal currency format correctly" do+ import = imports(:transaction)+ import.update!(+ number_format: "1,234",+ amount_col_label: "amount",+ date_col_label: "date",+ name_col_label: "name",+ date_format: "%m/%d/%Y"+ )++ csv_data = "date,amount,name\n01/01/2024,1234,Test"+ import.update!(raw_file_str: csv_data)+ import.generate_rows_from_csv++ row = import.rows.first+ assert_equal "1234", row.amount+ end++ test "currency from CSV takes precedence over default" do+ import = imports(:transaction)+ import.update!(+ amount_col_label: "amount",+ date_col_label: "date",+ name_col_label: "name",+ currency_col_label: "currency",+ number_format: "1,234.56",+ date_format: "%m/%d/%Y"+ )+ import.family.update!(currency: "USD")++ csv_data = "date,amount,name,currency\n01/01/2024,123.45,Test,EUR"+ import.update!(raw_file_str: csv_data)+ import.generate_rows_from_csv++ row = import.rows.first+ assert_equal "EUR", row.currency+ end++ test "uses default currency when CSV currency column is empty" do+ import = imports(:transaction)+ import.update!(+ amount_col_label: "amount",+ date_col_label: "date",+ name_col_label: "name",+ currency_col_label: "currency",+ number_format: "1,234.56",+ date_format: "%m/%d/%Y"+ )+ import.family.update!(currency: "USD")++ csv_data = "date,amount,name,currency\n01/01/2024,123.45,Test,"+ import.update!(raw_file_str: csv_data)+ import.generate_rows_from_csv++ row = import.rows.first+ assert_equal "USD", row.currency+ end++ test "uses default currency when CSV has no currency column" do+ import = imports(:transaction)+ import.update!(+ amount_col_label: "amount",+ date_col_label: "date",+ name_col_label: "name",+ number_format: "1,234.56",+ date_format: "%m/%d/%Y"+ )+ import.family.update!(currency: "USD")++ csv_data = "date,amount,name\n01/01/2024,123.45,Test"+ import.update!(raw_file_str: csv_data)+ import.generate_rows_from_csv++ row = import.rows.first+ assert_equal "USD", row.currency+ end++ test "generates rows with all optional fields" do+ import = imports(:transaction)+ import.update!(+ amount_col_label: "amount",+ date_col_label: "date",+ name_col_label: "name",+ account_col_label: "account",+ category_col_label: "category",+ tags_col_label: "tags",+ notes_col_label: "notes",+ currency_col_label: "currency",+ number_format: "1,234.56",+ date_format: "%m/%d/%Y"+ )++ csv_data = "date,amount,name,account,category,tags,notes,currency\n" \+ "01/01/2024,1234.56,Salary,Bank Account,Income,\"monthly,salary\",Salary payment,EUR"+ import.update!(raw_file_str: csv_data)+ import.generate_rows_from_csv++ row = import.rows.first+ assert_equal "01/01/2024", row.date+ assert_equal "1234.56", row.amount+ assert_equal "Salary", row.name+ assert_equal "Bank Account", row.account+ assert_equal "Income", row.category+ assert_equal "monthly,salary", row.tags+ assert_equal "Salary payment", row.notes+ assert_equal "EUR", row.currency+ end++ test "generates rows with minimal required fields" do+ import = imports(:transaction)+ import.update!(+ amount_col_label: "amount",+ date_col_label: "date",+ number_format: "1,234.56",+ date_format: "%m/%d/%Y"+ )++ csv_data = "date,amount\n01/01/2024,1234.56"+ import.update!(raw_file_str: csv_data)+ import.generate_rows_from_csv++ row = import.rows.first+ assert_equal "01/01/2024", row.date+ assert_equal "1234.56", row.amount+ assert_equal "Imported item", row.name # Default name+ assert_equal import.family.currency, row.currency # Default currency+ end++ test "handles empty values in optional fields" do+ import = imports(:transaction)+ import.update!(+ amount_col_label: "amount",+ date_col_label: "date",+ name_col_label: "name",+ category_col_label: "category",+ tags_col_label: "tags",+ number_format: "1,234.56",+ date_format: "%m/%d/%Y"+ )++ csv_data = "date,amount,name,category,tags\n01/01/2024,1234.56,,,"+ import.update!(raw_file_str: csv_data)+ import.generate_rows_from_csv++ row = import.rows.first+ assert_equal "01/01/2024", row.date+ assert_equal "1234.56", row.amount+ assert_equal "Imported item", row.name # Falls back to default+ assert_equal "", row.category+ assert_equal "", row.tags+ end++ test "handles trade-specific fields" do+ import = imports(:transaction)+ import.update!(+ amount_col_label: "amount",+ date_col_label: "date",+ qty_col_label: "quantity",+ ticker_col_label: "symbol",+ price_col_label: "price",+ number_format: "1,234.56",+ date_format: "%m/%d/%Y"+ )++ csv_data = "date,amount,quantity,symbol,price\n01/01/2024,1234.56,10,AAPL,123.456"+ import.update!(raw_file_str: csv_data)+ import.generate_rows_from_csv++ row = import.rows.first+ assert_equal "10", row.qty+ assert_equal "AAPL", row.ticker+ assert_equal "123.456", row.price+ endenddiff --git a/test/models/transaction_import_test.rb b/test/models/transaction_import_test.rbindex fc3e2ab9..2299194c 100644--- a/test/models/transaction_import_test.rb+++ b/test/models/transaction_import_test.rb@@ -28,14 +28,19 @@ class TransactionImportTest < ActiveSupport::TestCaseendtest "imports transactions, categories, tags, and accounts" do- import = <<-CSV+ import = <<~CSVdate,name,amount,category,tags,account,notes01/01/2024,Txn1,100,TestCategory1,TestTag1,TestAccount1,notes101/02/2024,Txn2,200,TestCategory2,TestTag1|TestTag2,TestAccount2,notes201/03/2024,Txn3,300,,,,notes3CSV- @import.update!(raw_file_str: import)+ @import.update!(+ raw_file_str: import,+ date_col_label: "date",+ amount_col_label: "amount",+ date_format: "%m/%d/%Y"+ )@import.generate_rows_from_csv