The personal finance app for everyone
Add support for different column separator in csv import logic (#1096)
Alexander Schrot4527482aa216
diff --git a/app/controllers/imports_controller.rb b/app/controllers/imports_controller.rbindex 90086684..4c2fdfc4 100644--- a/app/controllers/imports_controller.rb+++ b/app/controllers/imports_controller.rb@@ -18,14 +18,14 @@ class ImportsController < ApplicationControllerdef updateaccount = Current.family.accounts.find(params[:import][:account_id])+ @import.update! account: account, col_sep: params[:import][:col_sep]- @import.update! account: accountredirect_to load_import_path(@import), notice: t(".import_updated")enddef createaccount = Current.family.accounts.find(params[:import][:account_id])- @import = Import.create!(account: account)+ @import = Import.create! account: account, col_sep: params[:import][:col_sep]redirect_to load_import_path(@import), notice: t(".import_created")enddiff --git a/app/models/import.rb b/app/models/import.rbindex 2083113f..72bf883b 100644--- a/app/models/import.rb+++ b/app/models/import.rb@@ -2,6 +2,7 @@ class Import < ApplicationRecordbelongs_to :accountvalidate :raw_csv_must_be_parsable+ validates :col_sep, inclusion: { in: Csv::COL_SEP_LIST }before_save :initialize_csv, if: :should_initialize_csv?@@ -88,7 +89,7 @@ class Import < ApplicationRecorddef get_raw_csvreturn nil if raw_csv_str.nil?- Import::Csv.new(raw_csv_str)+ Import::Csv.new(raw_csv_str, col_sep:)enddef should_initialize_csv?@@ -102,7 +103,7 @@ class Import < ApplicationRecord# Uses the user-provided raw CSV + mappings to generate a normalized CSV for the importdef generate_normalized_csv(csv_str)- Import::Csv.create_with_field_mappings(csv_str, expected_fields, column_mappings)+ Import::Csv.create_with_field_mappings(csv_str, expected_fields, column_mappings, col_sep)enddef update_csv(row_idx, col_idx, value)@@ -176,7 +177,7 @@ class Import < ApplicationRecorddef raw_csv_must_be_parsablebegin- CSV.parse(raw_csv_str || "")+ CSV.parse(raw_csv_str || "", col_sep:)rescue CSV::MalformedCSVError# i18n-tasks-use t('activerecord.errors.models.import.attributes.raw_csv_str.invalid_csv_format')errors.add(:raw_csv_str, :invalid_csv_format)diff --git a/app/models/import/csv.rb b/app/models/import/csv.rbindex 873180de..b19fe641 100644--- a/app/models/import/csv.rb+++ b/app/models/import/csv.rb@@ -1,12 +1,20 @@class Import::Csv- def self.parse_csv(csv_str)- CSV.parse((csv_str || "").strip, headers: true, converters: [ ->(str) { str&.strip } ])+ DEFAULT_COL_SEP = ",".freeze+ COL_SEP_LIST = [ DEFAULT_COL_SEP, ";" ].freeze++ def self.parse_csv(csv_str, col_sep: DEFAULT_COL_SEP)+ CSV.parse(+ csv_str&.strip || "",+ headers: true,+ col_sep:,+ converters: [ ->(str) { str&.strip } ]+ )end- def self.create_with_field_mappings(raw_csv_str, fields, field_mappings)- raw_csv = self.parse_csv(raw_csv_str)+ def self.create_with_field_mappings(raw_csv_str, fields, field_mappings, col_sep = DEFAULT_COL_SEP)+ raw_csv = self.parse_csv(raw_csv_str, col_sep:)- generated_csv_str = CSV.generate headers: fields.map { |f| f.key }, write_headers: true do |csv|+ generated_csv_str = CSV.generate headers: fields.map { |f| f.key }, write_headers: true, col_sep: do |csv|raw_csv.each do |row|row_values = []@@ -22,18 +30,19 @@ class Import::Csvendend- new(generated_csv_str)+ new(generated_csv_str, col_sep:)end- attr_reader :csv_str+ attr_reader :csv_str, :col_sep- def initialize(csv_str, column_validators: nil)+ def initialize(csv_str, column_validators: nil, col_sep: DEFAULT_COL_SEP)@csv_str = csv_str+ @col_sep = col_sep@column_validators = column_validators || {}enddef table- @table ||= self.class.parse_csv(csv_str)+ @table ||= self.class.parse_csv(csv_str, col_sep:)enddef update_cell(row_idx, col_idx, value)diff --git a/app/views/imports/_form.html.erb b/app/views/imports/_form.html.erbindex f4178d6c..e7503016 100644--- a/app/views/imports/_form.html.erb+++ b/app/views/imports/_form.html.erb@@ -1,6 +1,7 @@<%= styled_form_with model: @import do |form| %>- <div class="mb-4">+ <div class="mb-4 space-y-3"><%= form.collection_select :account_id, Current.family.accounts.alphabetically, :id, :name, { prompt: t(".select_account"), label: t(".account"), required: true } %>+ <%= form.collection_select :col_sep, Import::Csv::COL_SEP_LIST, :to_s, -> { t(".col_sep_char.#{_1.ord}") }, { prompt: t(".select_col_sep"), label: t(".col_sep"), required: true } %></div><%= form.submit t(".next"), class: "px-4 py-2 block w-full rounded-lg bg-gray-900 text-white text-sm font-medium cursor-pointer hover:bg-gray-700" %>diff --git a/config/locales/views/imports/en.yml b/config/locales/views/imports/en.ymlindex c76e9721..d6166831 100644--- a/config/locales/views/imports/en.yml+++ b/config/locales/views/imports/en.yml@@ -58,8 +58,13 @@ en:new: New Importform:account: Account+ col_sep: CSV column separator+ col_sep_char:+ '44': Comma (,)+ '59': Semicolon (;)next: Nextselect_account: Select account+ select_col_sep: Select CSV column separatorimport:complete: Completecompleted_on: Completed on %{datetime}diff --git a/db/migrate/20240816071555_add_col_sep_to_imports.rb b/db/migrate/20240816071555_add_col_sep_to_imports.rbnew file mode 100644index 00000000..01d0f1d4--- /dev/null+++ b/db/migrate/20240816071555_add_col_sep_to_imports.rb@@ -0,0 +1,5 @@+class AddColSepToImports < ActiveRecord::Migration[7.2]+ def change+ add_column :imports, :col_sep, :string, default: ','+ end+enddiff --git a/db/schema.rb b/db/schema.rbindex 8c88d122..8bedff06 100644--- a/db/schema.rb+++ b/db/schema.rb@@ -10,7 +10,7 @@## It's strongly recommended that you check this file into your version control system.-ActiveRecord::Schema[7.2].define(version: 2024_08_15_190722) do+ActiveRecord::Schema[7.2].define(version: 2024_08_16_071555) do# These are extensions that must be enabled in order to support this databaseenable_extension "pgcrypto"enable_extension "plpgsql"@@ -118,7 +118,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_08_15_190722) dot.boolean "is_active", default: true, null: falset.date "last_sync_date"t.uuid "institution_id"- t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY ((ARRAY['Loan'::character varying, 'CreditCard'::character varying, 'OtherLiability'::character varying])::text[])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: true+ t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY (ARRAY[('Loan'::character varying)::text, ('CreditCard'::character varying)::text, ('OtherLiability'::character varying)::text])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: truet.index ["accountable_type"], name: "index_accounts_on_accountable_type"t.index ["family_id"], name: "index_accounts_on_family_id"t.index ["institution_id"], name: "index_accounts_on_institution_id"@@ -293,6 +293,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_08_15_190722) dot.string "normalized_csv_str"t.datetime "created_at", null: falset.datetime "updated_at", null: false+ t.string "col_sep", default: ","t.index ["account_id"], name: "index_imports_on_account_id"enddiff --git a/test/controllers/imports_controller_test.rb b/test/controllers/imports_controller_test.rbindex b1798c77..c5e87d4e 100644--- a/test/controllers/imports_controller_test.rb+++ b/test/controllers/imports_controller_test.rb@@ -29,7 +29,7 @@ class ImportsControllerTest < ActionDispatch::IntegrationTesttest "should create import" doassert_difference("Import.count") do- post imports_url, params: { import: { account_id: @user.family.accounts.first.id } }+ post imports_url, params: { import: { account_id: @user.family.accounts.first.id, col_sep: "," } }endassert_redirected_to load_import_path(Import.ordered.first)@@ -41,7 +41,7 @@ class ImportsControllerTest < ActionDispatch::IntegrationTestendtest "should update import" do- patch import_url(@empty_import), params: { import: { account_id: @empty_import.account_id } }+ patch import_url(@empty_import), params: { import: { account_id: @empty_import.account_id, col_sep: "," } }assert_redirected_to load_import_path(@empty_import)enddiff --git a/test/models/import/csv_test.rb b/test/models/import/csv_test.rbindex 5bbd0086..ee7f0f0a 100644--- a/test/models/import/csv_test.rb+++ b/test/models/import/csv_test.rb@@ -36,6 +36,14 @@ class Import::CsvTest < ActiveSupport::TestCaseassert_not invalid_csv.valid?end+ test "CSV with semicolon column separator" do+ csv = Import::Csv.new(valid_csv_str_with_semicolon_separator, col_sep: ";")++ assert_equal %w[ date name category tags amount ], csv.table.headers+ assert_equal 4, csv.table.size+ assert_equal "Paycheck", csv.table[3][1]+ end+test "csv with additional columns and empty values" docsv = Import::Csv.new valid_csv_with_missing_dataassert csv.valid?@@ -81,6 +89,35 @@ class Import::CsvTest < ActiveSupport::TestCaseassert_equal "Amazon stuff", csv.table[1][1]end+ test "can create CSV with expected columns, field mappings with validators and semicolon column separator" do+ date_field = Import::Field.new \+ key: "date",+ label: "Date",+ validator: method(:validate_iso_date)++ name_field = Import::Field.new \+ key: "name",+ label: "Name"++ fields = [ date_field, name_field ]++ raw_csv_str = <<-ROWS+ date;Custom Field Header;extra_field+ invalid_date_value;Starbucks drink;Food+ 2024-01-02;Amazon stuff;Shopping+ ROWS++ mappings = {+ "name" => "Custom Field Header"+ }++ csv = Import::Csv.create_with_field_mappings(raw_csv_str, fields, mappings, ";")++ assert_equal %w[ date name ], csv.table.headers+ assert_equal 2, csv.table.size+ assert_equal "Amazon stuff", csv.table[1][1]+ end+privatedef validate_iso_date(value)diff --git a/test/models/import_test.rb b/test/models/import_test.rbindex a26475af..255c6e38 100644--- a/test/models/import_test.rb+++ b/test/models/import_test.rb@@ -10,6 +10,21 @@ class ImportTest < ActiveSupport::TestCase@loaded_import.update! raw_csv_str: valid_csv_strend+ test "validates the correct col_sep" do+ assert_equal ",", @empty_import.col_sep++ assert @empty_import.valid?++ @empty_import.col_sep = "invalid"+ assert @empty_import.invalid?++ @empty_import.col_sep = ","+ assert @empty_import.valid?++ @empty_import.col_sep = ";"+ assert @empty_import.valid?+ end+test "raw csv input must conform to csv spec" do@empty_import.raw_csv_str = malformed_csv_strassert_not @empty_import.valid?@@ -83,4 +98,18 @@ class ImportTest < ActiveSupport::TestCase@empty_import.reloadassert @empty_import.failed?end++ test "can create transactions from csv with custom column separator" do+ loaded_import = @empty_import.dup++ loaded_import.update! raw_csv_str: valid_csv_str_with_semicolon_separator, col_sep: ";"+ transactions = loaded_import.dry_run++ assert_equal 4, transactions.count++ data = transactions.first.as_json(only: [ :name, :amount, :date ])+ assert_equal data, { "amount" => "8.55", "date" => "2024-01-01", "name" => "Starbucks drink" }++ assert_equal valid_csv_str, loaded_import.normalized_csv_str+ endenddiff --git a/test/support/import_test_helper.rb b/test/support/import_test_helper.rbindex 9d1e13f8..95a1adc8 100644--- a/test/support/import_test_helper.rb+++ b/test/support/import_test_helper.rb@@ -9,6 +9,16 @@ module ImportTestHelperROWSend+ def valid_csv_str_with_semicolon_separator+ <<~ROWS+ date;name;category;tags;amount+ 2024-01-01;Starbucks drink;Food & Drink;Tag1|Tag2;-8.55+ 2024-01-01;Etsy;Shopping;Tag1;-80.98+ 2024-01-02;Amazon stuff;Shopping;Tag2;-200+ 2024-01-03;Paycheck;Income;;1000+ ROWS+ end+def valid_csv_with_invalid_values<<~ROWSdate,name,category,tags,amount