The personal finance app for everyone
Refactor: Allow other import files (#1099)
Pedro Carmona0c1ff00c1e0f
diff --git a/app/controllers/imports_controller.rb b/app/controllers/imports_controller.rbindex 4c2fdfc4..233a9bd8 100644--- a/app/controllers/imports_controller.rb+++ b/app/controllers/imports_controller.rb@@ -40,7 +40,7 @@ class ImportsController < ApplicationControllerdef upload_csvbegin- @import.raw_csv_str = import_params[:raw_csv_str].read+ @import.raw_file_str = import_params[:raw_file_str].readrescue NoMethodErrorflash.now[:alert] = "Please select a file to upload"render :load, status: :unprocessable_entity and return@@ -113,6 +113,6 @@ class ImportsController < ApplicationControllerenddef import_params(permitted_mappings = nil)- params.require(:import).permit(:raw_csv_str, column_mappings: permitted_mappings, csv_update: [ :row_idx, :col_idx, :value ])+ params.require(:import).permit(:raw_file_str, column_mappings: permitted_mappings, csv_update: [ :row_idx, :col_idx, :value ])endenddiff --git a/app/javascript/controllers/csv_upload_controller.js b/app/javascript/controllers/import_upload_controller.jssimilarity index 79%rename from app/javascript/controllers/csv_upload_controller.jsrename to app/javascript/controllers/import_upload_controller.jsindex af133e61..a7301c22 100644--- a/app/javascript/controllers/csv_upload_controller.js+++ b/app/javascript/controllers/import_upload_controller.js@@ -2,6 +2,11 @@ import { Controller } from "@hotwired/stimulus"export default class extends Controller {static targets = ["input", "preview", "submit", "filename", "filesize"]+ static values = {+ acceptedTypes: Array, // ["text/csv", "application/csv", ".csv"]+ acceptedExtension: String, // "csv"+ unacceptableTypeLabel: String, // "Only CSV files are allowed."+ };connect() {this.submitTarget.disabled = true@@ -30,15 +35,19 @@ export default class extends Controller {event.currentTarget.classList.remove("bg-gray-100")const file = event.dataTransfer.files[0]- if (file && this._isCSVFile(file)) {+ if (file && this._formatAcceptable(file)) {this._setFileInput(file);this._fileAdded(file)} else {this.previewTarget.classList.add("text-red-500")- this.previewTarget.textContent = "Only CSV files are allowed."+ this.previewTarget.textContent = this.unacceptableTypeLabelValue}}+ click() {+ this.inputTarget.click();+ }+// Private_fetchFileSize(size) {@@ -57,7 +66,7 @@ export default class extends Controller {if (file) {if (file.size > fileSizeLimit) {this.previewTarget.classList.add("text-red-500")- this.previewTarget.textContent = "File size exceeds the limit of 5MB"+ this.previewTarget.textContent = this.unacceptableTypeLabelValuereturn}@@ -80,10 +89,9 @@ export default class extends Controller {}}- _isCSVFile(file) {- const acceptedTypes = ["text/csv", "application/csv", ".csv"]+ _formatAcceptable(file) {const extension = file.name.split('.').pop().toLowerCase()- return acceptedTypes.includes(file.type) || extension === "csv"+ return this.acceptedTypesValue.includes(file.type) || extension === this.acceptedExtensionValue}_setFileInput(file) {diff --git a/app/models/import.rb b/app/models/import.rbindex 72bf883b..590e94f4 100644--- a/app/models/import.rb+++ b/app/models/import.rb@@ -1,7 +1,7 @@class Import < ApplicationRecordbelongs_to :account- validate :raw_csv_must_be_parsable+ validate :raw_file_must_be_parsablevalidates :col_sep, inclusion: { in: Csv::COL_SEP_LIST }before_save :initialize_csv, if: :should_initialize_csv?@@ -19,7 +19,7 @@ class Import < ApplicationRecordenddef loaded?- raw_csv_str.present?+ raw_file_str.present?enddef configured?@@ -88,16 +88,16 @@ class Import < ApplicationRecordenddef get_raw_csv- return nil if raw_csv_str.nil?- Import::Csv.new(raw_csv_str, col_sep:)+ return nil if raw_file_str.nil?+ Import::Csv.new(raw_file_str, col_sep:)enddef should_initialize_csv?- raw_csv_str_changed? || column_mappings_changed?+ raw_file_str_changed? || column_mappings_changed?enddef initialize_csv- generated_csv = generate_normalized_csv(raw_csv_str)+ generated_csv = generate_normalized_csv(raw_file_str)self.normalized_csv_str = generated_csv.table.to_send@@ -175,12 +175,12 @@ class Import < ApplicationRecordendend- def raw_csv_must_be_parsable+ def raw_file_must_be_parsablebegin- CSV.parse(raw_csv_str || "", col_sep:)+ CSV.parse(raw_file_str || "", col_sep:)rescue CSV::MalformedCSVError- # i18n-tasks-use t('activerecord.errors.models.import.attributes.raw_csv_str.invalid_csv_format')- errors.add(:raw_csv_str, :invalid_csv_format)+ # i18n-tasks-use t('activerecord.errors.models.import.attributes.raw_file_str.invalid_csv_format')+ errors.add(:raw_file_str, :invalid_csv_format)endendenddiff --git a/app/models/import/csv.rb b/app/models/import/csv.rbindex b19fe641..8fe593e4 100644--- a/app/models/import/csv.rb+++ b/app/models/import/csv.rb@@ -11,8 +11,8 @@ class Import::Csv)end- def self.create_with_field_mappings(raw_csv_str, fields, field_mappings, col_sep = DEFAULT_COL_SEP)- raw_csv = self.parse_csv(raw_csv_str, col_sep:)+ def self.create_with_field_mappings(raw_file_str, fields, field_mappings, col_sep = DEFAULT_COL_SEP)+ raw_csv = self.parse_csv(raw_file_str, col_sep:)generated_csv_str = CSV.generate headers: fields.map { |f| f.key }, write_headers: true, col_sep: do |csv|raw_csv.each do |row|diff --git a/app/views/imports/_csv_paste.html.erb b/app/views/imports/_csv_paste.html.erbindex 59257173..6203cc11 100644--- a/app/views/imports/_csv_paste.html.erb+++ b/app/views/imports/_csv_paste.html.erb@@ -1,11 +1,11 @@<%= styled_form_with model: @import, url: load_import_path(@import), class: "space-y-4" do |form| %>- <%= form.text_area :raw_csv_str,+ <%= form.text_area :raw_file_str,rows: 10,required: true,placeholder: "Paste your CSV file contents here",class: "rounded-md w-full border text-sm border-alpha-black-100 bg-white placeholder:text-gray-400" %>- <%= form.submit t(".next"), class: "px-4 py-2 mb-4 block w-full rounded-lg bg-gray-900 text-white text-sm font-medium", data: { turbo_confirm: (@import.raw_csv_str? ? { title: t(".confirm_title"), body: t(".confirm_body"), accept: t(".confirm_accept") } : nil) } %>+ <%= form.submit t(".next"), class: "px-4 py-2 mb-4 block w-full rounded-lg bg-gray-900 text-white text-sm font-medium", data: { turbo_confirm: (@import.raw_file_str? ? { title: t(".confirm_title"), body: t(".confirm_body"), accept: t(".confirm_accept") } : nil) } %><% end %><div class="bg-alpha-black-25 rounded-xl p-1 mt-5">diff --git a/app/views/imports/_csv_upload.html.erb b/app/views/imports/_csv_upload.html.erbindex ec23db31..b3c31f43 100644--- a/app/views/imports/_csv_upload.html.erb+++ b/app/views/imports/_csv_upload.html.erb@@ -1,24 +1,24 @@-<%= styled_form_with model: @import, url: upload_import_path(@import), class: "dropzone space-y-4", data: { controller: "csv-upload" }, method: :patch, multipart: true do |form| %>+<%= styled_form_with model: @import, url: upload_import_path(@import), class: "dropzone space-y-4", data: { controller: "import-upload", import_upload_accepted_types_value: ["text/csv", "application/csv", ".csv"], import_upload_extension_value: "csv", import_upload_unacceptable_type_label_value: t(".allowed_filetypes") }, method: :patch, multipart: true do |form| %><div class="flex items-center justify-center w-full">- <label for="import_raw_csv_str" class="csv-drop-box flex flex-col items-center justify-center w-full h-64 border-2 border-gray-300 border-dashed rounded-lg cursor-pointer bg-gray-50" data-action="dragover->csv-upload#dragover dragleave->csv-upload#dragleave drop->csv-upload#drop">+ <label for="import_raw_file_str" class="raw-file-drop-box flex flex-col items-center justify-center w-full h-64 border-2 border-gray-300 border-dashed rounded-lg cursor-pointer bg-gray-50" data-action="dragover->import-upload#dragover dragleave->import-upload#dragleave drop->import-upload#drop click->import-upload#click"><div class="flex flex-col items-center justify-center pt-5 pb-6"><%= lucide_icon "plus", class: "w-5 h-5 text-gray-500" %>- <%= form.file_field :raw_csv_str, class: "hidden", direct_upload: false, accept: "text/csv,.csv,application/csv", data: { csv_upload_target: "input", action: "change->csv-upload#addFile" } %>+ <%= form.file_field :raw_file_str, class: "hidden", direct_upload: false, accept: "text/csv,.csv,application/csv", data: { import_upload_target: "input", action: "change->import-upload#addFile" } %><p class="mb-2 text-sm text-gray-500 mt-3">Drag and drop your csv file here or <span class="text-black">click to browse</span></p><p class="text-xs text-gray-500">CSV (Max. 5MB)</p>- <div class="csv-preview" data-csv-upload-target="preview"></div>+ <div class="csv-preview" data-import-upload-target="preview"></div></div></label></div>- <%= form.submit t(".next"), class: "px-4 py-2 mb-4 block w-full rounded-lg bg-alpha-black-25 text-gray text-sm font-medium", data: { csv_upload_target: "submit", turbo_confirm: (@import.raw_csv_str? ? { title: t(".confirm_title"), body: t(".confirm_body"), accept: t(".confirm_accept") } : nil) } %>+ <%= form.submit t(".next"), class: "px-4 py-2 mb-4 block w-full rounded-lg bg-alpha-black-25 text-gray text-sm font-medium", data: { import_upload_target: "submit", turbo_confirm: (@import.raw_file_str? ? { title: t(".confirm_title"), body: t(".confirm_body"), accept: t(".confirm_accept") } : nil) } %><% end %><div id="template-preview" class="hidden"><div class="flex flex-col items-center justify-center"><%= lucide_icon "file-text", class: "w-10 h-10 pt-2 text-black" %><div class="flex flex-row items-center justify-center gap-0.5">- <div><span data-csv-upload-target="filename"></span></div>- <div><span data-csv-upload-target="filesize" class="font-semibold"></span></div>+ <div><span data-import-upload-target="filename"></span></div>+ <div><span data-import-upload-target="filesize" class="font-semibold"></span></div></div></div></div>diff --git a/config/locales/models/import/en.yml b/config/locales/models/import/en.ymlindex 56cdec7d..0019291c 100644--- a/config/locales/models/import/en.yml+++ b/config/locales/models/import/en.yml@@ -5,5 +5,5 @@ en:models:import:attributes:- raw_csv_str:+ raw_file_str:invalid_csv_format: is not a valid CSV formatdiff --git a/config/locales/views/imports/en.yml b/config/locales/views/imports/en.ymlindex d6166831..37b11f4f 100644--- a/config/locales/views/imports/en.yml+++ b/config/locales/views/imports/en.yml@@ -40,6 +40,7 @@ en:"inflow" (income)requirement3: Can have 0 or more tags separated by |csv_upload:+ allowed_filetypes: Only CSV files are allowed.confirm_accept: Yep, start over!confirm_body: This will reset your import. Any changes you have made to theCSV will be erased.diff --git a/db/migrate/20240817144454_rename_import_raw_csv_str_to_raw_file_str.rb b/db/migrate/20240817144454_rename_import_raw_csv_str_to_raw_file_str.rbnew file mode 100644index 00000000..ff5279c4--- /dev/null+++ b/db/migrate/20240817144454_rename_import_raw_csv_str_to_raw_file_str.rb@@ -0,0 +1,5 @@+class RenameImportRawCsvStrToRawFileStr < ActiveRecord::Migration[7.2]+ def change+ rename_column :imports, :raw_csv_str, :raw_file_str+ end+enddiff --git a/db/schema.rb b/db/schema.rbindex 8bedff06..3a423928 100644--- a/db/schema.rb+++ b/db/schema.rb@@ -10,7 +10,7 @@## It's strongly recommended that you check this file into your version control system.-ActiveRecord::Schema[7.2].define(version: 2024_08_16_071555) do+ActiveRecord::Schema[7.2].define(version: 2024_08_17_144454) do# These are extensions that must be enabled in order to support this databaseenable_extension "pgcrypto"enable_extension "plpgsql"@@ -289,7 +289,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_08_16_071555) dot.uuid "account_id", null: falset.jsonb "column_mappings"t.enum "status", default: "pending", enum_type: "import_status"- t.string "raw_csv_str"+ t.string "raw_file_str"t.string "normalized_csv_str"t.datetime "created_at", null: falset.datetime "updated_at", null: falsediff --git a/test/controllers/imports_controller_test.rb b/test/controllers/imports_controller_test.rbindex c5e87d4e..1cbb862f 100644--- a/test/controllers/imports_controller_test.rb+++ b/test/controllers/imports_controller_test.rb@@ -8,7 +8,7 @@ class ImportsControllerTest < ActionDispatch::IntegrationTest@empty_import = imports(:empty_import)@loaded_import = @empty_import.dup- @loaded_import.update! raw_csv_str: valid_csv_str+ @loaded_import.update! raw_file_str: valid_csv_str@completed_import = imports(:completed_import)end@@ -59,7 +59,7 @@ class ImportsControllerTest < ActionDispatch::IntegrationTestendtest "should save raw CSV if valid" do- patch load_import_url(@empty_import), params: { import: { raw_csv_str: valid_csv_str } }+ patch load_import_url(@empty_import), params: { import: { raw_file_str: valid_csv_str } }assert_redirected_to configure_import_path(@empty_import)assert_equal "Import CSV loaded", flash[:notice]@@ -71,17 +71,17 @@ class ImportsControllerTest < ActionDispatch::IntegrationTestvalid_csv_str.split("\n").each { |row| csv << row.split(",") }end- patch upload_import_url(@empty_import), params: { import: { raw_csv_str: Rack::Test::UploadedFile.new(temp, ".csv") } }+ patch upload_import_url(@empty_import), params: { import: { raw_file_str: Rack::Test::UploadedFile.new(temp, ".csv") } }assert_redirected_to configure_import_path(@empty_import)assert_equal "CSV File loaded", flash[:notice]endendtest "should flash error message if invalid CSV input" do- patch load_import_url(@empty_import), params: { import: { raw_csv_str: malformed_csv_str } }+ patch load_import_url(@empty_import), params: { import: { raw_file_str: malformed_csv_str } }assert_response :unprocessable_entity- assert_equal "Raw csv str is not a valid CSV format", flash[:alert]+ assert_equal "Raw file str is not a valid CSV format", flash[:alert]endtest "should flash error message if invalid CSV file upload" do@@ -89,14 +89,14 @@ class ImportsControllerTest < ActionDispatch::IntegrationTesttemp.write(malformed_csv_str)temp.rewind- patch upload_import_url(@empty_import), params: { import: { raw_csv_str: Rack::Test::UploadedFile.new(temp, ".csv") } }+ patch upload_import_url(@empty_import), params: { import: { raw_file_str: Rack::Test::UploadedFile.new(temp, ".csv") } }assert_response :unprocessable_entity- assert_equal "Raw csv str is not a valid CSV format", flash[:alert]+ assert_equal "Raw file str is not a valid CSV format", flash[:alert]endendtest "should flash error message if no fileprovided for upload" do- patch upload_import_url(@empty_import), params: { import: { raw_csv_str: nil } }+ patch upload_import_url(@empty_import), params: { import: { raw_file_str: nil } }assert_response :unprocessable_entityassert_equal "Please select a file to upload", flash[:alert]end@@ -158,7 +158,7 @@ class ImportsControllerTest < ActionDispatch::IntegrationTestendtest "should redirect back to clean if data is invalid" do- @empty_import.update! raw_csv_str: valid_csv_with_invalid_values+ @empty_import.update! raw_file_str: valid_csv_with_invalid_valuesget confirm_import_url(@empty_import)assert_equal "You have invalid data, please fix before continuing", flash[:alert]diff --git a/test/fixtures/imports.yml b/test/fixtures/imports.ymlindex 4ab16a08..474472a4 100644--- a/test/fixtures/imports.yml+++ b/test/fixtures/imports.yml@@ -9,7 +9,7 @@ completed_import:name: namecategory: categoryamount: amount- raw_csv_str: |+ raw_file_str: |date,name,category,tags,amount2024-01-01,Starbucks drink,Food & Drink,Test Tag,-20normalized_csv_str: |diff --git a/test/jobs/import_job_test.rb b/test/jobs/import_job_test.rbindex 35a0f3b5..1b59994c 100644--- a/test/jobs/import_job_test.rb+++ b/test/jobs/import_job_test.rb@@ -5,7 +5,7 @@ class ImportJobTest < ActiveJob::TestCasetest "import is published" doimport = imports(:empty_import)- import.update! raw_csv_str: valid_csv_str+ import.update! raw_file_str: valid_csv_strassert import.pending?diff --git a/test/models/import/csv_test.rb b/test/models/import/csv_test.rbindex ee7f0f0a..83978bee 100644--- a/test/models/import/csv_test.rb+++ b/test/models/import/csv_test.rb@@ -72,7 +72,7 @@ class Import::CsvTest < ActiveSupport::TestCasefields = [ date_field, name_field ]- raw_csv_str = <<-ROWS+ raw_file_str = <<-ROWSdate,Custom Field Header,extra_fieldinvalid_date_value,Starbucks drink,Food2024-01-02,Amazon stuff,Shopping@@ -82,7 +82,7 @@ class Import::CsvTest < ActiveSupport::TestCase"name" => "Custom Field Header"}- csv = Import::Csv.create_with_field_mappings(raw_csv_str, fields, mappings)+ csv = Import::Csv.create_with_field_mappings(raw_file_str, fields, mappings)assert_equal %w[ date name ], csv.table.headersassert_equal 2, csv.table.size@@ -101,7 +101,7 @@ class Import::CsvTest < ActiveSupport::TestCasefields = [ date_field, name_field ]- raw_csv_str = <<-ROWS+ raw_file_str = <<-ROWSdate;Custom Field Header;extra_fieldinvalid_date_value;Starbucks drink;Food2024-01-02;Amazon stuff;Shopping@@ -111,7 +111,7 @@ class Import::CsvTest < ActiveSupport::TestCase"name" => "Custom Field Header"}- csv = Import::Csv.create_with_field_mappings(raw_csv_str, fields, mappings, ";")+ csv = Import::Csv.create_with_field_mappings(raw_file_str, fields, mappings, ";")assert_equal %w[ date name ], csv.table.headersassert_equal 2, csv.table.sizediff --git a/test/models/import_test.rb b/test/models/import_test.rbindex 255c6e38..ac754935 100644--- a/test/models/import_test.rb+++ b/test/models/import_test.rb@@ -7,7 +7,7 @@ class ImportTest < ActiveSupport::TestCase@empty_import = imports(:empty_import)@loaded_import = @empty_import.dup- @loaded_import.update! raw_csv_str: valid_csv_str+ @loaded_import.update! raw_file_str: valid_csv_strendtest "validates the correct col_sep" do@@ -26,17 +26,17 @@ class ImportTest < ActiveSupport::TestCaseendtest "raw csv input must conform to csv spec" do- @empty_import.raw_csv_str = malformed_csv_str+ @empty_import.raw_file_str = malformed_csv_strassert_not @empty_import.valid?- @empty_import.raw_csv_str = valid_csv_str+ @empty_import.raw_file_str = valid_csv_strassert @empty_import.valid?endtest "can update csv value without affecting raw input" doassert_equal "Starbucks drink", @loaded_import.csv.table[0][1]- prior_raw_csv_str_value = @loaded_import.raw_csv_str+ prior_raw_file_str_value = @loaded_import.raw_file_strprior_normalized_csv_str_value = @loaded_import.normalized_csv_str@loaded_import.update_csv! \@@ -45,7 +45,7 @@ class ImportTest < ActiveSupport::TestCasevalue: "new_category"assert_equal "new_category", @loaded_import.csv.table[0][1]- assert_equal prior_raw_csv_str_value, @loaded_import.raw_csv_str+ assert_equal prior_raw_file_str_value, @loaded_import.raw_file_strassert_not_equal prior_normalized_csv_str_value, @loaded_import.normalized_csv_strend@@ -74,7 +74,7 @@ class ImportTest < ActiveSupport::TestCaseendtest "publishes a valid import with missing data" do- @empty_import.update! raw_csv_str: valid_csv_with_missing_data+ @empty_import.update! raw_file_str: valid_csv_with_missing_dataassert_difference -> { Category.count } => 1,-> { Account::Transaction.count } => 2,-> { Account::Entry.count } => 2 do@@ -89,7 +89,7 @@ class ImportTest < ActiveSupport::TestCaseendtest "failed publish results in error status" do- @empty_import.update! raw_csv_str: valid_csv_with_invalid_values+ @empty_import.update! raw_file_str: valid_csv_with_invalid_valuesassert_difference "Account::Transaction.count", 0 do@empty_import.publish@@ -102,7 +102,7 @@ class ImportTest < ActiveSupport::TestCasetest "can create transactions from csv with custom column separator" doloaded_import = @empty_import.dup- loaded_import.update! raw_csv_str: valid_csv_str_with_semicolon_separator, col_sep: ";"+ loaded_import.update! raw_file_str: valid_csv_str_with_semicolon_separator, col_sep: ";"transactions = loaded_import.dry_runassert_equal 4, transactions.countdiff --git a/test/system/imports_test.rb b/test/system/imports_test.rbindex 0d1bc82e..e8ba2788 100644--- a/test/system/imports_test.rb+++ b/test/system/imports_test.rb@@ -57,7 +57,7 @@ class ImportsTest < ApplicationSystemTestCaseassert_selector "h1", text: "Load import"within "form" do- fill_in "import_raw_csv_str", with: <<-ROWS+ fill_in "import_raw_file_str", with: <<-ROWSdate,Custom Name Column,category,amountinvalid_date,Starbucks drink,Food,-20.502024-01-01,Amazon purchase,Shopping,-89.50@@ -115,7 +115,7 @@ class ImportsTest < ApplicationSystemTestCaseclick_button "Upload CSV"- find(".csv-drop-box").drop File.join(file_fixture_path, "transactions.csv")+ find(".raw-file-drop-box").drop File.join(file_fixture_path, "transactions.csv")assert_selector "div.csv-preview", text: "transactions.csv"click_button "Next"