The personal finance app for everyone
Allow CSV file upload in import flow (#986)
Tony Vincentcdbca5aff3b8
diff --git a/.gitignore b/.gitignoreindex c65d3262..19cd8f63 100644--- a/.gitignore+++ b/.gitignore@@ -51,6 +51,9 @@# Ignore .devcontainer filescompose-dev.yaml+# Ignore asdf ruby version file+.tool-versions+# Ignore GCP keyfilegcp-storage-keyfile.jsondiff --git a/app/controllers/imports_controller.rb b/app/controllers/imports_controller.rbindex 814ebf13..0857e278 100644--- a/app/controllers/imports_controller.rb+++ b/app/controllers/imports_controller.rb@@ -38,6 +38,21 @@ class ImportsController < ApplicationControllerdef loadend+ def upload_csv+ begin+ @import.raw_csv_str = import_params[:raw_csv_str].read+ rescue NoMethodError+ flash.now[:error] = "Please select a file to upload"+ render :load, status: :unprocessable_entity and return+ end+ if @import.save+ redirect_to configure_import_path(@import), notice: t(".import_loaded")+ else+ flash.now[:error] = @import.errors.full_messages.to_sentence+ render :load, status: :unprocessable_entity+ end+ end+def load_csvif @import.update(import_params)redirect_to configure_import_path(@import), notice: t(".import_loaded")diff --git a/app/javascript/controllers/csv_upload_controller.js b/app/javascript/controllers/csv_upload_controller.jsnew file mode 100644index 00000000..af133e61--- /dev/null+++ b/app/javascript/controllers/csv_upload_controller.js@@ -0,0 +1,94 @@+import { Controller } from "@hotwired/stimulus"++export default class extends Controller {+ static targets = ["input", "preview", "submit", "filename", "filesize"]++ connect() {+ this.submitTarget.disabled = true+ }++ addFile(event) {+ const file = event.target.files[0]+ this._fileAdded(file)+ }++ dragover(event) {+ event.preventDefault()+ event.stopPropagation()+ event.currentTarget.classList.add("bg-gray-100")+ }++ dragleave(event) {+ event.preventDefault()+ event.stopPropagation()+ event.currentTarget.classList.remove("bg-gray-100")+ }++ drop(event) {+ event.preventDefault()+ event.stopPropagation()+ event.currentTarget.classList.remove("bg-gray-100")++ const file = event.dataTransfer.files[0]+ if (file && this._isCSVFile(file)) {+ this._setFileInput(file);+ this._fileAdded(file)+ } else {+ this.previewTarget.classList.add("text-red-500")+ this.previewTarget.textContent = "Only CSV files are allowed."+ }+ }++ // Private++ _fetchFileSize(size) {+ let fileSize = '';+ if (size < 1024 * 1024) {+ fileSize = (size / 1024).toFixed(2) + ' KB'; // Convert bytes to KB+ } else {+ fileSize = (size / (1024 * 1024)).toFixed(2) + ' MB'; // Convert bytes to MB+ }+ return fileSize;+ }++ _fileAdded(file) {+ const fileSizeLimit = 5 * 1024 * 1024 // 5MB++ if (file) {+ if (file.size > fileSizeLimit) {+ this.previewTarget.classList.add("text-red-500")+ this.previewTarget.textContent = "File size exceeds the limit of 5MB"+ return+ }++ this.submitTarget.classList.remove([+ "bg-alpha-black-25",+ "text-gray",+ "cursor-not-allowed",+ ]);+ this.submitTarget.classList.add(+ "bg-gray-900",+ "text-white",+ "cursor-pointer",+ );+ this.submitTarget.disabled = false;+ this.previewTarget.innerHTML = document.querySelector("#template-preview").innerHTML;+ this.previewTarget.classList.remove("text-red-500")+ this.previewTarget.classList.add("text-gray-900")+ this.filenameTarget.textContent = file.name;+ this.filesizeTarget.textContent = this._fetchFileSize(file.size);+ }+ }++ _isCSVFile(file) {+ const acceptedTypes = ["text/csv", "application/csv", ".csv"]+ const extension = file.name.split('.').pop().toLowerCase()+ return acceptedTypes.includes(file.type) || extension === "csv"+ }++ _setFileInput(file) {+ const dataTransfer = new DataTransfer();+ dataTransfer.items.add(file);+ this.inputTarget.files = dataTransfer.files;+ }+}diff --git a/app/views/imports/_csv_paste.html.erb b/app/views/imports/_csv_paste.html.erbnew file mode 100644index 00000000..6006e874--- /dev/null+++ b/app/views/imports/_csv_paste.html.erb@@ -0,0 +1,27 @@+<%= form_with model: @import, url: load_import_path(@import) do |form| %>+ <div>+ <%= form.text_area :raw_csv_str,+ rows: 10,+ required: true,+ placeholder: "Paste your CSV file contents here",+ class: "rounded-md w-full border text-sm border-alpha-black-100 bg-white placeholder:text-gray-400 p-4" %>+ </div>++ <%= form.submit t(".next"), class: "px-4 py-2 mb-4 block w-full rounded-lg bg-gray-900 text-white text-sm font-medium", data: { turbo_confirm: (@import.raw_csv_str? ? { title: t(".confirm_title"), body: t(".confirm_body"), accept: t(".confirm_accept") } : nil) } %>+<% end %>++<div class="bg-alpha-black-25 rounded-xl p-1 mt-5">+ <div class="text-gray-500 p-2 mb-2">+ <div class="flex gap-2 mb-2">+ <%= lucide_icon("info", class: "w-5 h-5 shrink-0") %>+ <p class="text-sm"><%= t(".instructions") %></p>+ </div>++ <ul class="list-disc text-sm pl-10">+ <li><%= t(".requirement1") %></li>+ <li><%= t(".requirement2") %></li>+ <li><%= t(".requirement3") %></li>+ </ul>+ </div>+ <%= render partial: "imports/sample_table" %>+</div>diff --git a/app/views/imports/_csv_upload.html.erb b/app/views/imports/_csv_upload.html.erbnew file mode 100644index 00000000..69d598ef--- /dev/null+++ b/app/views/imports/_csv_upload.html.erb@@ -0,0 +1,39 @@+<%= form_with model: @import, url: upload_import_path(@import), class: "dropzone", data: { controller: "csv-upload" }, method: :patch, multipart: true do |form| %>+ <div class="flex items-center justify-center w-full">+ <label for="import_raw_csv_str" class="csv-drop-box flex flex-col items-center justify-center w-full h-64 border-2 border-gray-300 border-dashed rounded-lg cursor-pointer bg-gray-50" data-action="dragover->csv-upload#dragover dragleave->csv-upload#dragleave drop->csv-upload#drop">+ <div class="flex flex-col items-center justify-center pt-5 pb-6">+ <%= lucide_icon "plus", class: "w-5 h-5 text-gray-500" %>+ <%= form.file_field :raw_csv_str, class: "hidden", direct_upload: false, accept: "text/csv,.csv,application/csv", data: { csv_upload_target: "input", action: "change->csv-upload#addFile" } %>+ <p class="mb-2 text-sm text-gray-500 mt-3">Drag and drop your csv file here or <span class="text-black">click to browse</span></p>+ <p class="text-xs text-gray-500">CSV (Max. 5MB)</p>+ <div class="csv-preview" data-csv-upload-target="preview"></div>+ </div>+ </label>+ </div>+ <%= form.submit t(".next"), class: "px-4 py-2 mb-4 block w-full rounded-lg bg-alpha-black-25 text-gray text-sm font-medium", data: { csv_upload_target: "submit", turbo_confirm: (@import.raw_csv_str? ? { title: t(".confirm_title"), body: t(".confirm_body"), accept: t(".confirm_accept") } : nil) } %>+<% end %>++<div id="template-preview" class="hidden">+ <div class="flex flex-col items-center justify-center">+ <%= lucide_icon "file-text", class: "w-10 h-10 pt-2 text-black" %>+ <div class="flex flex-row items-center justify-center gap-0.5">+ <div><span data-csv-upload-target="filename"></span></div>+ <div><span data-csv-upload-target="filesize" class="font-semibold"></span></div>+ </div>+ </div>+</div>++<div class="bg-alpha-black-25 rounded-xl p-1 mt-5">+ <div class="text-gray-500 p-2 mb-2">+ <div class="flex gap-2 mb-2">+ <%= lucide_icon("info", class: "w-5 h-5 shrink-0") %>+ <p class="text-sm">+ <%= t(".instructions") %>+ <span class="text-black underline">+ <%= link_to "download this template", "/transactions.csv", download: "" %>+ </span>+ </p>+ </div>+ </div>+ <%= render partial: "imports/sample_table" %>+</div>diff --git a/app/views/imports/load.html.erb b/app/views/imports/load.html.erbindex 8bf7cb82..77f11559 100644--- a/app/views/imports/load.html.erb+++ b/app/views/imports/load.html.erb@@ -8,33 +8,18 @@<p class="text-gray-500 text-sm"><%= t(".description") %></p></div>- <%= form_with model: @import, url: load_import_path(@import) do |form| %>- <div>- <%= form.text_area :raw_csv_str,- rows: 10,- required: true,- placeholder: "Paste your CSV file contents here",- class: "rounded-md w-full border text-sm border-alpha-black-100 bg-white placeholder:text-gray-400 p-4" %>- </div>-- <%= form.submit t(".next"), class: "px-4 py-2 block w-full rounded-lg bg-gray-900 text-white text-sm font-medium cursor-pointer hover:bg-gray-700", data: { turbo_confirm: (@import.raw_csv_str? ? { title: t(".confirm_title"), body: t(".confirm_body"), accept: t(".confirm_accept") } : nil) } %>- <% end %>-- <div class="bg-alpha-black-25 rounded-xl p-1">- <div class="text-gray-500 p-2 mb-2">- <div class="flex gap-2 mb-2">- <%= lucide_icon("info", class: "w-5 h-5 shrink-0") %>- <p class="text-sm"><%= t(".instructions") %></p>+ <div data-controller="tabs" data-tabs-active-class="bg-white" data-tabs-default-tab-value="csv-upload-tab">+ <div class="flex justify-center mb-4">+ <div class="bg-gray-50 rounded-lg inline-flex p-1 space-x-2 text-sm text-gray-900 font-medium">+ <button data-id="csv-upload-tab" class="p-2 rounded-lg" data-tabs-target="btn" data-action="click->tabs#select">Upload CSV</button>+ <button data-id="csv-paste-tab" class="p-2 rounded-lg" data-tabs-target="btn" data-action="click->tabs#select">Copy & Paste</button></div>-- <ul class="list-disc text-sm pl-10">- <li><%= t(".requirement1") %></li>- <li><%= t(".requirement2") %></li>- <li><%= t(".requirement3") %></li>- </ul></div>-- <%= render partial: "imports/sample_table" %>-+ <div data-tabs-target="tab" id="csv-upload-tab">+ <%= render partial: "imports/csv_upload", locals: { import: @import } %>+ </div>+ <div data-tabs-target="tab" id="csv-paste-tab" class="hidden">+ <%= render partial: "imports/csv_paste", locals: { import: @import } %>+ </div></div></div>diff --git a/config/locales/views/imports/en.yml b/config/locales/views/imports/en.ymlindex 262e4623..c76e9721 100644--- a/config/locales/views/imports/en.yml+++ b/config/locales/views/imports/en.yml@@ -27,6 +27,25 @@ en:invalid_data: You have invalid data, please fix before continuingcreate:import_created: Import created+ csv_paste:+ confirm_accept: Yep, start over!+ confirm_body: This will reset your import. Any changes you have made to the+ CSV will be erased.+ confirm_title: Are you sure?+ instructions: Your CSV should have the following columns and formats for the+ best import experience.+ next: Next+ requirement1: Dates must be in ISO 8601 format (YYYY-MM-DD)+ requirement2: Negative transaction is an "outflow" (expense), positive is an+ "inflow" (income)+ requirement3: Can have 0 or more tags separated by |+ csv_upload:+ confirm_accept: Yep, start over!+ confirm_body: This will reset your import. Any changes you have made to the+ CSV will be erased.+ confirm_title: Are you sure?+ instructions: The csv file must be in the format below. You can also reuse and+ next: Nextdestroy:import_destroyed: Import destroyededit:@@ -56,20 +75,9 @@ en:new: New importtitle: Importsload:- confirm_accept: Yep, start over!- confirm_body: This will reset your import. Any changes you have made to the- CSV will be erased.- confirm_title: Are you sure?description: Create a spreadsheet or upload an exported CSV from your financialinstitution.- instructions: Your CSV should have the following columns and formats for the- best import experience.load_title: Load import- next: Next- requirement1: Dates must be in ISO 8601 format (YYYY-MM-DD)- requirement2: Negative transaction is an "outflow" (expense), positive is an- "inflow" (income)- requirement3: Can have 0 or more tags separated by |subtitle: Import your transactionsload_csv:import_loaded: Import CSV loaded@@ -95,3 +103,5 @@ en:import_updated: Import updatedupdate_mappings:column_mappings_saved: Column mappings saved+ upload_csv:+ import_loaded: CSV File loadeddiff --git a/config/routes.rb b/config/routes.rbindex eee50831..ea5d31d5 100644--- a/config/routes.rb+++ b/config/routes.rb@@ -25,6 +25,7 @@ Rails.application.routes.draw domember doget "load"patch "load" => "imports#load_csv"+ patch "upload" => "imports#upload_csv"get "configure"patch "configure" => "imports#update_mappings"diff --git a/public/transactions.csv b/public/transactions.csvnew file mode 100644index 00000000..a338951b--- /dev/null+++ b/public/transactions.csv@@ -0,0 +1,4 @@+date,name,category,tags,amount+2024-01-01,Amazon,Shopping,Tag1|Tag2,-24.99+2024-03-01,Spotify,,,-16.32+2023-01-06,Acme,Income,Tag3,151.22diff --git a/test/controllers/imports_controller_test.rb b/test/controllers/imports_controller_test.rbindex bf1ce1ac..e7cdbc3c 100644--- a/test/controllers/imports_controller_test.rb+++ b/test/controllers/imports_controller_test.rb@@ -65,6 +65,18 @@ class ImportsControllerTest < ActionDispatch::IntegrationTestassert_equal "Import CSV loaded", flash[:notice]end+ test "should upload CSV file if valid" do+ Tempfile.open([ "transactions.csv", ".csv" ]) do |temp|+ CSV.open(temp, "wb", headers: true) do |csv|+ valid_csv_str.split("\n").each { |row| csv << row.split(",") }+ end++ patch upload_import_url(@empty_import), params: { import: { raw_csv_str: Rack::Test::UploadedFile.new(temp, ".csv") } }+ assert_redirected_to configure_import_path(@empty_import)+ assert_equal "CSV File loaded", flash[:notice]+ end+ end+test "should flash error message if invalid CSV input" dopatch load_import_url(@empty_import), params: { import: { raw_csv_str: malformed_csv_str } }@@ -72,6 +84,23 @@ class ImportsControllerTest < ActionDispatch::IntegrationTestassert_equal "Raw csv str is not a valid CSV format", flash[:error]end+ test "should flash error message if invalid CSV file upload" do+ Tempfile.open([ "transactions.csv", ".csv" ]) do |temp|+ temp.write(malformed_csv_str)+ temp.rewind++ patch upload_import_url(@empty_import), params: { import: { raw_csv_str: Rack::Test::UploadedFile.new(temp, ".csv") } }+ assert_response :unprocessable_entity+ assert_equal "Raw csv str is not a valid CSV format", flash[:error]+ end+ end++ test "should flash error message if no fileprovided for upload" do+ patch upload_import_url(@empty_import), params: { import: { raw_csv_str: nil } }+ assert_response :unprocessable_entity+ assert_equal "Please select a file to upload", flash[:error]+ end+test "should get configure" doget configure_import_url(@loaded_import)assert_response :successdiff --git a/test/fixtures/files/transactions.csv b/test/fixtures/files/transactions.csvnew file mode 100644index 00000000..f998bd8c--- /dev/null+++ b/test/fixtures/files/transactions.csv@@ -0,0 +1,3 @@+date,Custom Name Column,category,amount+invalid_date,Starbucks drink,Food,-20.50+2024-01-01,Amazon purchase,Shopping,-89.50diff --git a/test/support/import_test_helper.rb b/test/support/import_test_helper.rbindex 3235b832..9d1e13f8 100644--- a/test/support/import_test_helper.rb+++ b/test/support/import_test_helper.rb@@ -1,6 +1,6 @@module ImportTestHelperdef valid_csv_str- <<-ROWS+ <<~ROWSdate,name,category,tags,amount2024-01-01,Starbucks drink,Food & Drink,Tag1|Tag2,-8.552024-01-01,Etsy,Shopping,Tag1,-80.98@@ -10,14 +10,14 @@ module ImportTestHelperenddef valid_csv_with_invalid_values- <<-ROWS+ <<~ROWSdate,name,category,tags,amountinvalid_date,Starbucks drink,Food,,invalid_amountROWSenddef valid_csv_with_missing_data- <<-ROWS+ <<~ROWSdate,name,category,"optional id",amount2024-01-01,Drink,Food,1234,-2002024-01-02,,,,-100@@ -25,7 +25,7 @@ module ImportTestHelperenddef malformed_csv_str- <<-ROWS+ <<~ROWSname,age"John Doe,23"Jane Doe",25diff --git a/test/system/imports_test.rb b/test/system/imports_test.rbindex 98f9614e..0d1bc82e 100644--- a/test/system/imports_test.rb+++ b/test/system/imports_test.rb@@ -51,6 +51,8 @@ class ImportsTest < ApplicationSystemTestCaseclick_button "Next"+ click_button "Copy & Paste"+# 2) Load Stepassert_selector "h1", text: "Load import"@@ -94,6 +96,60 @@ class ImportsTest < ApplicationSystemTestCaseassert_selector "h1", text: "Imports"end+ test "can perform import by CSV upload" do+ trigger_import_from_settings+ verify_import_modal++ within "#modal" do+ click_link "New import from CSV"+ end++ # 1) Create import step+ assert_selector "h1", text: "New import"++ within "form" do+ select "Checking Account", from: "import_account_id"+ end++ click_button "Next"++ click_button "Upload CSV"++ find(".csv-drop-box").drop File.join(file_fixture_path, "transactions.csv")+ assert_selector "div.csv-preview", text: "transactions.csv"++ click_button "Next"++ # 3) Configure step+ assert_selector "h1", text: "Configure import"++ within "form" do+ select "Custom Name Column", from: "import_column_mappings_name"+ end++ click_button "Next"++ # 4) Clean step+ assert_selector "h1", text: "Clean import"++ # We have an invalid value, so user cannot click next yet+ assert_no_text "Next"++ # Replace invalid date with valid date+ fill_in "cell-0-0", with: "2024-01-02"++ # Trigger blur event so value saves+ find("body").click++ click_link "Next"++ # 5) Confirm step+ assert_selector "h1", text: "Confirm import"+ assert_selector "#new_account_entry", count: 2+ click_button "Import 2 transactions"+ assert_selector "h1", text: "Imports"+ end+privatedef trigger_import_from_settings