Skip to content

armgit / maybe

Public Imported from GitHub
0

The personal finance app for everyone

Allow CSV file upload in import flow (#986)

Tony Vincentcdbca5aff3b8
diff --git a/.gitignore b/.gitignore
index c65d3262..19cd8f63 100644
--- a/.gitignore
+++ b/.gitignore
@@ -51,6 +51,9 @@
# Ignore .devcontainer files
compose-dev.yaml
+# Ignore asdf ruby version file
+.tool-versions
+
# Ignore GCP keyfile
gcp-storage-keyfile.json
diff --git a/app/controllers/imports_controller.rb b/app/controllers/imports_controller.rb
index 814ebf13..0857e278 100644
--- a/app/controllers/imports_controller.rb
+++ b/app/controllers/imports_controller.rb
@@ -38,6 +38,21 @@ class ImportsController < ApplicationController
def load
end
+ def upload_csv
+ begin
+ @import.raw_csv_str = import_params[:raw_csv_str].read
+ rescue NoMethodError
+ flash.now[:error] = "Please select a file to upload"
+ render :load, status: :unprocessable_entity and return
+ end
+ if @import.save
+ redirect_to configure_import_path(@import), notice: t(".import_loaded")
+ else
+ flash.now[:error] = @import.errors.full_messages.to_sentence
+ render :load, status: :unprocessable_entity
+ end
+ end
+
def load_csv
if @import.update(import_params)
redirect_to configure_import_path(@import), notice: t(".import_loaded")
diff --git a/app/javascript/controllers/csv_upload_controller.js b/app/javascript/controllers/csv_upload_controller.js
new file mode 100644
index 00000000..af133e61
--- /dev/null
+++ b/app/javascript/controllers/csv_upload_controller.js
@@ -0,0 +1,94 @@
+import { Controller } from "@hotwired/stimulus"
+
+export default class extends Controller {
+ static targets = ["input", "preview", "submit", "filename", "filesize"]
+
+ connect() {
+ this.submitTarget.disabled = true
+ }
+
+ addFile(event) {
+ const file = event.target.files[0]
+ this._fileAdded(file)
+ }
+
+ dragover(event) {
+ event.preventDefault()
+ event.stopPropagation()
+ event.currentTarget.classList.add("bg-gray-100")
+ }
+
+ dragleave(event) {
+ event.preventDefault()
+ event.stopPropagation()
+ event.currentTarget.classList.remove("bg-gray-100")
+ }
+
+ drop(event) {
+ event.preventDefault()
+ event.stopPropagation()
+ event.currentTarget.classList.remove("bg-gray-100")
+
+ const file = event.dataTransfer.files[0]
+ if (file && this._isCSVFile(file)) {
+ this._setFileInput(file);
+ this._fileAdded(file)
+ } else {
+ this.previewTarget.classList.add("text-red-500")
+ this.previewTarget.textContent = "Only CSV files are allowed."
+ }
+ }
+
+ // Private
+
+ _fetchFileSize(size) {
+ let fileSize = '';
+ if (size < 1024 * 1024) {
+ fileSize = (size / 1024).toFixed(2) + ' KB'; // Convert bytes to KB
+ } else {
+ fileSize = (size / (1024 * 1024)).toFixed(2) + ' MB'; // Convert bytes to MB
+ }
+ return fileSize;
+ }
+
+ _fileAdded(file) {
+ const fileSizeLimit = 5 * 1024 * 1024 // 5MB
+
+ if (file) {
+ if (file.size > fileSizeLimit) {
+ this.previewTarget.classList.add("text-red-500")
+ this.previewTarget.textContent = "File size exceeds the limit of 5MB"
+ return
+ }
+
+ this.submitTarget.classList.remove([
+ "bg-alpha-black-25",
+ "text-gray",
+ "cursor-not-allowed",
+ ]);
+ this.submitTarget.classList.add(
+ "bg-gray-900",
+ "text-white",
+ "cursor-pointer",
+ );
+ this.submitTarget.disabled = false;
+ this.previewTarget.innerHTML = document.querySelector("#template-preview").innerHTML;
+ this.previewTarget.classList.remove("text-red-500")
+ this.previewTarget.classList.add("text-gray-900")
+ this.filenameTarget.textContent = file.name;
+ this.filesizeTarget.textContent = this._fetchFileSize(file.size);
+ }
+ }
+
+ _isCSVFile(file) {
+ const acceptedTypes = ["text/csv", "application/csv", ".csv"]
+ const extension = file.name.split('.').pop().toLowerCase()
+ return acceptedTypes.includes(file.type) || extension === "csv"
+ }
+
+ _setFileInput(file) {
+ const dataTransfer = new DataTransfer();
+ dataTransfer.items.add(file);
+ this.inputTarget.files = dataTransfer.files;
+ }
+}
diff --git a/app/views/imports/_csv_paste.html.erb b/app/views/imports/_csv_paste.html.erb
new file mode 100644
index 00000000..6006e874
--- /dev/null
+++ b/app/views/imports/_csv_paste.html.erb
@@ -0,0 +1,27 @@
+<%= form_with model: @import, url: load_import_path(@import) do |form| %>
+ <div>
+ <%= form.text_area :raw_csv_str,
+ rows: 10,
+ required: true,
+ placeholder: "Paste your CSV file contents here",
+ class: "rounded-md w-full border text-sm border-alpha-black-100 bg-white placeholder:text-gray-400 p-4" %>
+ </div>
+
+ <%= form.submit t(".next"), class: "px-4 py-2 mb-4 block w-full rounded-lg bg-gray-900 text-white text-sm font-medium", data: { turbo_confirm: (@import.raw_csv_str? ? { title: t(".confirm_title"), body: t(".confirm_body"), accept: t(".confirm_accept") } : nil) } %>
+<% end %>
+
+<div class="bg-alpha-black-25 rounded-xl p-1 mt-5">
+ <div class="text-gray-500 p-2 mb-2">
+ <div class="flex gap-2 mb-2">
+ <%= lucide_icon("info", class: "w-5 h-5 shrink-0") %>
+ <p class="text-sm"><%= t(".instructions") %></p>
+ </div>
+
+ <ul class="list-disc text-sm pl-10">
+ <li><%= t(".requirement1") %></li>
+ <li><%= t(".requirement2") %></li>
+ <li><%= t(".requirement3") %></li>
+ </ul>
+ </div>
+ <%= render partial: "imports/sample_table" %>
+</div>
diff --git a/app/views/imports/_csv_upload.html.erb b/app/views/imports/_csv_upload.html.erb
new file mode 100644
index 00000000..69d598ef
--- /dev/null
+++ b/app/views/imports/_csv_upload.html.erb
@@ -0,0 +1,39 @@
+<%= form_with model: @import, url: upload_import_path(@import), class: "dropzone", data: { controller: "csv-upload" }, method: :patch, multipart: true do |form| %>
+ <div class="flex items-center justify-center w-full">
+ <label for="import_raw_csv_str" class="csv-drop-box flex flex-col items-center justify-center w-full h-64 border-2 border-gray-300 border-dashed rounded-lg cursor-pointer bg-gray-50" data-action="dragover->csv-upload#dragover dragleave->csv-upload#dragleave drop->csv-upload#drop">
+ <div class="flex flex-col items-center justify-center pt-5 pb-6">
+ <%= lucide_icon "plus", class: "w-5 h-5 text-gray-500" %>
+ <%= form.file_field :raw_csv_str, class: "hidden", direct_upload: false, accept: "text/csv,.csv,application/csv", data: { csv_upload_target: "input", action: "change->csv-upload#addFile" } %>
+ <p class="mb-2 text-sm text-gray-500 mt-3">Drag and drop your csv file here or <span class="text-black">click to browse</span></p>
+ <p class="text-xs text-gray-500">CSV (Max. 5MB)</p>
+ <div class="csv-preview" data-csv-upload-target="preview"></div>
+ </div>
+ </label>
+ </div>
+ <%= form.submit t(".next"), class: "px-4 py-2 mb-4 block w-full rounded-lg bg-alpha-black-25 text-gray text-sm font-medium", data: { csv_upload_target: "submit", turbo_confirm: (@import.raw_csv_str? ? { title: t(".confirm_title"), body: t(".confirm_body"), accept: t(".confirm_accept") } : nil) } %>
+<% end %>
+
+<div id="template-preview" class="hidden">
+ <div class="flex flex-col items-center justify-center">
+ <%= lucide_icon "file-text", class: "w-10 h-10 pt-2 text-black" %>
+ <div class="flex flex-row items-center justify-center gap-0.5">
+ <div><span data-csv-upload-target="filename"></span></div>
+ <div><span data-csv-upload-target="filesize" class="font-semibold"></span></div>
+ </div>
+ </div>
+</div>
+
+<div class="bg-alpha-black-25 rounded-xl p-1 mt-5">
+ <div class="text-gray-500 p-2 mb-2">
+ <div class="flex gap-2 mb-2">
+ <%= lucide_icon("info", class: "w-5 h-5 shrink-0") %>
+ <p class="text-sm">
+ <%= t(".instructions") %>
+ <span class="text-black underline">
+ <%= link_to "download this template", "/transactions.csv", download: "" %>
+ </span>
+ </p>
+ </div>
+ </div>
+ <%= render partial: "imports/sample_table" %>
+</div>
diff --git a/app/views/imports/load.html.erb b/app/views/imports/load.html.erb
index 8bf7cb82..77f11559 100644
--- a/app/views/imports/load.html.erb
+++ b/app/views/imports/load.html.erb
@@ -8,33 +8,18 @@
<p class="text-gray-500 text-sm"><%= t(".description") %></p>
</div>
- <%= form_with model: @import, url: load_import_path(@import) do |form| %>
- <div>
- <%= form.text_area :raw_csv_str,
- rows: 10,
- required: true,
- placeholder: "Paste your CSV file contents here",
- class: "rounded-md w-full border text-sm border-alpha-black-100 bg-white placeholder:text-gray-400 p-4" %>
- </div>
-
- <%= form.submit t(".next"), class: "px-4 py-2 block w-full rounded-lg bg-gray-900 text-white text-sm font-medium cursor-pointer hover:bg-gray-700", data: { turbo_confirm: (@import.raw_csv_str? ? { title: t(".confirm_title"), body: t(".confirm_body"), accept: t(".confirm_accept") } : nil) } %>
- <% end %>
-
- <div class="bg-alpha-black-25 rounded-xl p-1">
- <div class="text-gray-500 p-2 mb-2">
- <div class="flex gap-2 mb-2">
- <%= lucide_icon("info", class: "w-5 h-5 shrink-0") %>
- <p class="text-sm"><%= t(".instructions") %></p>
+ <div data-controller="tabs" data-tabs-active-class="bg-white" data-tabs-default-tab-value="csv-upload-tab">
+ <div class="flex justify-center mb-4">
+ <div class="bg-gray-50 rounded-lg inline-flex p-1 space-x-2 text-sm text-gray-900 font-medium">
+ <button data-id="csv-upload-tab" class="p-2 rounded-lg" data-tabs-target="btn" data-action="click->tabs#select">Upload CSV</button>
+ <button data-id="csv-paste-tab" class="p-2 rounded-lg" data-tabs-target="btn" data-action="click->tabs#select">Copy & Paste</button>
</div>
-
- <ul class="list-disc text-sm pl-10">
- <li><%= t(".requirement1") %></li>
- <li><%= t(".requirement2") %></li>
- <li><%= t(".requirement3") %></li>
- </ul>
</div>
-
- <%= render partial: "imports/sample_table" %>
-
+ <div data-tabs-target="tab" id="csv-upload-tab">
+ <%= render partial: "imports/csv_upload", locals: { import: @import } %>
+ </div>
+ <div data-tabs-target="tab" id="csv-paste-tab" class="hidden">
+ <%= render partial: "imports/csv_paste", locals: { import: @import } %>
+ </div>
</div>
</div>
diff --git a/config/locales/views/imports/en.yml b/config/locales/views/imports/en.yml
index 262e4623..c76e9721 100644
--- a/config/locales/views/imports/en.yml
+++ b/config/locales/views/imports/en.yml
@@ -27,6 +27,25 @@ en:
invalid_data: You have invalid data, please fix before continuing
create:
import_created: Import created
+ csv_paste:
+ confirm_accept: Yep, start over!
+ confirm_body: This will reset your import. Any changes you have made to the
+ CSV will be erased.
+ confirm_title: Are you sure?
+ instructions: Your CSV should have the following columns and formats for the
+ best import experience.
+ next: Next
+ requirement1: Dates must be in ISO 8601 format (YYYY-MM-DD)
+ requirement2: Negative transaction is an "outflow" (expense), positive is an
+ "inflow" (income)
+ requirement3: Can have 0 or more tags separated by |
+ csv_upload:
+ confirm_accept: Yep, start over!
+ confirm_body: This will reset your import. Any changes you have made to the
+ CSV will be erased.
+ confirm_title: Are you sure?
+ instructions: The csv file must be in the format below. You can also reuse and
+ next: Next
destroy:
import_destroyed: Import destroyed
edit:
@@ -56,20 +75,9 @@ en:
new: New import
title: Imports
load:
- confirm_accept: Yep, start over!
- confirm_body: This will reset your import. Any changes you have made to the
- CSV will be erased.
- confirm_title: Are you sure?
description: Create a spreadsheet or upload an exported CSV from your financial
institution.
- instructions: Your CSV should have the following columns and formats for the
- best import experience.
load_title: Load import
- next: Next
- requirement1: Dates must be in ISO 8601 format (YYYY-MM-DD)
- requirement2: Negative transaction is an "outflow" (expense), positive is an
- "inflow" (income)
- requirement3: Can have 0 or more tags separated by |
subtitle: Import your transactions
load_csv:
import_loaded: Import CSV loaded
@@ -95,3 +103,5 @@ en:
import_updated: Import updated
update_mappings:
column_mappings_saved: Column mappings saved
+ upload_csv:
+ import_loaded: CSV File loaded
diff --git a/config/routes.rb b/config/routes.rb
index eee50831..ea5d31d5 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -25,6 +25,7 @@ Rails.application.routes.draw do
member do
get "load"
patch "load" => "imports#load_csv"
+ patch "upload" => "imports#upload_csv"
get "configure"
patch "configure" => "imports#update_mappings"
diff --git a/public/transactions.csv b/public/transactions.csv
new file mode 100644
index 00000000..a338951b
--- /dev/null
+++ b/public/transactions.csv
@@ -0,0 +1,4 @@
+date,name,category,tags,amount
+2024-01-01,Amazon,Shopping,Tag1|Tag2,-24.99
+2024-03-01,Spotify,,,-16.32
+2023-01-06,Acme,Income,Tag3,151.22
diff --git a/test/controllers/imports_controller_test.rb b/test/controllers/imports_controller_test.rb
index bf1ce1ac..e7cdbc3c 100644
--- a/test/controllers/imports_controller_test.rb
+++ b/test/controllers/imports_controller_test.rb
@@ -65,6 +65,18 @@ class ImportsControllerTest < ActionDispatch::IntegrationTest
assert_equal "Import CSV loaded", flash[:notice]
end
+ test "should upload CSV file if valid" do
+ Tempfile.open([ "transactions.csv", ".csv" ]) do |temp|
+ CSV.open(temp, "wb", headers: true) do |csv|
+ valid_csv_str.split("\n").each { |row| csv << row.split(",") }
+ end
+
+ patch upload_import_url(@empty_import), params: { import: { raw_csv_str: Rack::Test::UploadedFile.new(temp, ".csv") } }
+ assert_redirected_to configure_import_path(@empty_import)
+ assert_equal "CSV File loaded", flash[:notice]
+ end
+ end
+
test "should flash error message if invalid CSV input" do
patch load_import_url(@empty_import), params: { import: { raw_csv_str: malformed_csv_str } }
@@ -72,6 +84,23 @@ class ImportsControllerTest < ActionDispatch::IntegrationTest
assert_equal "Raw csv str is not a valid CSV format", flash[:error]
end
+ test "should flash error message if invalid CSV file upload" do
+ Tempfile.open([ "transactions.csv", ".csv" ]) do |temp|
+ temp.write(malformed_csv_str)
+ temp.rewind
+
+ patch upload_import_url(@empty_import), params: { import: { raw_csv_str: Rack::Test::UploadedFile.new(temp, ".csv") } }
+ assert_response :unprocessable_entity
+ assert_equal "Raw csv str is not a valid CSV format", flash[:error]
+ end
+ end
+
+ test "should flash error message if no fileprovided for upload" do
+ patch upload_import_url(@empty_import), params: { import: { raw_csv_str: nil } }
+ assert_response :unprocessable_entity
+ assert_equal "Please select a file to upload", flash[:error]
+ end
+
test "should get configure" do
get configure_import_url(@loaded_import)
assert_response :success
diff --git a/test/fixtures/files/transactions.csv b/test/fixtures/files/transactions.csv
new file mode 100644
index 00000000..f998bd8c
--- /dev/null
+++ b/test/fixtures/files/transactions.csv
@@ -0,0 +1,3 @@
+date,Custom Name Column,category,amount
+invalid_date,Starbucks drink,Food,-20.50
+2024-01-01,Amazon purchase,Shopping,-89.50
diff --git a/test/support/import_test_helper.rb b/test/support/import_test_helper.rb
index 3235b832..9d1e13f8 100644
--- a/test/support/import_test_helper.rb
+++ b/test/support/import_test_helper.rb
@@ -1,6 +1,6 @@
module ImportTestHelper
def valid_csv_str
- <<-ROWS
+ <<~ROWS
date,name,category,tags,amount
2024-01-01,Starbucks drink,Food & Drink,Tag1|Tag2,-8.55
2024-01-01,Etsy,Shopping,Tag1,-80.98
@@ -10,14 +10,14 @@ module ImportTestHelper
end
def valid_csv_with_invalid_values
- <<-ROWS
+ <<~ROWS
date,name,category,tags,amount
invalid_date,Starbucks drink,Food,,invalid_amount
ROWS
end
def valid_csv_with_missing_data
- <<-ROWS
+ <<~ROWS
date,name,category,"optional id",amount
2024-01-01,Drink,Food,1234,-200
2024-01-02,,,,-100
@@ -25,7 +25,7 @@ module ImportTestHelper
end
def malformed_csv_str
- <<-ROWS
+ <<~ROWS
name,age
"John Doe,23
"Jane Doe",25
diff --git a/test/system/imports_test.rb b/test/system/imports_test.rb
index 98f9614e..0d1bc82e 100644
--- a/test/system/imports_test.rb
+++ b/test/system/imports_test.rb
@@ -51,6 +51,8 @@ class ImportsTest < ApplicationSystemTestCase
click_button "Next"
+ click_button "Copy & Paste"
+
# 2) Load Step
assert_selector "h1", text: "Load import"
@@ -94,6 +96,60 @@ class ImportsTest < ApplicationSystemTestCase
assert_selector "h1", text: "Imports"
end
+ test "can perform import by CSV upload" do
+ trigger_import_from_settings
+ verify_import_modal
+
+ within "#modal" do
+ click_link "New import from CSV"
+ end
+
+ # 1) Create import step
+ assert_selector "h1", text: "New import"
+
+ within "form" do
+ select "Checking Account", from: "import_account_id"
+ end
+
+ click_button "Next"
+
+ click_button "Upload CSV"
+
+ find(".csv-drop-box").drop File.join(file_fixture_path, "transactions.csv")
+ assert_selector "div.csv-preview", text: "transactions.csv"
+
+ click_button "Next"
+
+ # 3) Configure step
+ assert_selector "h1", text: "Configure import"
+
+ within "form" do
+ select "Custom Name Column", from: "import_column_mappings_name"
+ end
+
+ click_button "Next"
+
+ # 4) Clean step
+ assert_selector "h1", text: "Clean import"
+
+ # We have an invalid value, so user cannot click next yet
+ assert_no_text "Next"
+
+ # Replace invalid date with valid date
+ fill_in "cell-0-0", with: "2024-01-02"
+
+ # Trigger blur event so value saves
+ find("body").click
+
+ click_link "Next"
+
+ # 5) Confirm step
+ assert_selector "h1", text: "Confirm import"
+ assert_selector "#new_account_entry", count: 2
+ click_button "Import 2 transactions"
+ assert_selector "h1", text: "Imports"
+ end
+
private
def trigger_import_from_settings