Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
CSV imports (UI) (#3845)
ruslandogac263df580533
diff --git a/assets/js/liveview/live_socket.js b/assets/js/liveview/live_socket.jsindex a6208ff4cf..ed287267f7 100644--- a/assets/js/liveview/live_socket.js+++ b/assets/js/liveview/live_socket.js@@ -19,12 +19,30 @@ if (csrfToken && websocketUrl) {})}}+ let Uploaders = {}+ Uploaders.S3 = function (entries, onViewError) {+ entries.forEach(entry => {+ let xhr = new XMLHttpRequest()+ onViewError(() => xhr.abort())+ xhr.onload = () => xhr.status === 200 ? entry.progress(100) : entry.error()+ xhr.onerror = () => entry.error()+ xhr.upload.addEventListener("progress", (event) => {+ if (event.lengthComputable) {+ let percent = Math.round((event.loaded / event.total) * 100)+ if (percent < 100) { entry.progress(percent) }+ }+ })+ let url = entry.meta.url+ xhr.open("PUT", url, true)+ xhr.send(entry.file)+ })+ }let token = csrfToken.getAttribute("content")let url = websocketUrl.getAttribute("content")let liveUrl = (url === "") ? "/live" : new URL("/live", url).href;let liveSocket = new LiveSocket(liveUrl, Socket, {heartbeatIntervalMs: 10000,- params: { _csrf_token: token }, hooks: Hooks, dom: {+ params: { _csrf_token: token }, hooks: Hooks, uploaders: Uploaders, dom: {// for alpinejs integrationonBeforeElUpdated(from, to) {if (from._x_dataStack) {diff --git a/lib/plausible/exports.ex b/lib/plausible/exports.exindex dbabcd7ff8..c15e0bb78d 100644--- a/lib/plausible/exports.ex+++ b/lib/plausible/exports.ex@@ -12,14 +12,24 @@ defmodule Plausible.Exports doExamples:iex> archive_filename("plausible.io", ~D[2021-01-01], ~D[2024-12-31])- "plausible.io_20210101_20241231.zip"+ "plausible_io_20210101_20241231.zip"iex> archive_filename("Bücher.example", ~D[2021-01-01], ~D[2024-12-31])- "Bücher.example_20210101_20241231.zip"+ "Bücher_example_20210101_20241231.zip""""def archive_filename(domain, min_date, max_date) do- "#{domain}_#{Calendar.strftime(min_date, "%Y%m%d")}_#{Calendar.strftime(max_date, "%Y%m%d")}.zip"+ name =+ Enum.join(+ [+ String.replace(domain, ".", "_"),+ Calendar.strftime(min_date, "%Y%m%d"),+ Calendar.strftime(max_date, "%Y%m%d")+ ],+ "_"+ )++ name <> ".zip"end@doc """diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.exindex 2d10ca32e9..6376d851d3 100644--- a/lib/plausible/imported/csv_importer.ex+++ b/lib/plausible/imported/csv_importer.ex@@ -105,12 +105,12 @@ defmodule Plausible.Imported.CSVImporter doDate.range(~D[2019-01-01], ~D[2022-01-01])iex> date_range([])- ** (ArgumentError) empty uploads+ nil"""- @spec date_range([String.t() | %{String.t() => String.t()}, ...]) :: Date.Range.t()+ @spec date_range([String.t() | %{String.t() => String.t()}, ...]) :: Date.Range.t() | nildef date_range([_ | _] = uploads), do: date_range(uploads, _start_date = nil, _end_date = nil)- def date_range([]), do: raise(ArgumentError, "empty uploads")+ def date_range([]), do: nildefp date_range([upload | uploads], prev_start_date, prev_end_date) dofilename =@@ -174,4 +174,47 @@ defmodule Plausible.Imported.CSVImporter dodef parse_filename!(_filename) doraise ArgumentError, "invalid filename"end++ @doc """+ Checks if the provided filename conforms to the expected format.++ Examples:++ iex> valid_filename?("my_data.csv")+ false++ iex> valid_filename?("imported_devices_00010101_20250101.csv")+ true++ """+ @spec valid_filename?(String.t()) :: boolean+ def valid_filename?(filename) do+ try do+ parse_filename!(filename)+ else+ _ -> true+ rescue+ _ -> false+ end+ end++ @doc """+ Extracts the table name from the provided filename.++ Raises if the filename doesn't conform to the expected format.++ Examples:++ iex> extract_table("my_data.csv")+ ** (ArgumentError) invalid filename++ iex> extract_table("imported_devices_00010101_20250101.csv")+ "imported_devices"++ """+ @spec extract_table(String.t()) :: String.t()+ def extract_table(filename) do+ {table, _start_date, _end_date} = parse_filename!(filename)+ table+ endenddiff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.exindex 64cc4332df..e61a266111 100644--- a/lib/plausible_web/controllers/site_controller.ex+++ b/lib/plausible_web/controllers/site_controller.ex@@ -708,7 +708,7 @@ defmodule PlausibleWeb.SiteController do|> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain))end- def export(conn, _params) do+ def csv_export(conn, _params) do%{site: site, current_user: user} = conn.assignsOban.insert!(@@ -725,6 +725,15 @@ defmodule PlausibleWeb.SiteController do|> redirect(to: Routes.site_path(conn, :settings_imports_exports, site.domain))end+ def csv_import(conn, _params) do+ conn+ |> assign(:skip_plausible_tracking, true)+ |> render("csv_import.html",+ layout: {PlausibleWeb.LayoutView, "focus.html"},+ connect_live_socket: true+ )+ end+def change_domain(conn, _params) dochangeset = Plausible.Site.update_changeset(conn.assigns.site)diff --git a/lib/plausible_web/live/csv_import.ex b/lib/plausible_web/live/csv_import.exnew file mode 100644index 0000000000..38c97ca17f--- /dev/null+++ b/lib/plausible_web/live/csv_import.ex@@ -0,0 +1,228 @@+defmodule PlausibleWeb.Live.CSVImport do+ @moduledoc """+ LiveView allowing uploading CSVs for imported tables to S3+ """+ use PlausibleWeb, :live_view+ alias Plausible.Imported.CSVImporter++ @impl true+ def mount(_params, session, socket) do+ %{"site_id" => site_id, "user_id" => user_id} = session++ socket =+ socket+ |> assign(site_id: site_id, user_id: user_id)+ |> allow_upload(:import,+ accept: [".csv", "text/csv"],+ auto_upload: true,+ max_entries: length(Plausible.Imported.tables()),+ # 1GB+ max_file_size: 1_000_000_000,+ external: &presign_upload/2,+ progress: &handle_progress/3+ )+ |> process_imported_tables()++ {:ok, socket}+ end++ @impl true+ def render(assigns) do+ ~H"""+ <div>+ <form action="#" method="post" phx-change="validate-upload-form" phx-submit="submit-upload-form">+ <.csv_picker upload={@uploads.import} imported_tables={@imported_tables} />+ <.confirm_button date_range={@date_range} can_confirm?={@can_confirm?} />++ <p :for={error <- upload_errors(@uploads.import)} class="text-red-400">+ <%= error_to_string(error) %>+ </p>+ </form>+ </div>+ """+ end++ defp csv_picker(assigns) do+ ~H"""+ <label+ phx-drop-target={@upload.ref}+ class="block border-2 dark:border-gray-600 rounded p-4 group hover:border-indigo-500 dark:hover:border-indigo-600 transition cursor-pointer"+ >+ <div class="flex items-center">+ <div class="bg-gray-200 dark:bg-gray-600 rounded p-1 group-hover:bg-indigo-500 dark:group-hover:bg-indigo-600 transition">+ <Heroicons.document_plus class="w-5 h-5 group-hover:text-white transition" />+ </div>+ <span class="ml-2 text-sm text-gray-600 dark:text-gray-500">+ (or drag-and-drop your unzipped CSVs here)+ </span>+ <.live_file_input upload={@upload} class="hidden" />+ </div>++ <ul id="imported-tables" class="mt-3.5 mb-0.5 space-y-1.5">+ <.imported_table+ :for={{table, upload} <- @imported_tables}+ table={table}+ upload={upload}+ errors={if(upload, do: upload_errors(@upload, upload), else: [])}+ />+ </ul>+ </label>+ """+ end++ defp confirm_button(assigns) do+ ~H"""+ <button+ type="submit"+ disabled={not @can_confirm?}+ class={[+ "rounded-md w-full bg-indigo-600 px-3.5 py-2.5 text-sm font-semibold text-white shadow-sm hover:bg-indigo-700 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600 disabled:bg-gray-400 dark:disabled:text-gray-400 dark:disabled:bg-gray-700 mt-4",+ unless(@can_confirm?, do: "cursor-not-allowed")+ ]}+ >+ <%= if @date_range do %>+ Confirm import from <%= @date_range.first %> to <%= @date_range.last %>+ <% else %>+ Confirm import+ <% end %>+ </button>+ """+ end++ defp imported_table(assigns) do+ status =+ cond do+ assigns.upload && assigns.upload.progress == 100 -> :success+ assigns.upload && assigns.upload.progress > 0 -> :in_progress+ not Enum.empty?(assigns.errors) -> :error+ true -> :empty+ end++ assigns = assign(assigns, status: status)++ ~H"""+ <li id={@table} class="ml-1.5">+ <div class="flex items-center space-x-2">+ <Heroicons.document_check :if={@status == :success} class="w-4 h-4 text-indigo-600" />+ <PlausibleWeb.Components.Generic.spinner+ :if={@status == :in_progress}+ class="w-4 h-4 text-indigo-600"+ />+ <Heroicons.document :if={@status == :empty} class="w-4 h-4 text-gray-400 dark:text-gray-500" />+ <Heroicons.document :if={@status == :error} class="w-4 h-4 text-red-600 dark:text-red-700" />++ <span class={[+ "text-sm",+ if(@upload, do: "dark:text-gray-400", else: "text-gray-400 dark:text-gray-500"),+ if(@status == :error, do: "text-red-600 dark:text-red-700")+ ]}>+ <%= if @upload do %>+ <%= @upload.client_name %>+ <% else %>+ <%= @table %>_YYYYMMDD_YYYYMMDD.csv+ <% end %>+ </span>+ </div>++ <p :for={error <- @errors} class="ml-6 text-sm text-red-600 dark:text-red-700">+ <%= error_to_string(error) %>+ </p>+ </li>+ """+ end++ @impl true+ def handle_event("validate-upload-form", _params, socket) do+ {:noreply, process_imported_tables(socket)}+ end++ def handle_event("submit-upload-form", _params, socket) do+ %{site_id: site_id, user_id: user_id, date_range: date_range} = socket.assigns+ site = Plausible.Repo.get!(Plausible.Site, site_id)+ user = Plausible.Repo.get!(Plausible.Auth.User, user_id)++ uploads =+ consume_uploaded_entries(socket, :import, fn meta, entry ->+ {:ok, %{"s3_url" => meta.s3_url, "filename" => entry.client_name}}+ end)++ {:ok, _job} =+ CSVImporter.new_import(site, user,+ start_date: date_range.first,+ end_date: date_range.last,+ uploads: uploads+ )++ redirect_to =+ Routes.site_path(socket, :settings_imports_exports, URI.encode_www_form(site.domain))++ {:noreply, redirect(socket, external: redirect_to)}+ end++ defp error_to_string(:too_large), do: "is too large (max size is 1 gigabyte)"+ defp error_to_string(:too_many_files), do: "too many files"+ defp error_to_string(:not_accepted), do: "unacceptable file types"+ defp error_to_string(:external_client_failure), do: "browser upload failed"++ defp presign_upload(entry, socket) do+ %{s3_url: s3_url, presigned_url: upload_url} =+ Plausible.S3.import_presign_upload(socket.assigns.site_id, entry.client_name)++ {:ok, %{uploader: "S3", s3_url: s3_url, url: upload_url}, socket}+ end++ defp handle_progress(:import, entry, socket) do+ if entry.done? do+ {:noreply, process_imported_tables(socket)}+ else+ {:noreply, socket}+ end+ end++ defp process_imported_tables(socket) do+ tables = Plausible.Imported.tables()+ {completed, in_progress} = uploaded_entries(socket, :import)++ {valid_uploads, invalid_uploads} =+ Enum.split_with(completed ++ in_progress, &CSVImporter.valid_filename?(&1.client_name))++ imported_tables_all_uploads =+ Enum.map(tables, fn table ->+ uploads =+ Enum.filter(valid_uploads, fn upload ->+ CSVImporter.extract_table(upload.client_name) == table+ end)++ {upload, replaced_uploads} = List.pop_at(uploads, -1)+ {table, upload, replaced_uploads}+ end)++ imported_tables =+ Enum.map(imported_tables_all_uploads, fn {table, upload, _replaced_uploads} ->+ {table, upload}+ end)++ replaced_uploads =+ Enum.flat_map(imported_tables_all_uploads, fn {_table, _upload, replaced_uploads} ->+ replaced_uploads+ end)++ date_range = CSVImporter.date_range(Enum.map(valid_uploads, & &1.client_name))+ all_uploaded? = completed != [] and in_progress == []++ socket+ |> cancel_uploads(invalid_uploads)+ |> cancel_uploads(replaced_uploads)+ |> assign(+ imported_tables: imported_tables,+ can_confirm?: all_uploaded?,+ date_range: date_range+ )+ end++ defp cancel_uploads(socket, uploads) do+ Enum.reduce(uploads, socket, fn upload, socket ->+ cancel_upload(socket, :import, upload.ref)+ end)+ end+enddiff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.exindex 6a4fd7660c..914b549fba 100644--- a/lib/plausible_web/router.ex+++ b/lib/plausible_web/router.ex@@ -390,7 +390,8 @@ defmodule PlausibleWeb.Router dodelete "/:website/settings/forget-imported", SiteController, :forget_importeddelete "/:website/settings/forget-import/:import_id", SiteController, :forget_import- post "/:website/settings/export", SiteController, :export+ post "/:website/settings/export", SiteController, :csv_export+ get "/:website/settings/import", SiteController, :csv_importget "/:domain/export", StatsController, :csv_exportget "/:domain/*path", StatsController, :statsdiff --git a/lib/plausible_web/templates/site/csv_import.html.heex b/lib/plausible_web/templates/site/csv_import.html.heexnew file mode 100644index 0000000000..ae9fc4038d--- /dev/null+++ b/lib/plausible_web/templates/site/csv_import.html.heex@@ -0,0 +1,20 @@+<div class="max-w-lg w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded p-6 mb-4 mt-8">+ <h2 class="text-xl font-black dark:text-gray-100">Import from CSV files</h2>++ <div class="my-3 space-y-1.5 text-sm text-gray-400">+ <p>+ Please ensure each file follows+ <.styled_link href="https://plausible.io/docs/csv-import">+ our CSV format guidelines.+ </.styled_link>+ </p>++ <p>+ You can upload multiple files simultaneously by either selecting them in the file dialog or dragging and dropping them into the designated area.+ </p>+ </div>++ <%= live_render(@conn, PlausibleWeb.Live.CSVImport,+ session: %{"site_id" => @site.id, "user_id" => @current_user.id}+ ) %>+</div>diff --git a/lib/plausible_web/templates/site/settings_imports_exports.html.heex b/lib/plausible_web/templates/site/settings_imports_exports.html.heexindex afc374c32e..4011ac033c 100644--- a/lib/plausible_web/templates/site/settings_imports_exports.html.heex+++ b/lib/plausible_web/templates/site/settings_imports_exports.html.heex@@ -20,9 +20,9 @@</PlausibleWeb.Components.Generic.button_link><PlausibleWeb.Components.Generic.button_link- class="w-36 h-20 opacity-40 cursor-not-allowed"+ class="w-36 h-20"theme="bright"- href=""+ href={"/#{URI.encode_www_form(@site.domain)}/settings/import"}><img class="h-16" src="/images/icon/csv_logo.svg" alt="New CSV import" /></PlausibleWeb.Components.Generic.button_link>