Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

CSV imports (UI) (#3845)

ruslandogac263df580533
diff --git a/assets/js/liveview/live_socket.js b/assets/js/liveview/live_socket.js
index a6208ff4cf..ed287267f7 100644
--- a/assets/js/liveview/live_socket.js
+++ b/assets/js/liveview/live_socket.js
@@ -19,12 +19,30 @@ if (csrfToken && websocketUrl) {
})
}
}
+ let Uploaders = {}
+ Uploaders.S3 = function (entries, onViewError) {
+ entries.forEach(entry => {
+ let xhr = new XMLHttpRequest()
+ onViewError(() => xhr.abort())
+ xhr.onload = () => xhr.status === 200 ? entry.progress(100) : entry.error()
+ xhr.onerror = () => entry.error()
+ xhr.upload.addEventListener("progress", (event) => {
+ if (event.lengthComputable) {
+ let percent = Math.round((event.loaded / event.total) * 100)
+ if (percent < 100) { entry.progress(percent) }
+ }
+ })
+ let url = entry.meta.url
+ xhr.open("PUT", url, true)
+ xhr.send(entry.file)
+ })
+ }
let token = csrfToken.getAttribute("content")
let url = websocketUrl.getAttribute("content")
let liveUrl = (url === "") ? "/live" : new URL("/live", url).href;
let liveSocket = new LiveSocket(liveUrl, Socket, {
heartbeatIntervalMs: 10000,
- params: { _csrf_token: token }, hooks: Hooks, dom: {
+ params: { _csrf_token: token }, hooks: Hooks, uploaders: Uploaders, dom: {
// for alpinejs integration
onBeforeElUpdated(from, to) {
if (from._x_dataStack) {
diff --git a/lib/plausible/exports.ex b/lib/plausible/exports.ex
index dbabcd7ff8..c15e0bb78d 100644
--- a/lib/plausible/exports.ex
+++ b/lib/plausible/exports.ex
@@ -12,14 +12,24 @@ defmodule Plausible.Exports do
Examples:
iex> archive_filename("plausible.io", ~D[2021-01-01], ~D[2024-12-31])
- "plausible.io_20210101_20241231.zip"
+ "plausible_io_20210101_20241231.zip"
iex> archive_filename("Bücher.example", ~D[2021-01-01], ~D[2024-12-31])
- "Bücher.example_20210101_20241231.zip"
+ "Bücher_example_20210101_20241231.zip"
"""
def archive_filename(domain, min_date, max_date) do
- "#{domain}_#{Calendar.strftime(min_date, "%Y%m%d")}_#{Calendar.strftime(max_date, "%Y%m%d")}.zip"
+ name =
+ Enum.join(
+ [
+ String.replace(domain, ".", "_"),
+ Calendar.strftime(min_date, "%Y%m%d"),
+ Calendar.strftime(max_date, "%Y%m%d")
+ ],
+ "_"
+ )
+
+ name <> ".zip"
end
@doc """
diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex
index 2d10ca32e9..6376d851d3 100644
--- a/lib/plausible/imported/csv_importer.ex
+++ b/lib/plausible/imported/csv_importer.ex
@@ -105,12 +105,12 @@ defmodule Plausible.Imported.CSVImporter do
Date.range(~D[2019-01-01], ~D[2022-01-01])
iex> date_range([])
- ** (ArgumentError) empty uploads
+ nil
"""
- @spec date_range([String.t() | %{String.t() => String.t()}, ...]) :: Date.Range.t()
+ @spec date_range([String.t() | %{String.t() => String.t()}, ...]) :: Date.Range.t() | nil
def date_range([_ | _] = uploads), do: date_range(uploads, _start_date = nil, _end_date = nil)
- def date_range([]), do: raise(ArgumentError, "empty uploads")
+ def date_range([]), do: nil
defp date_range([upload | uploads], prev_start_date, prev_end_date) do
filename =
@@ -174,4 +174,47 @@ defmodule Plausible.Imported.CSVImporter do
def parse_filename!(_filename) do
raise ArgumentError, "invalid filename"
end
+
+ @doc """
+ Checks if the provided filename conforms to the expected format.
+
+ Examples:
+
+ iex> valid_filename?("my_data.csv")
+ false
+
+ iex> valid_filename?("imported_devices_00010101_20250101.csv")
+ true
+
+ """
+ @spec valid_filename?(String.t()) :: boolean
+ def valid_filename?(filename) do
+ try do
+ parse_filename!(filename)
+ else
+ _ -> true
+ rescue
+ _ -> false
+ end
+ end
+
+ @doc """
+ Extracts the table name from the provided filename.
+
+ Raises if the filename doesn't conform to the expected format.
+
+ Examples:
+
+ iex> extract_table("my_data.csv")
+ ** (ArgumentError) invalid filename
+
+ iex> extract_table("imported_devices_00010101_20250101.csv")
+ "imported_devices"
+
+ """
+ @spec extract_table(String.t()) :: String.t()
+ def extract_table(filename) do
+ {table, _start_date, _end_date} = parse_filename!(filename)
+ table
+ end
end
diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex
index 64cc4332df..e61a266111 100644
--- a/lib/plausible_web/controllers/site_controller.ex
+++ b/lib/plausible_web/controllers/site_controller.ex
@@ -708,7 +708,7 @@ defmodule PlausibleWeb.SiteController do
|> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain))
end
- def export(conn, _params) do
+ def csv_export(conn, _params) do
%{site: site, current_user: user} = conn.assigns
Oban.insert!(
@@ -725,6 +725,15 @@ defmodule PlausibleWeb.SiteController do
|> redirect(to: Routes.site_path(conn, :settings_imports_exports, site.domain))
end
+ def csv_import(conn, _params) do
+ conn
+ |> assign(:skip_plausible_tracking, true)
+ |> render("csv_import.html",
+ layout: {PlausibleWeb.LayoutView, "focus.html"},
+ connect_live_socket: true
+ )
+ end
+
def change_domain(conn, _params) do
changeset = Plausible.Site.update_changeset(conn.assigns.site)
diff --git a/lib/plausible_web/live/csv_import.ex b/lib/plausible_web/live/csv_import.ex
new file mode 100644
index 0000000000..38c97ca17f
--- /dev/null
+++ b/lib/plausible_web/live/csv_import.ex
@@ -0,0 +1,228 @@
+defmodule PlausibleWeb.Live.CSVImport do
+ @moduledoc """
+ LiveView allowing uploading CSVs for imported tables to S3
+ """
+ use PlausibleWeb, :live_view
+ alias Plausible.Imported.CSVImporter
+
+ @impl true
+ def mount(_params, session, socket) do
+ %{"site_id" => site_id, "user_id" => user_id} = session
+
+ socket =
+ socket
+ |> assign(site_id: site_id, user_id: user_id)
+ |> allow_upload(:import,
+ accept: [".csv", "text/csv"],
+ auto_upload: true,
+ max_entries: length(Plausible.Imported.tables()),
+ # 1GB
+ max_file_size: 1_000_000_000,
+ external: &presign_upload/2,
+ progress: &handle_progress/3
+ )
+ |> process_imported_tables()
+
+ {:ok, socket}
+ end
+
+ @impl true
+ def render(assigns) do
+ ~H"""
+ <div>
+ <form action="#" method="post" phx-change="validate-upload-form" phx-submit="submit-upload-form">
+ <.csv_picker upload={@uploads.import} imported_tables={@imported_tables} />
+ <.confirm_button date_range={@date_range} can_confirm?={@can_confirm?} />
+
+ <p :for={error <- upload_errors(@uploads.import)} class="text-red-400">
+ <%= error_to_string(error) %>
+ </p>
+ </form>
+ </div>
+ """
+ end
+
+ defp csv_picker(assigns) do
+ ~H"""
+ <label
+ phx-drop-target={@upload.ref}
+ class="block border-2 dark:border-gray-600 rounded p-4 group hover:border-indigo-500 dark:hover:border-indigo-600 transition cursor-pointer"
+ >
+ <div class="flex items-center">
+ <div class="bg-gray-200 dark:bg-gray-600 rounded p-1 group-hover:bg-indigo-500 dark:group-hover:bg-indigo-600 transition">
+ <Heroicons.document_plus class="w-5 h-5 group-hover:text-white transition" />
+ </div>
+ <span class="ml-2 text-sm text-gray-600 dark:text-gray-500">
+ (or drag-and-drop your unzipped CSVs here)
+ </span>
+ <.live_file_input upload={@upload} class="hidden" />
+ </div>
+
+ <ul id="imported-tables" class="mt-3.5 mb-0.5 space-y-1.5">
+ <.imported_table
+ :for={{table, upload} <- @imported_tables}
+ table={table}
+ upload={upload}
+ errors={if(upload, do: upload_errors(@upload, upload), else: [])}
+ />
+ </ul>
+ </label>
+ """
+ end
+
+ defp confirm_button(assigns) do
+ ~H"""
+ <button
+ type="submit"
+ disabled={not @can_confirm?}
+ class={[
+ "rounded-md w-full bg-indigo-600 px-3.5 py-2.5 text-sm font-semibold text-white shadow-sm hover:bg-indigo-700 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600 disabled:bg-gray-400 dark:disabled:text-gray-400 dark:disabled:bg-gray-700 mt-4",
+ unless(@can_confirm?, do: "cursor-not-allowed")
+ ]}
+ >
+ <%= if @date_range do %>
+ Confirm import from <%= @date_range.first %> to <%= @date_range.last %>
+ <% else %>
+ Confirm import
+ <% end %>
+ </button>
+ """
+ end
+
+ defp imported_table(assigns) do
+ status =
+ cond do
+ assigns.upload && assigns.upload.progress == 100 -> :success
+ assigns.upload && assigns.upload.progress > 0 -> :in_progress
+ not Enum.empty?(assigns.errors) -> :error
+ true -> :empty
+ end
+
+ assigns = assign(assigns, status: status)
+
+ ~H"""
+ <li id={@table} class="ml-1.5">
+ <div class="flex items-center space-x-2">
+ <Heroicons.document_check :if={@status == :success} class="w-4 h-4 text-indigo-600" />
+ <PlausibleWeb.Components.Generic.spinner
+ :if={@status == :in_progress}
+ class="w-4 h-4 text-indigo-600"
+ />
+ <Heroicons.document :if={@status == :empty} class="w-4 h-4 text-gray-400 dark:text-gray-500" />
+ <Heroicons.document :if={@status == :error} class="w-4 h-4 text-red-600 dark:text-red-700" />
+
+ <span class={[
+ "text-sm",
+ if(@upload, do: "dark:text-gray-400", else: "text-gray-400 dark:text-gray-500"),
+ if(@status == :error, do: "text-red-600 dark:text-red-700")
+ ]}>
+ <%= if @upload do %>
+ <%= @upload.client_name %>
+ <% else %>
+ <%= @table %>_YYYYMMDD_YYYYMMDD.csv
+ <% end %>
+ </span>
+ </div>
+
+ <p :for={error <- @errors} class="ml-6 text-sm text-red-600 dark:text-red-700">
+ <%= error_to_string(error) %>
+ </p>
+ </li>
+ """
+ end
+
+ @impl true
+ def handle_event("validate-upload-form", _params, socket) do
+ {:noreply, process_imported_tables(socket)}
+ end
+
+ def handle_event("submit-upload-form", _params, socket) do
+ %{site_id: site_id, user_id: user_id, date_range: date_range} = socket.assigns
+ site = Plausible.Repo.get!(Plausible.Site, site_id)
+ user = Plausible.Repo.get!(Plausible.Auth.User, user_id)
+
+ uploads =
+ consume_uploaded_entries(socket, :import, fn meta, entry ->
+ {:ok, %{"s3_url" => meta.s3_url, "filename" => entry.client_name}}
+ end)
+
+ {:ok, _job} =
+ CSVImporter.new_import(site, user,
+ start_date: date_range.first,
+ end_date: date_range.last,
+ uploads: uploads
+ )
+
+ redirect_to =
+ Routes.site_path(socket, :settings_imports_exports, URI.encode_www_form(site.domain))
+
+ {:noreply, redirect(socket, external: redirect_to)}
+ end
+
+ defp error_to_string(:too_large), do: "is too large (max size is 1 gigabyte)"
+ defp error_to_string(:too_many_files), do: "too many files"
+ defp error_to_string(:not_accepted), do: "unacceptable file types"
+ defp error_to_string(:external_client_failure), do: "browser upload failed"
+
+ defp presign_upload(entry, socket) do
+ %{s3_url: s3_url, presigned_url: upload_url} =
+ Plausible.S3.import_presign_upload(socket.assigns.site_id, entry.client_name)
+
+ {:ok, %{uploader: "S3", s3_url: s3_url, url: upload_url}, socket}
+ end
+
+ defp handle_progress(:import, entry, socket) do
+ if entry.done? do
+ {:noreply, process_imported_tables(socket)}
+ else
+ {:noreply, socket}
+ end
+ end
+
+ defp process_imported_tables(socket) do
+ tables = Plausible.Imported.tables()
+ {completed, in_progress} = uploaded_entries(socket, :import)
+
+ {valid_uploads, invalid_uploads} =
+ Enum.split_with(completed ++ in_progress, &CSVImporter.valid_filename?(&1.client_name))
+
+ imported_tables_all_uploads =
+ Enum.map(tables, fn table ->
+ uploads =
+ Enum.filter(valid_uploads, fn upload ->
+ CSVImporter.extract_table(upload.client_name) == table
+ end)
+
+ {upload, replaced_uploads} = List.pop_at(uploads, -1)
+ {table, upload, replaced_uploads}
+ end)
+
+ imported_tables =
+ Enum.map(imported_tables_all_uploads, fn {table, upload, _replaced_uploads} ->
+ {table, upload}
+ end)
+
+ replaced_uploads =
+ Enum.flat_map(imported_tables_all_uploads, fn {_table, _upload, replaced_uploads} ->
+ replaced_uploads
+ end)
+
+ date_range = CSVImporter.date_range(Enum.map(valid_uploads, & &1.client_name))
+ all_uploaded? = completed != [] and in_progress == []
+
+ socket
+ |> cancel_uploads(invalid_uploads)
+ |> cancel_uploads(replaced_uploads)
+ |> assign(
+ imported_tables: imported_tables,
+ can_confirm?: all_uploaded?,
+ date_range: date_range
+ )
+ end
+
+ defp cancel_uploads(socket, uploads) do
+ Enum.reduce(uploads, socket, fn upload, socket ->
+ cancel_upload(socket, :import, upload.ref)
+ end)
+ end
+end
diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex
index 6a4fd7660c..914b549fba 100644
--- a/lib/plausible_web/router.ex
+++ b/lib/plausible_web/router.ex
@@ -390,7 +390,8 @@ defmodule PlausibleWeb.Router do
delete "/:website/settings/forget-imported", SiteController, :forget_imported
delete "/:website/settings/forget-import/:import_id", SiteController, :forget_import
- post "/:website/settings/export", SiteController, :export
+ post "/:website/settings/export", SiteController, :csv_export
+ get "/:website/settings/import", SiteController, :csv_import
get "/:domain/export", StatsController, :csv_export
get "/:domain/*path", StatsController, :stats
diff --git a/lib/plausible_web/templates/site/csv_import.html.heex b/lib/plausible_web/templates/site/csv_import.html.heex
new file mode 100644
index 0000000000..ae9fc4038d
--- /dev/null
+++ b/lib/plausible_web/templates/site/csv_import.html.heex
@@ -0,0 +1,20 @@
+<div class="max-w-lg w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded p-6 mb-4 mt-8">
+ <h2 class="text-xl font-black dark:text-gray-100">Import from CSV files</h2>
+
+ <div class="my-3 space-y-1.5 text-sm text-gray-400">
+ <p>
+ Please ensure each file follows
+ <.styled_link href="https://plausible.io/docs/csv-import">
+ our CSV format guidelines.
+ </.styled_link>
+ </p>
+
+ <p>
+ You can upload multiple files simultaneously by either selecting them in the file dialog or dragging and dropping them into the designated area.
+ </p>
+ </div>
+
+ <%= live_render(@conn, PlausibleWeb.Live.CSVImport,
+ session: %{"site_id" => @site.id, "user_id" => @current_user.id}
+ ) %>
+</div>
diff --git a/lib/plausible_web/templates/site/settings_imports_exports.html.heex b/lib/plausible_web/templates/site/settings_imports_exports.html.heex
index afc374c32e..4011ac033c 100644
--- a/lib/plausible_web/templates/site/settings_imports_exports.html.heex
+++ b/lib/plausible_web/templates/site/settings_imports_exports.html.heex
@@ -20,9 +20,9 @@
</PlausibleWeb.Components.Generic.button_link>
<PlausibleWeb.Components.Generic.button_link
- class="w-36 h-20 opacity-40 cursor-not-allowed"
+ class="w-36 h-20"
theme="bright"
- href=""
+ href={"/#{URI.encode_www_form(@site.domain)}/settings/import"}
>
<img class="h-16" src="/images/icon/csv_logo.svg" alt="New CSV import" />
</PlausibleWeb.Components.Generic.button_link>