Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Improve import procedure to ensure no time range overlaps (#3970)
Adrian Gruntkowskia7603c9e49d5
diff --git a/lib/plausible/imported.ex b/lib/plausible/imported.exindex 742886a7ba..82cdc9a8eb 100644--- a/lib/plausible/imported.ex+++ b/lib/plausible/imported.ex@@ -62,9 +62,9 @@ defmodule Plausible.Imported do}end- @spec get_import(non_neg_integer()) :: SiteImport.t() | nil- def get_import(import_id) do- Repo.get(SiteImport, import_id)+ @spec get_import(Site.t(), non_neg_integer()) :: SiteImport.t() | nil+ def get_import(site, import_id) do+ Repo.get_by(SiteImport, id: import_id, site_id: site.id)enddefdelegate listen(), to: Imported.Importer@@ -83,6 +83,16 @@ defmodule Plausible.Imported doendend+ @spec other_imports_in_progress?(SiteImport.t()) :: boolean()+ def other_imports_in_progress?(site_import) do+ Repo.exists?(+ from(i in SiteImport,+ where: i.site_id == ^site_import.site_id and i.id != ^site_import.id,+ where: i.status in ^[SiteImport.pending(), SiteImport.importing()]+ )+ )+ end+defp maybe_filter_by_status(query, nil), do: querydefp maybe_filter_by_status(query, status) dodiff --git a/lib/plausible/imported/importer.ex b/lib/plausible/imported/importer.exindex 7edb713440..bbac488099 100644--- a/lib/plausible/imported/importer.ex+++ b/lib/plausible/imported/importer.ex@@ -91,6 +91,7 @@ defmodule Plausible.Imported.Importer dolike LiveView, where notifications can be listened for via `handle_info/2`."""+ alias Plausible.Importedalias Plausible.Imported.SiteImportalias Plausible.Repo@@ -98,7 +99,7 @@ defmodule Plausible.Imported.Importer do@callback label() :: String.t()@callback email_template() :: String.t()@callback parse_args(map()) :: Keyword.t()- @callback import_data(Plausible.Imported.SiteImport.t(), Keyword.t()) :: :ok | {:error, any()}+ @callback import_data(SiteImport.t(), Keyword.t()) :: :ok | {:error, any()}@callback before_start(SiteImport.t()) :: :ok | {:ok, map()} | {:error, any()}@callback on_success(SiteImport.t(), map()) :: :ok@callback on_failure(SiteImport.t()) :: :ok@@ -108,7 +109,7 @@ defmodule Plausible.Imported.Importer do@behaviour Plausible.Imported.Importer@spec new_import(Plausible.Site.t(), Plausible.Auth.User.t(), Keyword.t()) ::- {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()}+ {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t() | :import_in_progress}def new_import(site, user, opts) doPlausible.Imported.Importer.new_import(name(), site, user, opts, &before_start/1)end@@ -222,11 +223,15 @@ defmodule Plausible.Imported.Importer do:ok = listen()end- opts- |> Keyword.put(:import_id, site_import.id)- |> Map.new()- |> Plausible.Workers.ImportAnalytics.new()- |> Oban.insert()+ if not Imported.other_imports_in_progress?(site_import) do+ opts+ |> Keyword.put(:import_id, site_import.id)+ |> Map.new()+ |> Plausible.Workers.ImportAnalytics.new()+ |> Oban.insert()+ else+ {:error, :import_in_progress}+ endenddefp mark_complete(site_import, extra_data, on_success_fun) dodiff --git a/lib/plausible_web/controllers/google_analytics_controller.ex b/lib/plausible_web/controllers/google_analytics_controller.exindex 49eda933dc..ade4752358 100644--- a/lib/plausible_web/controllers/google_analytics_controller.ex+++ b/lib/plausible_web/controllers/google_analytics_controller.ex@@ -289,48 +289,48 @@ defmodule PlausibleWeb.GoogleAnalyticsController doRoutes.site_path(conn, :settings_imports_exports, site.domain)end- case Imported.check_dates(site, start_date, end_date) do- {:ok, start_date, end_date} ->- if Google.API.property?(property_or_view) do- {:ok, _} =- Imported.GoogleAnalytics4.new_import(- site,- current_user,- property: property_or_view,- label: property_or_view,- start_date: start_date,- end_date: end_date,- access_token: access_token,- refresh_token: refresh_token,- token_expires_at: expires_at- )- else- {:ok, _} =- Imported.UniversalAnalytics.new_import(- site,- current_user,- view_id: property_or_view,- label: property_or_view,- start_date: start_date,- end_date: end_date,- access_token: access_token,- refresh_token: refresh_token,- token_expires_at: expires_at,- legacy: legacy == "true"- )- end-+ import_opts = [+ label: property_or_view,+ start_date: start_date,+ end_date: end_date,+ access_token: access_token,+ refresh_token: refresh_token,+ token_expires_at: expires_at,+ legacy: legacy == "true"+ ]++ with {:ok, start_date, end_date} <- Imported.check_dates(site, start_date, end_date),+ import_opts = [{:start_date, start_date}, {:end_date, end_date} | import_opts],+ {:ok, _} <- schedule_job(site, current_user, property_or_view, import_opts) do+ conn+ |> put_flash(:success, "Import scheduled. An email will be sent when it completes.")+ |> redirect(external: redirect_route)+ else+ {:error, :no_time_window} ->conn- |> put_flash(:success, "Import scheduled. An email will be sent when it completes.")+ |> put_flash(+ :error,+ "Import failed. No data could be imported because date range overlaps with existing data."+ )|> redirect(external: redirect_route)- {:error, :no_time_window} ->+ {:error, :import_in_progress} ->conn|> put_flash(:error,- "Import failed. No data could be imported because date range overlaps with existing data."+ "There's another import still in progress. Please wait until it's completed or cancel it before starting a new one.")|> redirect(external: redirect_route)endend++ defp schedule_job(site, current_user, property_or_view, opts) do+ if Google.API.property?(property_or_view) do+ opts = Keyword.put(opts, :property, property_or_view)+ Imported.GoogleAnalytics4.new_import(site, current_user, opts)+ else+ opts = Keyword.put(opts, :view_id, property_or_view)+ Imported.UniversalAnalytics.new_import(site, current_user, opts)+ end+ endenddiff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.exindex 7d1d89ff96..30a83a7de1 100644--- a/lib/plausible_web/controllers/site_controller.ex+++ b/lib/plausible_web/controllers/site_controller.ex@@ -652,7 +652,7 @@ defmodule PlausibleWeb.SiteController do|> Plausible.Site.remove_imported_data()|> Repo.update!()- site_import = Plausible.Imported.get_import(import_id) ->+ site_import = Plausible.Imported.get_import(site, import_id) ->Oban.cancel_all_jobs(from(j in Oban.Job,where:diff --git a/lib/plausible_web/live/imports_exports_settings.ex b/lib/plausible_web/live/imports_exports_settings.exindex dffd6b244e..0bdc767808 100644--- a/lib/plausible_web/live/imports_exports_settings.ex+++ b/lib/plausible_web/live/imports_exports_settings.ex@@ -42,7 +42,39 @@ defmodule PlausibleWeb.Live.ImportsExportsSettings doenddef render(assigns) do+ import_in_progress? =+ Enum.any?(+ assigns.site_imports,+ &(&1.live_status in [SiteImport.pending(), SiteImport.importing()])+ )++ assigns = assign(assigns, :import_in_progress?, import_in_progress?)+~H"""+ <div class="mt-5 flex gap-x-4">+ <.button_link+ class="w-36 h-20"+ theme="bright"+ disabled={@import_in_progress?}+ href={Plausible.Google.API.import_authorize_url(@site.id, "import", legacy: false)}+ >+ <img src="/images/icon/google_analytics_logo.svg" alt="Google Analytics import" />+ </.button_link>++ <.button_link+ class="w-36 h-20"+ theme="bright"+ disabled={@import_in_progress?}+ href={"/#{URI.encode_www_form(@site.domain)}/settings/import"}+ >+ <img class="h-16" src="/images/icon/csv_logo.svg" alt="New CSV import" />+ </.button_link>+ </div>++ <p :if={@import_in_progress?} class="mt-4 text-red-400 text-sm">+ No new imports can be started until the import in progress is completed or cancelled.+ </p>+<header class="relative border-b border-gray-200 pb-4"><h3 class="mt-8 text-md leading-6 font-medium text-gray-900 dark:text-gray-100">Existing Importsdiff --git a/lib/plausible_web/templates/site/settings_imports_exports.html.heex b/lib/plausible_web/templates/site/settings_imports_exports.html.heexindex 4011ac033c..5236c16012 100644--- a/lib/plausible_web/templates/site/settings_imports_exports.html.heex+++ b/lib/plausible_web/templates/site/settings_imports_exports.html.heex@@ -10,24 +10,6 @@<PlausibleWeb.Components.Generic.docs_info slug="google-analytics-import" /></header>- <div class="mt-5 flex gap-x-4">- <PlausibleWeb.Components.Generic.button_link- class="w-36 h-20"- theme="bright"- href={Plausible.Google.API.import_authorize_url(@site.id, "import", legacy: false)}- >- <img src="/images/icon/google_analytics_logo.svg" alt="Google Analytics import" />- </PlausibleWeb.Components.Generic.button_link>-- <PlausibleWeb.Components.Generic.button_link- class="w-36 h-20"- theme="bright"- href={"/#{URI.encode_www_form(@site.domain)}/settings/import"}- >- <img class="h-16" src="/images/icon/csv_logo.svg" alt="New CSV import" />- </PlausibleWeb.Components.Generic.button_link>- </div>-<%= live_render(@conn, PlausibleWeb.Live.ImportsExportsSettings,session: %{"domain" => @site.domain}) %>diff --git a/test/plausible/imported/google_analytics4_test.exs b/test/plausible/imported/google_analytics4_test.exsindex 66fa346f4a..097d6c2c63 100644--- a/test/plausible/imported/google_analytics4_test.exs+++ b/test/plausible/imported/google_analytics4_test.exs@@ -43,7 +43,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test dotoken_expires_at: DateTime.to_iso8601(past))- site_import = Plausible.Imported.get_import(job.args.import_id)+ site_import = Plausible.Imported.get_import(site, job.args.import_id)assert site_import.label == "properties/123456"diff --git a/test/plausible_web/controllers/google_analytics_controller_test.exs b/test/plausible_web/controllers/google_analytics_controller_test.exsindex 1977912da0..ccaccf946d 100644--- a/test/plausible_web/controllers/google_analytics_controller_test.exs+++ b/test/plausible_web/controllers/google_analytics_controller_test.exs@@ -855,6 +855,46 @@ defmodule PlausibleWeb.GoogleAnalyticsControllerTest do)end+ test "does not start another import when there's any other in progress for the same site", %{+ conn: conn,+ site: site,+ user: user+ } do+ {:ok, job} =+ Plausible.Imported.NoopImporter.new_import(site, user,+ start_date: ~D[2020-01-02],+ end_date: ~D[2021-10-11]+ )++ post(conn, "/#{site.domain}/settings/google-import", %{+ "property_or_view" => "properties/123456",+ "start_date" => "2018-03-01",+ "end_date" => "2022-03-01",+ "access_token" => "token",+ "refresh_token" => "foo",+ "expires_at" => "2022-09-22T20:01:37.112777",+ "legacy" => "false"+ })++ conn =+ post(conn, "/#{site.domain}/settings/google-import", %{+ "property_or_view" => "123456",+ "start_date" => "2018-03-01",+ "end_date" => "2022-03-01",+ "access_token" => "token",+ "refresh_token" => "foo",+ "expires_at" => "2022-09-22T20:01:37.112777",+ "legacy" => "true"+ })++ assert Phoenix.Flash.get(conn.assigns.flash, :error) =~+ "There's another import still in progress."++ assert [%{id: import_id, legacy: false}] = Plausible.Imported.list_all_imports(site)++ assert job.args.import_id == import_id+ end+for {legacy, view} <- [{"true", :settings_integrations}, {"false", :settings_imports_exports}] dotest "redirects to #{view} with no time window error flash error (legacy: #{legacy})", %{conn: conn,diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exsindex 6b6d0cb22d..eae1a62c0e 100644--- a/test/plausible_web/controllers/site_controller_test.exs+++ b/test/plausible_web/controllers/site_controller_test.exs@@ -680,6 +680,25 @@ defmodule PlausibleWeb.SiteControllerTest doassert resp =~ "Google Analytics (123456)"assert resp =~ "(98 page views)"end++ test "disables import buttons when there's import in progress", %{conn: conn, site: site} do+ _site_import1 = insert(:site_import, site: site, status: SiteImport.completed())+ _site_import2 = insert(:site_import, site: site, status: SiteImport.importing())++ conn = get(conn, "/#{site.domain}/settings/imports-exports")+ assert html_response(conn, 200) =~ "No new imports can be started"+ end++ test "enables import buttons when all imports are in completed or failed state", %{+ conn: conn,+ site: site+ } do+ _site_import1 = insert(:site_import, site: site, status: SiteImport.completed())+ _site_import2 = insert(:site_import, site: site, status: SiteImport.failed())++ conn = get(conn, "/#{site.domain}/settings/imports-exports")+ refute html_response(conn, 200) =~ "No new imports can be started"+ endenddescribe "GET /:website/settings/integrations for self-hosting" do