Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Improve import procedure to ensure no time range overlaps (#3970)

Adrian Gruntkowskia7603c9e49d5
diff --git a/lib/plausible/imported.ex b/lib/plausible/imported.ex
index 742886a7ba..82cdc9a8eb 100644
--- a/lib/plausible/imported.ex
+++ b/lib/plausible/imported.ex
@@ -62,9 +62,9 @@ defmodule Plausible.Imported do
}
end
- @spec get_import(non_neg_integer()) :: SiteImport.t() | nil
- def get_import(import_id) do
- Repo.get(SiteImport, import_id)
+ @spec get_import(Site.t(), non_neg_integer()) :: SiteImport.t() | nil
+ def get_import(site, import_id) do
+ Repo.get_by(SiteImport, id: import_id, site_id: site.id)
end
defdelegate listen(), to: Imported.Importer
@@ -83,6 +83,16 @@ defmodule Plausible.Imported do
end
end
+ @spec other_imports_in_progress?(SiteImport.t()) :: boolean()
+ def other_imports_in_progress?(site_import) do
+ Repo.exists?(
+ from(i in SiteImport,
+ where: i.site_id == ^site_import.site_id and i.id != ^site_import.id,
+ where: i.status in ^[SiteImport.pending(), SiteImport.importing()]
+ )
+ )
+ end
+
defp maybe_filter_by_status(query, nil), do: query
defp maybe_filter_by_status(query, status) do
diff --git a/lib/plausible/imported/importer.ex b/lib/plausible/imported/importer.ex
index 7edb713440..bbac488099 100644
--- a/lib/plausible/imported/importer.ex
+++ b/lib/plausible/imported/importer.ex
@@ -91,6 +91,7 @@ defmodule Plausible.Imported.Importer do
like LiveView, where notifications can be listened for via `handle_info/2`.
"""
+ alias Plausible.Imported
alias Plausible.Imported.SiteImport
alias Plausible.Repo
@@ -98,7 +99,7 @@ defmodule Plausible.Imported.Importer do
@callback label() :: String.t()
@callback email_template() :: String.t()
@callback parse_args(map()) :: Keyword.t()
- @callback import_data(Plausible.Imported.SiteImport.t(), Keyword.t()) :: :ok | {:error, any()}
+ @callback import_data(SiteImport.t(), Keyword.t()) :: :ok | {:error, any()}
@callback before_start(SiteImport.t()) :: :ok | {:ok, map()} | {:error, any()}
@callback on_success(SiteImport.t(), map()) :: :ok
@callback on_failure(SiteImport.t()) :: :ok
@@ -108,7 +109,7 @@ defmodule Plausible.Imported.Importer do
@behaviour Plausible.Imported.Importer
@spec new_import(Plausible.Site.t(), Plausible.Auth.User.t(), Keyword.t()) ::
- {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()}
+ {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t() | :import_in_progress}
def new_import(site, user, opts) do
Plausible.Imported.Importer.new_import(name(), site, user, opts, &before_start/1)
end
@@ -222,11 +223,15 @@ defmodule Plausible.Imported.Importer do
:ok = listen()
end
- opts
- |> Keyword.put(:import_id, site_import.id)
- |> Map.new()
- |> Plausible.Workers.ImportAnalytics.new()
- |> Oban.insert()
+ if not Imported.other_imports_in_progress?(site_import) do
+ opts
+ |> Keyword.put(:import_id, site_import.id)
+ |> Map.new()
+ |> Plausible.Workers.ImportAnalytics.new()
+ |> Oban.insert()
+ else
+ {:error, :import_in_progress}
+ end
end
defp mark_complete(site_import, extra_data, on_success_fun) do
diff --git a/lib/plausible_web/controllers/google_analytics_controller.ex b/lib/plausible_web/controllers/google_analytics_controller.ex
index 49eda933dc..ade4752358 100644
--- a/lib/plausible_web/controllers/google_analytics_controller.ex
+++ b/lib/plausible_web/controllers/google_analytics_controller.ex
@@ -289,48 +289,48 @@ defmodule PlausibleWeb.GoogleAnalyticsController do
Routes.site_path(conn, :settings_imports_exports, site.domain)
end
- case Imported.check_dates(site, start_date, end_date) do
- {:ok, start_date, end_date} ->
- if Google.API.property?(property_or_view) do
- {:ok, _} =
- Imported.GoogleAnalytics4.new_import(
- site,
- current_user,
- property: property_or_view,
- label: property_or_view,
- start_date: start_date,
- end_date: end_date,
- access_token: access_token,
- refresh_token: refresh_token,
- token_expires_at: expires_at
- )
- else
- {:ok, _} =
- Imported.UniversalAnalytics.new_import(
- site,
- current_user,
- view_id: property_or_view,
- label: property_or_view,
- start_date: start_date,
- end_date: end_date,
- access_token: access_token,
- refresh_token: refresh_token,
- token_expires_at: expires_at,
- legacy: legacy == "true"
- )
- end
-
+ import_opts = [
+ label: property_or_view,
+ start_date: start_date,
+ end_date: end_date,
+ access_token: access_token,
+ refresh_token: refresh_token,
+ token_expires_at: expires_at,
+ legacy: legacy == "true"
+ ]
+
+ with {:ok, start_date, end_date} <- Imported.check_dates(site, start_date, end_date),
+ import_opts = [{:start_date, start_date}, {:end_date, end_date} | import_opts],
+ {:ok, _} <- schedule_job(site, current_user, property_or_view, import_opts) do
+ conn
+ |> put_flash(:success, "Import scheduled. An email will be sent when it completes.")
+ |> redirect(external: redirect_route)
+ else
+ {:error, :no_time_window} ->
conn
- |> put_flash(:success, "Import scheduled. An email will be sent when it completes.")
+ |> put_flash(
+ :error,
+ "Import failed. No data could be imported because date range overlaps with existing data."
+ )
|> redirect(external: redirect_route)
- {:error, :no_time_window} ->
+ {:error, :import_in_progress} ->
conn
|> put_flash(
:error,
- "Import failed. No data could be imported because date range overlaps with existing data."
+ "There's another import still in progress. Please wait until it's completed or cancel it before starting a new one."
)
|> redirect(external: redirect_route)
end
end
+
+ defp schedule_job(site, current_user, property_or_view, opts) do
+ if Google.API.property?(property_or_view) do
+ opts = Keyword.put(opts, :property, property_or_view)
+ Imported.GoogleAnalytics4.new_import(site, current_user, opts)
+ else
+ opts = Keyword.put(opts, :view_id, property_or_view)
+ Imported.UniversalAnalytics.new_import(site, current_user, opts)
+ end
+ end
end
diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex
index 7d1d89ff96..30a83a7de1 100644
--- a/lib/plausible_web/controllers/site_controller.ex
+++ b/lib/plausible_web/controllers/site_controller.ex
@@ -652,7 +652,7 @@ defmodule PlausibleWeb.SiteController do
|> Plausible.Site.remove_imported_data()
|> Repo.update!()
- site_import = Plausible.Imported.get_import(import_id) ->
+ site_import = Plausible.Imported.get_import(site, import_id) ->
Oban.cancel_all_jobs(
from(j in Oban.Job,
where:
diff --git a/lib/plausible_web/live/imports_exports_settings.ex b/lib/plausible_web/live/imports_exports_settings.ex
index dffd6b244e..0bdc767808 100644
--- a/lib/plausible_web/live/imports_exports_settings.ex
+++ b/lib/plausible_web/live/imports_exports_settings.ex
@@ -42,7 +42,39 @@ defmodule PlausibleWeb.Live.ImportsExportsSettings do
end
def render(assigns) do
+ import_in_progress? =
+ Enum.any?(
+ assigns.site_imports,
+ &(&1.live_status in [SiteImport.pending(), SiteImport.importing()])
+ )
+
+ assigns = assign(assigns, :import_in_progress?, import_in_progress?)
+
~H"""
+ <div class="mt-5 flex gap-x-4">
+ <.button_link
+ class="w-36 h-20"
+ theme="bright"
+ disabled={@import_in_progress?}
+ href={Plausible.Google.API.import_authorize_url(@site.id, "import", legacy: false)}
+ >
+ <img src="/images/icon/google_analytics_logo.svg" alt="Google Analytics import" />
+ </.button_link>
+
+ <.button_link
+ class="w-36 h-20"
+ theme="bright"
+ disabled={@import_in_progress?}
+ href={"/#{URI.encode_www_form(@site.domain)}/settings/import"}
+ >
+ <img class="h-16" src="/images/icon/csv_logo.svg" alt="New CSV import" />
+ </.button_link>
+ </div>
+
+ <p :if={@import_in_progress?} class="mt-4 text-red-400 text-sm">
+ No new imports can be started until the import in progress is completed or cancelled.
+ </p>
+
<header class="relative border-b border-gray-200 pb-4">
<h3 class="mt-8 text-md leading-6 font-medium text-gray-900 dark:text-gray-100">
Existing Imports
diff --git a/lib/plausible_web/templates/site/settings_imports_exports.html.heex b/lib/plausible_web/templates/site/settings_imports_exports.html.heex
index 4011ac033c..5236c16012 100644
--- a/lib/plausible_web/templates/site/settings_imports_exports.html.heex
+++ b/lib/plausible_web/templates/site/settings_imports_exports.html.heex
@@ -10,24 +10,6 @@
<PlausibleWeb.Components.Generic.docs_info slug="google-analytics-import" />
</header>
- <div class="mt-5 flex gap-x-4">
- <PlausibleWeb.Components.Generic.button_link
- class="w-36 h-20"
- theme="bright"
- href={Plausible.Google.API.import_authorize_url(@site.id, "import", legacy: false)}
- >
- <img src="/images/icon/google_analytics_logo.svg" alt="Google Analytics import" />
- </PlausibleWeb.Components.Generic.button_link>
-
- <PlausibleWeb.Components.Generic.button_link
- class="w-36 h-20"
- theme="bright"
- href={"/#{URI.encode_www_form(@site.domain)}/settings/import"}
- >
- <img class="h-16" src="/images/icon/csv_logo.svg" alt="New CSV import" />
- </PlausibleWeb.Components.Generic.button_link>
- </div>
-
<%= live_render(@conn, PlausibleWeb.Live.ImportsExportsSettings,
session: %{"domain" => @site.domain}
) %>
diff --git a/test/plausible/imported/google_analytics4_test.exs b/test/plausible/imported/google_analytics4_test.exs
index 66fa346f4a..097d6c2c63 100644
--- a/test/plausible/imported/google_analytics4_test.exs
+++ b/test/plausible/imported/google_analytics4_test.exs
@@ -43,7 +43,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
token_expires_at: DateTime.to_iso8601(past)
)
- site_import = Plausible.Imported.get_import(job.args.import_id)
+ site_import = Plausible.Imported.get_import(site, job.args.import_id)
assert site_import.label == "properties/123456"
diff --git a/test/plausible_web/controllers/google_analytics_controller_test.exs b/test/plausible_web/controllers/google_analytics_controller_test.exs
index 1977912da0..ccaccf946d 100644
--- a/test/plausible_web/controllers/google_analytics_controller_test.exs
+++ b/test/plausible_web/controllers/google_analytics_controller_test.exs
@@ -855,6 +855,46 @@ defmodule PlausibleWeb.GoogleAnalyticsControllerTest do
)
end
+ test "does not start another import when there's any other in progress for the same site", %{
+ conn: conn,
+ site: site,
+ user: user
+ } do
+ {:ok, job} =
+ Plausible.Imported.NoopImporter.new_import(site, user,
+ start_date: ~D[2020-01-02],
+ end_date: ~D[2021-10-11]
+ )
+
+ post(conn, "/#{site.domain}/settings/google-import", %{
+ "property_or_view" => "properties/123456",
+ "start_date" => "2018-03-01",
+ "end_date" => "2022-03-01",
+ "access_token" => "token",
+ "refresh_token" => "foo",
+ "expires_at" => "2022-09-22T20:01:37.112777",
+ "legacy" => "false"
+ })
+
+ conn =
+ post(conn, "/#{site.domain}/settings/google-import", %{
+ "property_or_view" => "123456",
+ "start_date" => "2018-03-01",
+ "end_date" => "2022-03-01",
+ "access_token" => "token",
+ "refresh_token" => "foo",
+ "expires_at" => "2022-09-22T20:01:37.112777",
+ "legacy" => "true"
+ })
+
+ assert Phoenix.Flash.get(conn.assigns.flash, :error) =~
+ "There's another import still in progress."
+
+ assert [%{id: import_id, legacy: false}] = Plausible.Imported.list_all_imports(site)
+
+ assert job.args.import_id == import_id
+ end
+
for {legacy, view} <- [{"true", :settings_integrations}, {"false", :settings_imports_exports}] do
test "redirects to #{view} with no time window error flash error (legacy: #{legacy})", %{
conn: conn,
diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exs
index 6b6d0cb22d..eae1a62c0e 100644
--- a/test/plausible_web/controllers/site_controller_test.exs
+++ b/test/plausible_web/controllers/site_controller_test.exs
@@ -680,6 +680,25 @@ defmodule PlausibleWeb.SiteControllerTest do
assert resp =~ "Google Analytics (123456)"
assert resp =~ "(98 page views)"
end
+
+ test "disables import buttons when there's import in progress", %{conn: conn, site: site} do
+ _site_import1 = insert(:site_import, site: site, status: SiteImport.completed())
+ _site_import2 = insert(:site_import, site: site, status: SiteImport.importing())
+
+ conn = get(conn, "/#{site.domain}/settings/imports-exports")
+ assert html_response(conn, 200) =~ "No new imports can be started"
+ end
+
+ test "enables import buttons when all imports are in completed or failed state", %{
+ conn: conn,
+ site: site
+ } do
+ _site_import1 = insert(:site_import, site: site, status: SiteImport.completed())
+ _site_import2 = insert(:site_import, site: site, status: SiteImport.failed())
+
+ conn = get(conn, "/#{site.domain}/settings/imports-exports")
+ refute html_response(conn, 200) =~ "No new imports can be started"
+ end
end
describe "GET /:website/settings/integrations for self-hosting" do