Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Implement UI for multiple imports (#3727)

Adrian Gruntkowski39aa81a16f6e
diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex
index 24e6771a3a..085513886d 100644
--- a/lib/plausible/google/api.ex
+++ b/lib/plausible/google/api.ex
@@ -17,9 +17,9 @@ defmodule Plausible.Google.Api do
Jason.encode!([site_id, redirect_to])
end
- def import_authorize_url(site_id, redirect_to) do
+ def import_authorize_url(site_id, redirect_to, legacy \\ true) do
"https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@import_scope}&state=" <>
- Jason.encode!([site_id, redirect_to])
+ Jason.encode!([site_id, redirect_to, legacy])
end
def fetch_verified_properties(auth) do
diff --git a/lib/plausible/imported.ex b/lib/plausible/imported.ex
index f141780288..18058f6ec0 100644
--- a/lib/plausible/imported.ex
+++ b/lib/plausible/imported.ex
@@ -40,6 +40,9 @@ defmodule Plausible.Imported do
@spec tables() :: [String.t()]
def tables, do: @table_names
+ @spec max_complete_imports() :: non_neg_integer()
+ def max_complete_imports(), do: @max_complete_imports
+
@spec load_import_data(Site.t()) :: Site.t()
def load_import_data(%{import_data_loaded: true} = site), do: site
@@ -56,10 +59,24 @@ defmodule Plausible.Imported do
}
end
+ @spec get_import(non_neg_integer()) :: SiteImport.t() | nil
+ def get_import(import_id) do
+ Repo.get(SiteImport, import_id)
+ end
+
+ defdelegate listen(), to: Imported.Importer
+
@spec list_all_imports(Site.t()) :: [SiteImport.t()]
def list_all_imports(site) do
- from(i in SiteImport, where: i.site_id == ^site.id)
- |> Repo.all()
+ imports =
+ from(i in SiteImport, where: i.site_id == ^site.id, order_by: [desc: i.inserted_at])
+ |> Repo.all()
+
+ if site.imported_data && not Enum.any?(imports, & &1.legacy) do
+ imports ++ [SiteImport.from_legacy(site.imported_data)]
+ else
+ imports
+ end
end
@spec list_complete_import_ids(Site.t()) :: [non_neg_integer()]
diff --git a/lib/plausible/imported/importer.ex b/lib/plausible/imported/importer.ex
index 60cebea1c1..d196bb678b 100644
--- a/lib/plausible/imported/importer.ex
+++ b/lib/plausible/imported/importer.ex
@@ -155,7 +155,7 @@ defmodule Plausible.Imported.Importer do
def new_import(source, site, user, opts, before_start_fun) do
import_params =
opts
- |> Keyword.take([:start_date, :end_date])
+ |> Keyword.take([:start_date, :end_date, :legacy])
|> Keyword.put(:source, source)
|> Map.new()
@@ -205,11 +205,21 @@ defmodule Plausible.Imported.Importer do
Oban.Notifier.notify(Oban, @oban_channel, %{event => site_import.id})
end
+ @doc """
+ Allows to explicitly start listening for importer job notifications.
+
+ Listener must explicitly filter out a subset of imports that apply to the given context.
+ """
+ @spec listen() :: :ok
+ def listen() do
+ :ok = Oban.Notifier.listen([@oban_channel])
+ end
+
defp schedule_job(site_import, opts) do
{listen?, opts} = Keyword.pop(opts, :listen?, false)
if listen? do
- :ok = Oban.Notifier.listen([@oban_channel])
+ :ok = listen()
end
opts
diff --git a/lib/plausible/imported/site_import.ex b/lib/plausible/imported/site_import.ex
index d8774fcfb1..594fe8db52 100644
--- a/lib/plausible/imported/site_import.ex
+++ b/lib/plausible/imported/site_import.ex
@@ -20,6 +20,7 @@ defmodule Plausible.Imported.SiteImport do
field :end_date, :date
field :source, Ecto.Enum, values: ImportSources.names()
field :status, Ecto.Enum, values: @statuses
+ field :legacy, :boolean, default: false
belongs_to :site, Site
belongs_to :imported_by, User
@@ -37,10 +38,29 @@ defmodule Plausible.Imported.SiteImport do
def label(%__MODULE__{source: source}), do: ImportSources.by_name(source).label()
def label(%Site.ImportedData{source: source}), do: source
+ @spec from_legacy(Site.ImportedData.t()) :: t()
+ def from_legacy(%Site.ImportedData{} = data) do
+ status =
+ case data.status do
+ "ok" -> completed()
+ "error" -> failed()
+ _ -> importing()
+ end
+
+ %__MODULE__{
+ id: 0,
+ legacy: true,
+ start_date: data.start_date,
+ end_date: data.end_date,
+ source: :universal_analytics,
+ status: status
+ }
+ end
+
@spec create_changeset(Site.t(), User.t(), map()) :: Ecto.Changeset.t()
def create_changeset(site, user, params) do
%__MODULE__{}
- |> cast(params, [:source, :start_date, :end_date])
+ |> cast(params, [:source, :start_date, :end_date, :legacy])
|> validate_required([:source])
|> put_assoc(:site, site)
|> put_assoc(:imported_by, user)
diff --git a/lib/plausible/imported/universal_analytics.ex b/lib/plausible/imported/universal_analytics.ex
index 246be89f4d..1c7a1e093e 100644
--- a/lib/plausible/imported/universal_analytics.ex
+++ b/lib/plausible/imported/universal_analytics.ex
@@ -20,37 +20,43 @@ defmodule Plausible.Imported.UniversalAnalytics do
@impl true
def before_start(site_import) do
- site = Repo.preload(site_import, :site).site
-
- site
- |> Plausible.Site.start_import(
- site_import.start_date,
- site_import.end_date,
- label()
- )
- |> Repo.update!()
+ if site_import.legacy do
+ site = Repo.preload(site_import, :site).site
+
+ site
+ |> Plausible.Site.start_import(
+ site_import.start_date,
+ site_import.end_date,
+ label()
+ )
+ |> Repo.update!()
+ end
:ok
end
@impl true
def on_success(site_import, _extra_data) do
- site = Repo.preload(site_import, :site).site
+ if site_import.legacy do
+ site = Repo.preload(site_import, :site).site
- site
- |> Plausible.Site.import_success()
- |> Repo.update!()
+ site
+ |> Plausible.Site.import_success()
+ |> Repo.update!()
+ end
:ok
end
@impl true
def on_failure(site_import) do
- site = Repo.preload(site_import, :site).site
+ if site_import.legacy do
+ site = Repo.preload(site_import, :site).site
- site
- |> Plausible.Site.import_failure()
- |> Repo.update!()
+ site
+ |> Plausible.Site.import_failure()
+ |> Repo.update!()
+ end
:ok
end
diff --git a/lib/plausible/purge.ex b/lib/plausible/purge.ex
index 9fe11b077e..50fd3ed22e 100644
--- a/lib/plausible/purge.ex
+++ b/lib/plausible/purge.ex
@@ -58,17 +58,19 @@ defmodule Plausible.Purge do
def delete_imported_stats!(%Plausible.Imported.SiteImport{} = site_import) do
site_import = Repo.preload(site_import, :site)
+ delete_imported_stats!(site_import.site, site_import.id)
+ :ok
+ end
+
+ def delete_imported_stats!(%Plausible.Site{} = site, import_id) do
Enum.each(Plausible.Imported.tables(), fn table ->
sql = "ALTER TABLE #{table} DELETE WHERE site_id = {$0:UInt64} AND import_id = {$1:UInt64}"
- Ecto.Adapters.SQL.query!(Plausible.ImportDeletionRepo, sql, [
- site_import.site_id,
- site_import.id
- ])
+ Ecto.Adapters.SQL.query!(Plausible.ImportDeletionRepo, sql, [site.id, import_id])
end)
- Plausible.Sites.clear_stats_start_date!(site_import.site)
+ Plausible.Sites.clear_stats_start_date!(site)
:ok
end
diff --git a/lib/plausible/sites.ex b/lib/plausible/sites.ex
index 1c8ac51e18..12d4235adf 100644
--- a/lib/plausible/sites.ex
+++ b/lib/plausible/sites.ex
@@ -246,7 +246,7 @@ defmodule Plausible.Sites do
start_date =
[
site.earliest_import_start_date,
- Plausible.Stats.Clickhouse.pageview_start_date_local(site)
+ native_stats_start_date(site)
]
|> Enum.reject(&is_nil/1)
|> Enum.min(Date, fn -> nil end)
@@ -261,6 +261,11 @@ defmodule Plausible.Sites do
end
end
+ @spec native_stats_start_date(Site.t()) :: Date.t() | nil
+ def native_stats_start_date(site) do
+ Plausible.Stats.Clickhouse.pageview_start_date_local(site)
+ end
+
def has_stats?(site) do
!!stats_start_date(site)
end
diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex
index 782b868320..139e92e99f 100644
--- a/lib/plausible/stats/clickhouse.ex
+++ b/lib/plausible/stats/clickhouse.ex
@@ -39,6 +39,17 @@ defmodule Plausible.Stats.Clickhouse do
)
end
+ @spec imported_pageview_counts(Plausible.Site.t()) :: %{non_neg_integer() => non_neg_integer()}
+ def imported_pageview_counts(site) do
+ from(i in "imported_visitors",
+ where: i.site_id == ^site.id,
+ group_by: i.import_id,
+ select: {i.import_id, sum(i.pageviews)}
+ )
+ |> Plausible.ClickhouseRepo.all()
+ |> Map.new()
+ end
+
def usage_breakdown([d | _] = domains, date_range) when is_binary(d) do
Enum.chunk_every(domains, 300)
|> Enum.reduce({0, 0}, fn domains, {pageviews_total, custom_events_total} ->
diff --git a/lib/plausible_web/components/generic.ex b/lib/plausible_web/components/generic.ex
index e0b792a5a6..1d2a62b06e 100644
--- a/lib/plausible_web/components/generic.ex
+++ b/lib/plausible_web/components/generic.ex
@@ -19,7 +19,18 @@ defmodule PlausibleWeb.Components.Generic do
}
}
+ @button_themes %{
+ "primary" => "bg-indigo-600 text-white hover:bg-indigo-700 focus-visible:outline-indigo-600",
+ "bright" =>
+ "border border-gray-200 bg-gray-100 dark:bg-gray-300 text-gray-800 hover:bg-gray-200 focus-visible:outline-gray-100",
+ "danger" =>
+ "border border-gray-300 dark:border-gray-500 text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:border-blue-300 active:text-red-800"
+ }
+
+ @button_base_class "inline-flex items-center justify-center gap-x-2 rounded-md px-3.5 py-2.5 text-sm font-semibold shadow-sm focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 disabled:bg-gray-400 dark:disabled:text-white dark:disabled:text-gray-400 dark:disabled:bg-gray-700"
+
attr(:type, :string, default: "button")
+ attr(:theme, :string, default: "primary")
attr(:class, :string, default: "")
attr(:disabled, :boolean, default: false)
attr(:rest, :global)
@@ -27,12 +38,19 @@ defmodule PlausibleWeb.Components.Generic do
slot(:inner_block)
def button(assigns) do
+ assigns =
+ assign(assigns,
+ button_base_class: @button_base_class,
+ theme_class: @button_themes[assigns.theme]
+ )
+
~H"""
<button
type={@type}
disabled={@disabled}
class={[
- "inline-flex items-center justify-center gap-x-2 rounded-md bg-indigo-600 px-3.5 py-2.5 text-sm font-semibold text-white shadow-sm hover:bg-indigo-700 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600 disabled:bg-gray-400 dark:disabled:text-white dark:disabled:text-gray-400 dark:disabled:bg-gray-700",
+ @button_base_class,
+ @theme_class,
@class
]}
{@rest}
@@ -44,16 +62,32 @@ defmodule PlausibleWeb.Components.Generic do
attr(:href, :string, required: true)
attr(:class, :string, default: "")
+ attr(:theme, :string, default: "primary")
+ attr(:disabled, :boolean, default: false)
attr(:rest, :global)
slot(:inner_block)
def button_link(assigns) do
+ theme_class =
+ if assigns.disabled do
+ "bg-gray-400 text-white dark:text-white dark:text-gray-400 dark:bg-gray-700 pointer-events-none cursor-default"
+ else
+ @button_themes[assigns.theme]
+ end
+
+ assigns =
+ assign(assigns,
+ button_base_class: @button_base_class,
+ theme_class: theme_class
+ )
+
~H"""
<.link
href={@href}
class={[
- "inline-flex items-center justify-center gap-x-2 rounded-md bg-indigo-600 px-3.5 py-2.5 text-sm font-semibold text-white shadow-sm hover:bg-indigo-700 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600 disabled:bg-gray-400 dark:disabled:bg-gray-800",
+ @button_base_class,
+ @theme_class,
@class
]}
{@rest}
diff --git a/lib/plausible_web/controllers/auth_controller.ex b/lib/plausible_web/controllers/auth_controller.ex
index fcf9ebee2e..11a68efc52 100644
--- a/lib/plausible_web/controllers/auth_controller.ex
+++ b/lib/plausible_web/controllers/auth_controller.ex
@@ -697,9 +697,24 @@ defmodule PlausibleWeb.AuthController do
end
def google_auth_callback(conn, %{"error" => error, "state" => state} = params) do
- [site_id, _redirect_to] = Jason.decode!(state)
+ [site_id, _redirected_to, legacy] =
+ case Jason.decode!(state) do
+ [site_id, redirect_to] ->
+ [site_id, redirect_to, true]
+
+ [site_id, redirect_to, legacy] ->
+ [site_id, redirect_to, legacy]
+ end
+
site = Repo.get(Plausible.Site, site_id)
+ redirect_route =
+ if legacy do
+ Routes.site_path(conn, :settings_integrations, site.domain)
+ else
+ Routes.site_path(conn, :settings_imports_exports, site.domain)
+ end
+
case error do
"access_denied" ->
conn
@@ -707,7 +722,7 @@ defmodule PlausibleWeb.AuthController do
:error,
"We were unable to authenticate your Google Analytics account. Please check that you have granted us permission to 'See and download your Google Analytics data' and try again."
)
- |> redirect(external: Routes.site_path(conn, :settings_general, site.domain))
+ |> redirect(external: redirect_route)
message when message in ["server_error", "temporarily_unavailable"] ->
conn
@@ -715,7 +730,7 @@ defmodule PlausibleWeb.AuthController do
:error,
"We are unable to authenticate your Google Analytics account because Google's authentication service is temporarily unavailable. Please try again in a few moments."
)
- |> redirect(external: Routes.site_path(conn, :settings_general, site.domain))
+ |> redirect(external: redirect_route)
_any ->
Sentry.capture_message("Google OAuth callback failed. Reason: #{inspect(params)}")
@@ -725,13 +740,22 @@ defmodule PlausibleWeb.AuthController do
:error,
"We were unable to authenticate your Google Analytics account. If the problem persists, please contact support for assistance."
)
- |> redirect(external: Routes.site_path(conn, :settings_general, site.domain))
+ |> redirect(external: redirect_route)
end
end
def google_auth_callback(conn, %{"code" => code, "state" => state}) do
res = Plausible.Google.HTTP.fetch_access_token(code)
- [site_id, redirect_to] = Jason.decode!(state)
+
+ [site_id, redirect_to, legacy] =
+ case Jason.decode!(state) do
+ [site_id, redirect_to] ->
+ [site_id, redirect_to, true]
+
+ [site_id, redirect_to, legacy] ->
+ [site_id, redirect_to, legacy]
+ end
+
site = Repo.get(Plausible.Site, site_id)
expires_at = NaiveDateTime.add(NaiveDateTime.utc_now(), res["expires_in"])
@@ -742,7 +766,8 @@ defmodule PlausibleWeb.AuthController do
Routes.site_path(conn, :import_from_google_view_id_form, site.domain,
access_token: res["access_token"],
refresh_token: res["refresh_token"],
- expires_at: NaiveDateTime.to_iso8601(expires_at)
+ expires_at: NaiveDateTime.to_iso8601(expires_at),
+ legacy: legacy
)
)
diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex
index 1dcf679e8b..51c04c7991 100644
--- a/lib/plausible_web/controllers/site_controller.ex
+++ b/lib/plausible_web/controllers/site_controller.ex
@@ -4,10 +4,12 @@ defmodule PlausibleWeb.SiteController do
alias Plausible.Sites
alias Plausible.Billing.Quota
- plug PlausibleWeb.RequireAccountPlug
+ plug(PlausibleWeb.RequireAccountPlug)
- plug PlausibleWeb.AuthorizeSiteAccess,
- [:owner, :admin, :super_admin] when action not in [:new, :create_site]
+ plug(
+ PlausibleWeb.AuthorizeSiteAccess,
+ [:owner, :admin, :super_admin] when action not in [:new, :create_site]
+ )
def new(conn, _params) do
current_user = conn.assigns[:current_user]
@@ -62,10 +64,11 @@ defmodule PlausibleWeb.SiteController do
is_first_site =
!Repo.exists?(
- from sm in Plausible.Site.Membership,
+ from(sm in Plausible.Site.Membership,
where:
sm.user_id == ^user.id and
sm.site_id != ^site.id
+ )
)
conn
@@ -144,7 +147,7 @@ defmodule PlausibleWeb.SiteController do
def settings_visibility(conn, _params) do
site = conn.assigns[:site]
- shared_links = Repo.all(from l in Plausible.Site.SharedLink, where: l.site_id == ^site.id)
+ shared_links = Repo.all(from(l in Plausible.Site.SharedLink, where: l.site_id == ^site.id))
conn
|> render("settings_visibility.html",
@@ -267,6 +270,23 @@ defmodule PlausibleWeb.SiteController do
)
end
+ def settings_imports_exports(conn, _params) do
+ site = conn.assigns.site
+
+ if FunWithFlags.enabled?(:imports_exports, for: site) do
+ conn
+ |> render("settings_imports_exports.html",
+ site: site,
+ dogfood_page_path: "/:dashboard/settings/imports-exports",
+ connect_live_socket: true,
+ layout: {PlausibleWeb.LayoutView, "site_settings.html"}
+ )
+ else
+ conn
+ |> redirect(external: Routes.site_path(conn, :settings, site.domain))
+ end
+ end
+
def update_google_auth(conn, %{"google_auth" => attrs}) do
site = conn.assigns[:site] |> Repo.preload(:google_auth)
@@ -374,7 +394,7 @@ defmodule PlausibleWeb.SiteController do
def disable_weekly_report(conn, _params) do
site = conn.assigns[:site]
- Repo.delete_all(from wr in Plausible.Site.WeeklyReport, where: wr.site_id == ^site.id)
+ Repo.delete_all(from(wr in Plausible.Site.WeeklyReport, where: wr.site_id == ^site.id))
conn
|> put_flash(:success, "You will not receive weekly email reports going forward")
@@ -428,7 +448,7 @@ defmodule PlausibleWeb.SiteController do
def disable_monthly_report(conn, _params) do
site = conn.assigns[:site]
- Repo.delete_all(from mr in Plausible.Site.MonthlyReport, where: mr.site_id == ^site.id)
+ Repo.delete_all(from(mr in Plausible.Site.MonthlyReport, where: mr.site_id == ^site.id))
conn
|> put_flash(:success, "You will not receive monthly email reports going forward")
@@ -488,7 +508,7 @@ defmodule PlausibleWeb.SiteController do
def disable_spike_notification(conn, _params) do
site = conn.assigns[:site]
- Repo.delete_all(from mr in Plausible.Site.SpikeNotification, where: mr.site_id == ^site.id)
+ Repo.delete_all(from(mr in Plausible.Site.SpikeNotification, where: mr.site_id == ^site.id))
conn
|> put_flash(:success, "Spike notification disabled")
@@ -604,9 +624,10 @@ defmodule PlausibleWeb.SiteController do
site_id = site.id
case Repo.delete_all(
- from l in Plausible.Site.SharedLink,
+ from(l in Plausible.Site.SharedLink,
where: l.slug == ^slug,
where: l.site_id == ^site_id
+ )
) do
{1, _} ->
conn
@@ -624,7 +645,8 @@ defmodule PlausibleWeb.SiteController do
"view_id" => view_id,
"access_token" => access_token,
"refresh_token" => refresh_token,
- "expires_at" => expires_at
+ "expires_at" => expires_at,
+ "legacy" => legacy
}) do
site = conn.assigns[:site]
@@ -636,6 +658,7 @@ defmodule PlausibleWeb.SiteController do
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
+ legacy: legacy,
layout: {PlausibleWeb.LayoutView, "focus.html"}
)
end
@@ -643,8 +666,16 @@ defmodule PlausibleWeb.SiteController do
def import_from_google_view_id_form(conn, %{
"access_token" => access_token,
"refresh_token" => refresh_token,
- "expires_at" => expires_at
+ "expires_at" => expires_at,
+ "legacy" => legacy
}) do
+ redirect_route =
+ if legacy == "true" do
+ Routes.site_path(conn, :settings_integrations, conn.assigns.site.domain)
+ else
+ Routes.site_path(conn, :settings_imports_exports, conn.assigns.site.domain)
+ end
+
case Plausible.Google.Api.list_views(access_token) do
{:ok, view_ids} ->
conn
@@ -655,6 +686,7 @@ defmodule PlausibleWeb.SiteController do
expires_at: expires_at,
site: conn.assigns.site,
view_ids: view_ids,
+ legacy: legacy,
layout: {PlausibleWeb.LayoutView, "focus.html"}
)
@@ -664,7 +696,7 @@ defmodule PlausibleWeb.SiteController do
:error,
"We were unable to authenticate your Google Analytics account. Please check that you have granted us permission to 'See and download your Google Analytics data' and try again."
)
- |> redirect(to: Routes.site_path(conn, :settings_general, conn.assigns.site.domain))
+ |> redirect(external: redirect_route)
{:error, _any} ->
conn
@@ -672,7 +704,7 @@ defmodule PlausibleWeb.SiteController do
:error,
"We were unable to list your Google Analytics properties. If the problem persists, please contact support for assistance."
)
- |> redirect(to: Routes.site_path(conn, :settings_general, conn.assigns.site.domain))
+ |> redirect(external: redirect_route)
end
end
@@ -682,7 +714,8 @@ defmodule PlausibleWeb.SiteController do
"view_id" => view_id,
"access_token" => access_token,
"refresh_token" => refresh_token,
- "expires_at" => expires_at
+ "expires_at" => expires_at,
+ "legacy" => legacy
}) do
site = conn.assigns[:site]
start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token)
@@ -701,6 +734,7 @@ defmodule PlausibleWeb.SiteController do
site: site,
view_ids: view_ids,
selected_view_id_error: "No data found. Nothing to import",
+ legacy: legacy,
layout: {PlausibleWeb.LayoutView, "focus.html"}
)
@@ -712,7 +746,8 @@ defmodule PlausibleWeb.SiteController do
view_id: view_id,
access_token: access_token,
refresh_token: refresh_token,
- expires_at: expires_at
+ expires_at: expires_at,
+ legacy: legacy
)
)
else
@@ -722,7 +757,8 @@ defmodule PlausibleWeb.SiteController do
view_id: view_id,
access_token: access_token,
refresh_token: refresh_token,
- expires_at: expires_at
+ expires_at: expires_at,
+ legacy: legacy
)
)
end
@@ -733,12 +769,14 @@ defmodule PlausibleWeb.SiteController do
"view_id" => view_id,
"access_token" => access_token,
"refresh_token" => refresh_token,
- "expires_at" => expires_at
+ "expires_at" => expires_at,
+ "legacy" => legacy
}) do
site = conn.assigns[:site]
start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token)
- end_date = Plausible.Sites.stats_start_date(site) || Timex.today(site.timezone)
+
+ end_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone)
{:ok, {view_name, view_id}} = Plausible.Google.Api.get_view(access_token, view_id)
@@ -753,6 +791,7 @@ defmodule PlausibleWeb.SiteController do
selected_view_id_name: view_name,
start_date: start_date,
end_date: end_date,
+ legacy: legacy,
layout: {PlausibleWeb.LayoutView, "focus.html"}
)
end
@@ -763,11 +802,19 @@ defmodule PlausibleWeb.SiteController do
"end_date" => end_date,
"access_token" => access_token,
"refresh_token" => refresh_token,
- "expires_at" => expires_at
+ "expires_at" => expires_at,
+ "legacy" => legacy
}) do
site = conn.assigns.site
current_user = conn.assigns.current_user
+ redirect_route =
+ if legacy == "true" do
+ Routes.site_path(conn, :settings_integrations, site.domain)
+ else
+ Routes.site_path(conn, :settings_imports_exports, site.domain)
+ end
+
{:ok, _} =
Plausible.Imported.UniversalAnalytics.new_import(
site,
@@ -777,12 +824,49 @@ defmodule PlausibleWeb.SiteController do
end_date: end_date,
access_token: access_token,
refresh_token: refresh_token,
- token_expires_at: expires_at
+ token_expires_at: expires_at,
+ legacy: legacy == "true"
)
conn
|> put_flash(:success, "Import scheduled. An email will be sent when it completes.")
- |> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain))
+ |> redirect(external: redirect_route)
+ end
+
+ def forget_import(conn, %{"import_id" => import_id}) do
+ site = conn.assigns.site
+
+ cond do
+ import_id == "0" ->
+ Plausible.Purge.delete_imported_stats!(site, 0)
+
+ site
+ |> Plausible.Site.remove_imported_data()
+ |> Repo.update!()
+
+ site_import = Plausible.Imported.get_import(import_id) ->
+ Oban.cancel_all_jobs(
+ from(j in Oban.Job,
+ where:
+ j.queue == "analytics_imports" and
+ fragment("(? ->> 'import_id')::int", j.args) == ^site_import.id
+ )
+ )
+
+ Plausible.Purge.delete_imported_stats!(site_import)
+
+ Plausible.Repo.delete!(site_import)
+
+ if site_import.legacy do
+ site
+ |> Plausible.Site.remove_imported_data()
+ |> Repo.update!()
+ end
+ end
+
+ conn
+ |> put_flash(:success, "Imported data has been cleared")
+ |> redirect(external: Routes.site_path(conn, :settings_imports_exports, site.domain))
end
def forget_imported(conn, _params) do
@@ -793,19 +877,13 @@ defmodule PlausibleWeb.SiteController do
|> Plausible.Imported.list_all_imports()
|> Enum.map(& &1.id)
- import_ids =
- if site.imported_data do
- [0 | import_ids]
- else
- import_ids
- end
-
if import_ids != [] do
Oban.cancel_all_jobs(
- from j in Oban.Job,
+ from(j in Oban.Job,
where:
j.queue == "analytics_imports" and
fragment("(? ->> 'import_id')::int", j.args) in ^import_ids
+ )
)
Plausible.Purge.delete_imported_stats!(site)
diff --git a/lib/plausible_web/live/imports_exports_settings.ex b/lib/plausible_web/live/imports_exports_settings.ex
new file mode 100644
index 0000000000..fc9d170dff
--- /dev/null
+++ b/lib/plausible_web/live/imports_exports_settings.ex
@@ -0,0 +1,144 @@
+defmodule PlausibleWeb.Live.ImportsExportsSettings do
+ @moduledoc """
+ LiveView allowing listing and deleting imports.
+ """
+ use PlausibleWeb, :live_view
+ use Phoenix.HTML
+
+ import PlausibleWeb.Components.Generic
+ import PlausibleWeb.TextHelpers
+
+ alias Plausible.Imported
+ alias Plausible.Imported.SiteImport
+ alias Plausible.Sites
+
+ require Plausible.Imported.SiteImport
+
+ def mount(
+ _params,
+ %{"domain" => domain, "current_user_id" => user_id},
+ socket
+ ) do
+ socket =
+ socket
+ |> assign_new(:site, fn ->
+ Sites.get_for_user!(user_id, domain, [:owner, :admin, :super_admin])
+ end)
+ |> assign_new(:site_imports, fn %{site: site} ->
+ site
+ |> Imported.list_all_imports()
+ |> Enum.map(&%{site_import: &1, live_status: &1.status})
+ end)
+ |> assign_new(:pageview_counts, fn %{site: site} ->
+ Plausible.Stats.Clickhouse.imported_pageview_counts(site)
+ end)
+ |> assign_new(:current_user, fn ->
+ Plausible.Repo.get(Plausible.Auth.User, user_id)
+ end)
+
+ :ok = Imported.listen()
+
+ {:ok, assign(socket, max_imports: Imported.max_complete_imports())}
+ end
+
+ def render(assigns) do
+ ~H"""
+ <header class="relative border-b border-gray-200 pb-4">
+ <h3 class="mt-8 text-md leading-6 font-medium text-gray-900 dark:text-gray-100">
+ Existing Imports
+ </h3>
+ <p class="mt-1 text-sm leading-5 text-gray-500 dark:text-gray-200">
+ A maximum of <%= @max_imports %> imports at any time is allowed.
+ </p>
+ </header>
+
+ <div
+ :if={Enum.empty?(@site_imports)}
+ class="text-gray-800 dark:text-gray-200 text-center mt-8 mb-12"
+ >
+ <p>There are no imports yet for this site.</p>
+ </div>
+ <ul :if={not Enum.empty?(@site_imports)}>
+ <li :for={entry <- @site_imports} class="py-4 flex items-center justify-between space-x-4">
+ <div class="flex flex-col">
+ <p class="text-sm leading-5 font-medium text-gray-900 dark:text-gray-100">
+ Import from <%= Plausible.Imported.SiteImport.label(entry.site_import) %>
+ <span :if={entry.live_status == SiteImport.completed()} class="text-xs font-normal">
+ (<%= Map.get(@pageview_counts, entry.site_import.id, 0) %> page views)
+ </span>
+ <Heroicons.clock
+ :if={entry.live_status == SiteImport.pending()}
+ class="inline-block h-6 w-5 text-indigo-600 dark:text-green-600"
+ />
+ <.spinner
+ :if={entry.live_status == SiteImport.importing()}
+ class="inline-block h-6 w-5 text-indigo-600 dark:text-green-600"
+ />
+ <Heroicons.check
+ :if={entry.live_status == SiteImport.completed()}
+ class="inline-block h-6 w-5 text-indigo-600 dark:text-green-600"
+ />
+ <Heroicons.exclamation_triangle
+ :if={entry.live_status == SiteImport.failed()}
+ class="inline-block h-6 w-5 text-indigo-600 dark:text-green-600"
+ />
+ </p>
+ <p class="text-sm leading-5 text-gray-500 dark:text-gray-200">
+ From <%= format_date(entry.site_import.start_date) %> to <%= format_date(
+ entry.site_import.end_date
+ ) %> (created on <%= format_date(
+ entry.site_import.inserted_at || entry.site_import.start_date
+ ) %>)
+ </p>
+ </div>
+ <.button
+ data-to={"/#{URI.encode_www_form(@site.domain)}/settings/forget-import/#{entry.site_import.id}"}
+ theme="danger"
+ data-method="delete"
+ data-csrf={Plug.CSRFProtection.get_csrf_token()}
+ class="sm:ml-3 sm:w-auto w-full"
+ data-confirm="Are you sure you want to delete this import?"
+ >
+ <span :if={entry.live_status in [SiteImport.completed(), SiteImport.failed()]}>
+ Delete Import
+ </span>
+ <span :if={entry.live_status not in [SiteImport.completed(), SiteImport.failed()]}>
+ Cancel Import
+ </span>
+ </.button>
+ </li>
+ </ul>
+ """
+ end
+
+ def handle_info({:notification, :analytics_imports_jobs, status}, socket) do
+ [{status_str, import_id}] = Enum.to_list(status)
+ {site_imports, updated?} = update_imports(socket.assigns.site_imports, import_id, status_str)
+
+ pageview_counts =
+ if updated? do
+ Plausible.Stats.Clickhouse.imported_pageview_counts(socket.assigns.site)
+ else
+ socket.assigns.pageview_counts
+ end
+
+ {:noreply, assign(socket, site_imports: site_imports, pageview_counts: pageview_counts)}
+ end
+
+ defp update_imports(site_imports, import_id, status_str) do
+ Enum.map_reduce(site_imports, false, fn
+ %{site_import: %{id: ^import_id}} = entry, _changed? ->
+ new_status =
+ case status_str do
+ "complete" -> SiteImport.completed()
+ "fail" -> SiteImport.failed()
+ "transient_fail" -> SiteImport.importing()
+ end
+
+ {%{entry | live_status: new_status}, true}
+
+ entry, changed? ->
+ {entry, changed?}
+ end)
+ end
+end
diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex
index a685701b22..d1c414ac76 100644
--- a/lib/plausible_web/router.ex
+++ b/lib/plausible_web/router.ex
@@ -360,6 +360,7 @@ defmodule PlausibleWeb.Router do
get "/:website/settings/danger-zone", SiteController, :settings_danger_zone
get "/:website/settings/integrations", SiteController, :settings_integrations
get "/:website/settings/shields/:shield", SiteController, :settings_shields
+ get "/:website/settings/imports-exports", SiteController, :settings_imports_exports
put "/:website/settings/features/visibility/:setting",
SiteController,
@@ -385,6 +386,7 @@ defmodule PlausibleWeb.Router do
get "/:website/import/google-analytics/confirm", SiteController, :import_from_google_confirm
post "/:website/settings/google-import", SiteController, :import_from_google
delete "/:website/settings/forget-imported", SiteController, :forget_imported
+ delete "/:website/settings/forget-import/:import_id", SiteController, :forget_import
get "/:domain/export", StatsController, :csv_export
get "/:domain/*path", StatsController, :stats
diff --git a/lib/plausible_web/templates/site/import_from_google_confirm.html.eex b/lib/plausible_web/templates/site/import_from_google_confirm.html.eex
index 4e1018faa3..c46459aa94 100644
--- a/lib/plausible_web/templates/site/import_from_google_confirm.html.eex
+++ b/lib/plausible_web/templates/site/import_from_google_confirm.html.eex
@@ -4,6 +4,7 @@
<%= hidden_input(f, :access_token, value: @access_token) %>
<%= hidden_input(f, :refresh_token, value: @refresh_token) %>
<%= hidden_input(f, :expires_at, value: @expires_at) %>
+ <%= hidden_input(f, :legacy, value: @legacy) %>
<%= case @start_date do %>
<% {:ok, start_date} -> %>
diff --git a/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex b/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex
index d8f69f2e5e..e5792ed9f9 100644
--- a/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex
+++ b/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex
@@ -22,5 +22,5 @@
</p>
</div>
- <%= link("Continue ->", to: Routes.site_path(@conn, :import_from_google_confirm, @site.domain, view_id: @view_id, access_token: @access_token, refresh_token: @refresh_token, expires_at: @expires_at), class: "button mt-6") %>
+ <%= link("Continue ->", to: Routes.site_path(@conn, :import_from_google_confirm, @site.domain, view_id: @view_id, access_token: @access_token, refresh_token: @refresh_token, expires_at: @expires_at, legacy: @legacy), class: "button mt-6") %>
</div>
diff --git a/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex b/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex
index 42099ed0ae..d8b8db4442 100644
--- a/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex
+++ b/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex
@@ -4,6 +4,7 @@
<%= hidden_input(f, :access_token, value: @access_token) %>
<%= hidden_input(f, :refresh_token, value: @refresh_token) %>
<%= hidden_input(f, :expires_at, value: @expires_at) %>
+ <%= hidden_input(f, :legacy, value: @legacy) %>
<div class="mt-6 text-sm text-gray-500 dark:text-gray-200">
Choose the view in your Google Analytics account that will be imported to the <%= @site.domain %> dashboard.
diff --git a/lib/plausible_web/templates/site/settings_google_import.html.heex b/lib/plausible_web/templates/site/settings_google_import.html.heex
index acca9718ca..fb223f2982 100644
--- a/lib/plausible_web/templates/site/settings_google_import.html.heex
+++ b/lib/plausible_web/templates/site/settings_google_import.html.heex
@@ -92,7 +92,7 @@
<% end %>
<PlausibleWeb.Components.Google.button
id="analytics-connect"
- to={Plausible.Google.Api.import_authorize_url(@site.id, "import")}
+ to={Plausible.Google.Api.import_authorize_url(@site.id, "import", true)}
/>
<% end %>
<% else %>
diff --git a/lib/plausible_web/templates/site/settings_imports_exports.html.heex b/lib/plausible_web/templates/site/settings_imports_exports.html.heex
new file mode 100644
index 0000000000..9369143a17
--- /dev/null
+++ b/lib/plausible_web/templates/site/settings_imports_exports.html.heex
@@ -0,0 +1,61 @@
+<div class="shadow bg-white dark:bg-gray-800 sm:rounded-md sm:overflow-hidden py-6 px-4 sm:p-6">
+ <header class="relative border-b border-gray-200 pb-4">
+ <h2 class="text-lg leading-6 font-medium text-gray-900 dark:text-gray-100">
+ Import Data
+ </h2>
+ <p class="mt-1 text-sm leading-5 text-gray-500 dark:text-gray-200">
+ Import existing data from external sources. Pick one of the options below to start a new import.
+ </p>
+
+ <PlausibleWeb.Components.Generic.docs_info slug="google-analytics-import" />
+ </header>
+
+ <div class="mt-5 flex gap-x-4">
+ <PlausibleWeb.Components.Generic.button_link
+ class="w-36 h-20"
+ theme="bright"
+ href={Plausible.Google.Api.import_authorize_url(@site.id, "import", false)}
+ >
+ <img src="/images/icon/universal_analytics_logo.svg" alt="New Universal Analytics import" />
+ </PlausibleWeb.Components.Generic.button_link>
+
+ <PlausibleWeb.Components.Generic.button_link
+ class="w-36 h-20 opacity-40 cursor-not-allowed"
+ theme="bright"
+ href=""
+ >
+ <img
+ src="/images/icon/google_analytics_4_logo.svg"
+ width="110"
+ alt="New Universal Analytics import"
+ />
+ </PlausibleWeb.Components.Generic.button_link>
+
+ <PlausibleWeb.Components.Generic.button_link
+ class="w-36 h-20 opacity-40 cursor-not-allowed"
+ theme="bright"
+ href=""
+ >
+ <img class="h-16" src="/images/icon/csv_logo.svg" alt="New CSV import" />
+ </PlausibleWeb.Components.Generic.button_link>
+ </div>
+
+ <%= live_render(@conn, PlausibleWeb.Live.ImportsExportsSettings,
+ session: %{"domain" => @site.domain}
+ ) %>
+
+ <header class="relative border-b border-gray-200 pb-4">
+ <h2 class="mt-8 text-lg leading-6 font-medium text-gray-900 dark:text-gray-100">
+ Export Data
+ </h2>
+ <p class="mt-1 text-sm leading-5 text-gray-500 dark:text-gray-200">
+ Export all your data into CSV format.
+ </p>
+ </header>
+
+ <div class="mt-4">
+ <PlausibleWeb.Components.Generic.button_link disabled href="">
+ Export to CSV
+ </PlausibleWeb.Components.Generic.button_link>
+ </div>
+</div>
diff --git a/lib/plausible_web/views/layout_view.ex b/lib/plausible_web/views/layout_view.ex
index 8bdd199bd6..0b587379d9 100644
--- a/lib/plausible_web/views/layout_view.ex
+++ b/lib/plausible_web/views/layout_view.ex
@@ -59,6 +59,9 @@ defmodule PlausibleWeb.LayoutView do
end,
%{key: "Custom Properties", value: "properties", icon: :document_text},
%{key: "Integrations", value: "integrations", icon: :arrow_path_rounded_square},
+ if FunWithFlags.enabled?(:imports_exports, for: conn.assigns.site) do
+ %{key: "Imports & Exports", value: "imports-exports", icon: :arrows_up_down}
+ end,
%{
key: "Shields",
icon: :shield_exclamation,
diff --git a/priv/repo/seeds.exs b/priv/repo/seeds.exs
index 37b0b309ea..e1b499f76c 100644
--- a/priv/repo/seeds.exs
+++ b/priv/repo/seeds.exs
@@ -10,6 +10,8 @@
# We recommend using the bang functions (`insert!`, `update!`
# and so on) as they will fail if something goes wrong.
+FunWithFlags.enable(:imports_exports)
+
user = Plausible.Factory.insert(:user, email: "[email protected]", password: "plausible")
native_stats_range =
@@ -18,9 +20,15 @@ native_stats_range =
Date.utc_today()
)
-imported_stats_range =
+legacy_imported_stats_range =
Date.range(
Date.add(native_stats_range.first, -360),
+ Date.add(native_stats_range.first, -180)
+ )
+
+imported_stats_range =
+ Date.range(
+ Date.add(native_stats_range.first, -180),
Date.add(native_stats_range.first, -1)
)
@@ -39,7 +47,7 @@ site =
Plausible.Factory.insert(:site,
domain: "dummy.site",
native_stats_start_at: NaiveDateTime.new!(native_stats_range.first, ~T[00:00:00]),
- stats_start_date: NaiveDateTime.new!(imported_stats_range.first, ~T[00:00:00]),
+ stats_start_date: NaiveDateTime.new!(legacy_imported_stats_range.first, ~T[00:00:00]),
memberships: [
Plausible.Factory.build(:site_membership, user: user, role: :owner),
Plausible.Factory.build(:site_membership,
@@ -250,13 +258,13 @@ end)
site =
site
|> Plausible.Site.start_import(
- imported_stats_range.first,
- imported_stats_range.last,
+ legacy_imported_stats_range.first,
+ legacy_imported_stats_range.last,
"Google Analytics"
)
|> Plausible.Repo.update!()
-imported_stats_range
+legacy_imported_stats_range
|> Enum.flat_map(fn date ->
Enum.flat_map(0..Enum.random(1..500), fn _ ->
[
@@ -291,3 +299,50 @@ end)
site
|> Plausible.Site.import_success()
|> Plausible.Repo.update!()
+
+site_import =
+ site
+ |> Plausible.Imported.SiteImport.create_changeset(user, %{
+ source: :universal_analytics,
+ start_date: imported_stats_range.first,
+ end_date: imported_stats_range.last,
+ legacy: false
+ })
+ |> Plausible.Imported.SiteImport.start_changeset()
+ |> Plausible.Repo.insert!()
+
+imported_stats_range
+|> Enum.flat_map(fn date ->
+ Enum.flat_map(0..Enum.random(1..500), fn _ ->
+ [
+ Plausible.Factory.build(:imported_visitors,
+ date: date,
+ pageviews: Enum.random(1..20),
+ visitors: Enum.random(1..20),
+ bounces: Enum.random(1..20),
+ visits: Enum.random(1..200),
+ visit_duration: Enum.random(1000..10000)
+ ),
+ Plausible.Factory.build(:imported_sources,
+ date: date,
+ source: Enum.random(["", "Facebook", "Twitter", "DuckDuckGo", "Google"]),
+ visitors: Enum.random(1..20),
+ visits: Enum.random(1..200),
+ bounces: Enum.random(1..20),
+ visit_duration: Enum.random(1000..10000)
+ ),
+ Plausible.Factory.build(:imported_pages,
+ date: date,
+ visitors: Enum.random(1..20),
+ pageviews: Enum.random(1..20),
+ exits: Enum.random(1..20),
+ time_on_page: Enum.random(1000..10000)
+ )
+ ]
+ end)
+end)
+|> then(&Plausible.TestUtils.populate_stats(site, site_import.id, &1))
+
+site_import
+|> Plausible.Imported.SiteImport.complete_changeset()
+|> Plausible.Repo.update!()
diff --git a/priv/static/images/icon/csv_logo.svg b/priv/static/images/icon/csv_logo.svg
new file mode 100644
index 0000000000..d36e864e5c
--- /dev/null
+++ b/priv/static/images/icon/csv_logo.svg
@@ -0,0 +1 @@
+<svg height="64" viewBox="0 0 56 64" width="56" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="m5.106 0c-2.802 0-5.073 2.272-5.073 5.074v53.841c0 2.803 2.271 5.074 5.073 5.074h45.774c2.801 0 5.074-2.271 5.074-5.074v-38.605l-18.903-20.31h-31.945z" fill="#45b058" fill-rule="evenodd"/><path d="m20.306 43.197c.126.144.198.324.198.522 0 .378-.306.72-.703.72-.18 0-.378-.072-.504-.234-.702-.846-1.891-1.387-3.007-1.387-2.629 0-4.627 2.017-4.627 4.88 0 2.845 1.999 4.879 4.627 4.879 1.134 0 2.25-.486 3.007-1.369.125-.144.324-.233.504-.233.415 0 .703.359.703.738 0 .18-.072.36-.198.504-.937.972-2.215 1.693-4.015 1.693-3.457 0-6.176-2.521-6.176-6.212s2.719-6.212 6.176-6.212c1.8.001 3.096.721 4.015 1.711zm6.802 10.714c-1.782 0-3.187-.594-4.213-1.495-.162-.144-.234-.342-.234-.54 0-.361.27-.757.702-.757.144 0 .306.036.432.144.828.739 1.98 1.314 3.367 1.314 2.143 0 2.827-1.152 2.827-2.071 0-3.097-7.112-1.386-7.112-5.672 0-1.98 1.764-3.331 4.123-3.331 1.548 0 2.881.467 3.853 1.278.162.144.252.342.252.54 0 .36-.306.72-.703.72-.144 0-.306-.054-.432-.162-.882-.72-1.98-1.044-3.079-1.044-1.44 0-2.467.774-2.467 1.909 0 2.701 7.112 1.152 7.112 5.636.001 1.748-1.187 3.531-4.428 3.531zm16.994-11.254-4.159 10.335c-.198.486-.685.81-1.188.81h-.036c-.522 0-1.008-.324-1.207-.81l-4.142-10.335c-.036-.09-.054-.18-.054-.288 0-.36.323-.793.81-.793.306 0 .594.18.72.486l3.889 9.992 3.889-9.992c.108-.288.396-.486.72-.486.468 0 .81.378.81.793.001.09-.017.198-.052.288z" fill="#fff"/><g clip-rule="evenodd" fill-rule="evenodd"><path d="m56.001 20.357v1h-12.8s-6.312-1.26-6.128-6.707c0 0 .208 5.707 6.003 5.707z" fill="#349c42"/><path d="m37.098.006v14.561c0 1.656 1.104 5.791 6.104 5.791h12.8l-18.904-20.352z" fill="#fff" opacity=".5"/></g></svg>
\ No newline at end of file
diff --git a/priv/static/images/icon/google_analytics_4_logo.svg b/priv/static/images/icon/google_analytics_4_logo.svg
new file mode 100644
index 0000000000..951af735ff
--- /dev/null
+++ b/priv/static/images/icon/google_analytics_4_logo.svg
@@ -0,0 +1 @@
+<?xml version="1.0" encoding="UTF-8"?> <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="265.361" height="128.733" viewBox="0 0 265.361 128.733"><defs><clipPath id="clip-path"><rect width="265.361" height="128.733" fill="none"></rect></clipPath></defs><g id="Repeat_Grid_1" data-name="Repeat Grid 1" clip-path="url(#clip-path)"><g id="Group_51" data-name="Group 51" transform="translate(-595.244 -549.85)"><rect id="Rectangle_3" data-name="Rectangle 3" width="29.944" height="124.055" rx="14.972" transform="translate(673.313 549.85)" fill="#ffa800"></rect><rect id="Rectangle_4" data-name="Rectangle 4" width="29.944" height="77.689" rx="14.972" transform="translate(633.922 596.215)" fill="#f46d01"></rect><circle id="Ellipse_2" data-name="Ellipse 2" cx="14.616" cy="14.616" r="14.616" transform="translate(595.244 644.673)" fill="#f46d01"></circle><path id="Path_12" data-name="Path 12" d="M18.259-10.974H11.377V-8.8h4.247v6.231a8.8,8.8,0,0,1-3.6.558c-4.867,0-8.06-3.131-8.06-8.432,0-5.239,3.317-8.4,8.4-8.4a10.83,10.83,0,0,1,4.619.9l.651-2.2a12.917,12.917,0,0,0-5.208-.961c-7.006,0-11.284,4.557-11.315,10.788A10.749,10.749,0,0,0,4.03-2.542,10.74,10.74,0,0,0,11.842.217,19.465,19.465,0,0,0,18.259-.93ZM28.8-15.345c-4.154,0-7.44,2.945-7.44,7.967,0,4.743,3.131,7.719,7.192,7.719,3.627,0,7.471-2.418,7.471-7.967C36.022-12.214,33.108-15.345,28.8-15.345ZM28.737-13.3c3.224,0,4.5,3.224,4.5,5.766,0,3.379-1.953,5.828-4.557,5.828-2.666,0-4.557-2.48-4.557-5.766C24.118-10.323,25.513-13.3,28.737-13.3Zm17.081-2.046c-4.154,0-7.44,2.945-7.44,7.967,0,4.743,3.131,7.719,7.192,7.719,3.627,0,7.471-2.418,7.471-7.967C53.041-12.214,50.127-15.345,45.818-15.345ZM45.756-13.3c3.224,0,4.495,3.224,4.495,5.766,0,3.379-1.953,5.828-4.557,5.828-2.666,0-4.557-2.48-4.557-5.766C41.137-10.323,42.532-13.3,45.756-13.3ZM67.022-15,66.9-12.741h-.062a4.99,4.99,0,0,0-4.681-2.6c-3.441,0-6.758,2.883-6.758,7.967,0,4.154,2.666,7.316,6.386,7.316a5.356,5.356,0,0,0,4.774-2.511h.062V-.9c0,3.782-2.046,5.239-4.836,5.239a8.461,8.461,0,0,1-4.4-1.178L56.7,5.239a9.684,9.684,0,0,0,4.991,1.24,7.954,7.954,0,0,0,5.456-1.8c1.457-1.3,2.139-3.41,2.139-6.851v-8.711c0-1.829.062-3.038.124-4.123Zm-.434,8.618a4.746,4.746,0,0,1-.217,1.457,3.988,3.988,0,0,1-3.782,2.79c-2.945,0-4.433-2.449-4.433-5.394,0-3.472,1.86-5.673,4.464-5.673a3.93,3.93,0,0,1,3.813,2.883,3.885,3.885,0,0,1,.155,1.24ZM73.811,0h2.728V-22.01H73.811ZM93.186-7.006a10.289,10.289,0,0,0,.093-1.271c0-2.759-1.3-7.068-6.2-7.068-4.371,0-7.037,3.565-7.037,8.091S82.8.31,87.42.31a12.153,12.153,0,0,0,4.991-.93l-.465-1.953a9.865,9.865,0,0,1-4.154.775c-2.728,0-5.084-1.519-5.146-5.208ZM82.677-8.959c.217-1.891,1.426-4.433,4.185-4.433,3.069,0,3.813,2.7,3.782,4.433ZM13.144,21.428,15.376,28h2.883L11.129,7.106H7.874L.775,28h2.79l2.17-6.572ZM6.293,19.32l2.046-6.045c.4-1.271.744-2.542,1.054-3.782h.062c.31,1.209.62,2.449,1.085,3.813l2.046,6.014ZM21.142,28H23.87V18.979a3.922,3.922,0,0,1,.186-1.271,3.876,3.876,0,0,1,3.658-2.79c2.573,0,3.472,2.015,3.472,4.433V28h2.728V19.041c0-5.146-3.224-6.386-5.3-6.386a5.517,5.517,0,0,0-4.96,2.821h-.062L23.436,13H21.018c.093,1.24.124,2.511.124,4.061Zm27.993,0a21.125,21.125,0,0,1-.248-3.6V18.793c0-3.007-1.116-6.138-5.7-6.138a9.312,9.312,0,0,0-4.929,1.333l.62,1.8A7.476,7.476,0,0,1,42.78,14.67c3.069,0,3.41,2.232,3.41,3.472v.31c-5.8-.031-9.021,1.953-9.021,5.58a4.275,4.275,0,0,0,4.588,4.309,5.588,5.588,0,0,0,4.588-2.232h.093L46.655,28Zm-2.883-5.053a2.847,2.847,0,0,1-.155.868,3.733,3.733,0,0,1-3.627,2.511,2.414,2.414,0,0,1-2.573-2.6c0-2.914,3.379-3.441,6.355-3.379ZM53.289,28h2.728V5.99H53.289Zm5.332-15L64.17,26.822a2.379,2.379,0,0,1,.186.713,1.958,1.958,0,0,1-.217.651,8.271,8.271,0,0,1-2.294,3.038,7.216,7.216,0,0,1-2.387,1.333l.682,2.294a7.392,7.392,0,0,0,3.348-1.767c1.86-1.612,3.193-4.247,5.146-9.393L72.726,13H69.843l-2.976,8.8c-.372,1.085-.682,2.232-.961,3.131h-.062c-.248-.9-.62-2.077-.961-3.069L61.6,13ZM75.826,9.4V13H73.5v2.077h2.325v8.184c0,1.767.279,3.1,1.054,3.906a3.78,3.78,0,0,0,2.945,1.147,6.678,6.678,0,0,0,2.418-.372l-.124-2.046a6.069,6.069,0,0,1-1.581.186c-1.519,0-2.046-1.054-2.046-2.914V15.073H82.4V13H78.492V8.687ZM88.195,28V13H85.467V28ZM86.831,7.075A1.662,1.662,0,0,0,85.126,8.78a1.69,1.69,0,0,0,3.379,0A1.619,1.619,0,0,0,86.831,7.075ZM102.951,25.4a7.75,7.75,0,0,1-3.348.744c-2.976,0-5.208-2.139-5.208-5.611,0-3.131,1.86-5.673,5.3-5.673a6.326,6.326,0,0,1,3.162.713l.62-2.108a9.119,9.119,0,0,0-3.782-.775,7.624,7.624,0,0,0-8.06,7.967c0,4.588,2.945,7.657,7.471,7.657a10.048,10.048,0,0,0,4.309-.868Zm2.6,1.891a8.557,8.557,0,0,0,4.216,1.023c3.565,0,5.611-1.86,5.611-4.495,0-2.232-1.333-3.534-3.937-4.526-1.953-.744-2.852-1.3-2.852-2.542,0-1.116.9-2.046,2.511-2.046a5.839,5.839,0,0,1,3.069.868l.682-1.984a7.489,7.489,0,0,0-3.689-.93c-3.224,0-5.177,1.984-5.177,4.4,0,1.8,1.271,3.286,3.968,4.247,2.015.744,2.79,1.457,2.79,2.759,0,1.24-.93,2.232-2.914,2.232a7.149,7.149,0,0,1-3.6-1.085ZM135.594,28V22.513h2.821V20.374h-2.821V7.85h-2.945l-8.99,12.865v1.8h9.362V28Zm-9.176-7.626v-.062l4.929-6.851c.558-.93,1.054-1.8,1.674-2.976h.093q-.093,1.581-.093,3.162v6.727Z" transform="translate(722.19 643.732)" fill="gray"></path></g></g></svg>
\ No newline at end of file
diff --git a/priv/static/images/icon/universal_analytics_logo.svg b/priv/static/images/icon/universal_analytics_logo.svg
new file mode 100644
index 0000000000..5ce2f3df3e
--- /dev/null
+++ b/priv/static/images/icon/universal_analytics_logo.svg
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="26 -29 120 60" width="120" height="60"><linearGradient id="A" gradientUnits="userSpaceOnUse" x1="173.867" y1="-72.65" x2="216.749" y2="-72.65"><stop offset="0" stop-color="#e96f0b"/><stop offset="1" stop-color="#f37901"/></linearGradient><g fill-rule="evenodd"><path d="M47.33 16H36.226c-1.364 0-2.468-1.104-2.468-2.468V8.208c0-1.364 1.104-2.468 2.468-2.468h7.013v-7.792c0-1.364 1.104-2.468 2.468-2.468H53.5v20.455h-6.17z" fill="#ffc517"/><path d="M61.3 16h-7.8v-27.532c0-1.364 1.104-2.468 2.468-2.468h5.325c1.364 0 2.468 1.104 2.468 2.468v25.065C63.693 14.896 62.6 16 61.3 16z" fill="#f57e02"/><path d="M216.708-72.65v32.646c0 5.713-4.625 10.338-10.338 10.338h-32.646v-85.968l42.984 42.984z" fill="url(#A)" transform="matrix(.238687 0 0 .238687 11.968065 23.081216)"/></g><g fill="#757575"><path d="M74.093 7.228l2.388 6.27h-4.627l2.24-6.27zm-.896-2.1l-5.224 13.73h1.94l1.343-3.73h5.82l1.343 3.73h1.94l-5.224-13.73h-1.94zm28.2 13.742h1.8V5.138h-1.8v13.73zm-18.35-8.07c.597-.896 1.8-1.642 2.985-1.642 2.388 0 3.582 1.642 3.582 4.03v5.82h-1.8v-5.522c0-1.94-1.045-2.687-2.388-2.687-1.493 0-2.537 1.493-2.537 2.836v5.224h-1.8V9.32h1.8l.15 1.493zm8.06 5.234c0-2.1 1.94-3.284 4.18-3.284 1.343 0 2.24.3 2.537.597v-.3c0-1.493-1.194-2.24-2.388-2.24-1.045 0-1.94.448-2.24 1.343l-1.642-.746c.3-.896 1.493-2.24 3.88-2.24 2.24 0 4.18 1.343 4.18 4.03v5.672h-1.642v-1.343h-.15c-.448.746-1.493 1.642-3.134 1.642-1.94 0-3.582-1.194-3.582-3.134m6.716-1.194s-.746-.597-2.24-.597c-1.8 0-2.537 1.045-2.537 1.8 0 1.045 1.045 1.493 1.94 1.493 1.343 0 2.836-1.194 2.836-2.687"/><path d="M105.735 23.05l2.1-4.925-3.73-8.657h1.8l2.836 6.567 2.836-6.567h1.8l-5.82 13.582h-1.8zm26.27-10.45c-.448-1.194-1.493-1.94-2.537-1.94-1.493 0-2.836 1.343-2.836 3.284s1.343 3.284 2.836 3.284c1.045 0 2.1-.746 2.537-1.8l1.493.896c-.746 1.642-2.24 2.687-4.03 2.687-2.537 0-4.627-2.24-4.627-5.075 0-2.985 2.1-5.075 4.627-5.075 1.8 0 3.284 1.045 4.03 2.687l-1.493 1.045z" fill-rule="evenodd"/><path d="M138.422 19.168c2.388 0 3.582-1.343 3.582-2.985 0-3.582-5.224-2.24-5.224-4.328 0-.746.597-1.194 1.642-1.194s2.1.448 2.537 1.194l1.045-1.045c-.597-.746-2.24-1.8-3.73-1.8-2.24 0-3.433 1.343-3.433 2.985 0 3.433 5.373 2.24 5.373 4.03 0 .896-.597 1.493-1.8 1.493s-1.8-.746-2.388-1.493L134.7 17.08c.896.896 2.24 2.1 3.73 2.1zm-17.015-.298h1.8V9.318h-1.8v9.552z"/><path d="M122.153 4.7c.746 0 1.194.597 1.194 1.194 0 .746-.597 1.194-1.194 1.194s-1.194-.597-1.194-1.194.597-1.194 1.194-1.194zm-2.537 12.687l.3 1.343h-1.8c-1.642 0-2.388-1.194-2.388-2.985V11.1h-1.8V9.3h1.8V6.482h1.8v2.836h2.1v1.8h-2.1v4.925c0 1.343 1.045 1.343 2.1 1.343z" fill-rule="evenodd"/><path d="M77.048-10.042v2.24h5.224c-.15 1.194-.597 2.1-1.194 2.687-.746.746-1.94 1.642-4.03 1.642-3.134 0-5.672-2.537-5.672-5.82 0-3.134 2.537-5.82 5.672-5.82 1.8 0 2.985.746 3.88 1.493l1.493-1.493c-1.343-1.194-2.985-2.24-5.373-2.24-4.328 0-8.06 3.582-8.06 7.9s3.73 7.9 8.06 7.9c2.388 0 4.18-.746 5.522-2.24 1.493-1.493 1.94-3.433 1.94-5.075 0-.448 0-1.045-.15-1.343h-7.313zm13.282-1.8c-2.836 0-5.075 2.1-5.075 5.075s2.24 5.075 5.075 5.075a5.03 5.03 0 0 0 5.075-5.075c0-2.985-2.24-5.075-5.075-5.075zm0 8.2c-1.493 0-2.836-1.343-2.836-3.134S88.837-9.9 90.33-9.9s2.837 1.2 2.837 3.13-1.343 3.134-2.836 3.134zm24.47-7.156c-.597-.597-1.493-1.194-2.836-1.194-2.537 0-4.776 2.24-4.776 5.075s2.24 5.075 4.776 5.075c1.194 0 2.24-.597 2.687-1.194h.15v.746c0 1.94-1.045 2.985-2.687 2.985-1.343 0-2.24-1.045-2.537-1.8l-1.94.746c.597 1.343 2.1 2.985 4.478 2.985 2.687 0 4.925-1.493 4.925-5.373V-12h-2.1v1.194zm-2.537 7.164c-1.493 0-2.687-1.343-2.687-3.134s1.194-3.134 2.687-3.134 2.687 1.343 2.687 3.134-1.194 3.134-2.687 3.134zm-10.897-8.2c-2.836 0-5.075 2.1-5.075 5.075s2.24 5.075 5.075 5.075a5.03 5.03 0 0 0 5.075-5.075c.15-2.985-2.24-5.075-5.075-5.075zm0 8.2c-1.493 0-2.836-1.343-2.836-3.134s1.343-3.134 2.836-3.134 2.836 1.194 2.836 3.134-1.343 3.134-2.836 3.134zM118.7-17.206h2.24v15.522h-2.24v-15.522zm8.358 13.582c-1.194 0-1.94-.597-2.537-1.493l6.866-2.836-.3-.597c-.448-1.194-1.8-3.284-4.328-3.284-2.687 0-4.776 2.1-4.776 5.075a5.03 5.03 0 0 0 5.075 5.075c2.388 0 3.73-1.493 4.328-2.24l-1.8-1.194c-.597.896-1.343 1.493-2.537 1.493zm-.15-6.27c.896 0 1.642.448 1.94 1.194l-4.627 1.94c0-2.24 1.493-3.134 2.687-3.134z"/></g></svg>
\ No newline at end of file
diff --git a/test/plausible/imported/universal_analytics_test.exs b/test/plausible/imported/universal_analytics_test.exs
index 80ad9caab6..2d878fbc69 100644
--- a/test/plausible/imported/universal_analytics_test.exs
+++ b/test/plausible/imported/universal_analytics_test.exs
@@ -20,7 +20,8 @@ defmodule Plausible.Imported.UniversalAnalyticsTest do
end_date: "2024-01-02",
access_token: "access123",
refresh_token: "refresh123",
- token_expires_at: expires_at
+ token_expires_at: expires_at,
+ legacy: true
)
assert %Oban.Job{
@@ -42,7 +43,8 @@ defmodule Plausible.Imported.UniversalAnalyticsTest do
source: :universal_analytics,
start_date: ~D[2023-10-01],
end_date: ~D[2024-01-02],
- status: SiteImport.pending()
+ status: SiteImport.pending(),
+ legacy: true
}
] = Plausible.Imported.list_all_imports(site)
@@ -61,6 +63,35 @@ defmodule Plausible.Imported.UniversalAnalyticsTest do
assert opts[:date_range] == Date.range(~D[2023-10-01], ~D[2024-01-02])
assert opts[:auth] == {"access123", "refresh123", expires_at}
end
+
+ test "creates SiteImport with legacy flag set to false when instructed", %{
+ user: user,
+ site: site
+ } do
+ expires_at = NaiveDateTime.to_iso8601(NaiveDateTime.utc_now())
+
+ assert {:ok, job} =
+ UniversalAnalytics.new_import(site, user,
+ view_id: 123,
+ start_date: "2023-10-01",
+ end_date: "2024-01-02",
+ access_token: "access123",
+ refresh_token: "refresh123",
+ token_expires_at: expires_at,
+ legacy: false
+ )
+
+ assert %Oban.Job{args: %{"import_id" => import_id}} = Repo.reload!(job)
+
+ assert [
+ %{
+ id: ^import_id,
+ legacy: false
+ }
+ ] = Plausible.Imported.list_all_imports(site)
+
+ assert %{imported_data: nil} = Repo.reload!(site)
+ end
end
describe "import_data/2" do
diff --git a/test/plausible/imported_test.exs b/test/plausible/imported_test.exs
new file mode 100644
index 0000000000..ee98f5c37e
--- /dev/null
+++ b/test/plausible/imported_test.exs
@@ -0,0 +1,62 @@
+defmodule Plausible.ImportedTest do
+ use Plausible.DataCase
+ use Plausible
+
+ alias Plausible.Imported
+
+ describe "list_all_imports/1" do
+ test "returns empty when there are no imports" do
+ site = insert(:site)
+
+ assert Imported.list_all_imports(site) == []
+ end
+
+ test "returns imports in various states" do
+ site = insert(:site)
+
+ _rogue_import = insert(:site_import)
+
+ import1 = insert(:site_import, site: site, status: :pending)
+ import2 = insert(:site_import, site: site, status: :importing)
+ import3 = insert(:site_import, site: site, status: :completed)
+ import4 = insert(:site_import, site: site, status: :failed)
+
+ assert [%{id: id1}, %{id: id2}, %{id: id3}, %{id: id4}] = Imported.list_all_imports(site)
+
+ ids = [id1, id2, id3, id4]
+
+ assert import1.id in ids
+ assert import2.id in ids
+ assert import3.id in ids
+ assert import4.id in ids
+ end
+
+ test "returns one legacy import when present with respective site import entry" do
+ site = insert(:site)
+ {:ok, opts} = add_imported_data(%{site: site})
+ site = Map.new(opts).site
+ site_import = insert(:site_import, site: site, legacy: true)
+ site_import_id = site_import.id
+
+ assert [%{id: ^site_import_id}] = Imported.list_all_imports(site)
+ end
+
+ test "returns legacy import without respective site import entry" do
+ site = insert(:site)
+ {:ok, opts} = add_imported_data(%{site: site})
+ site = Map.new(opts).site
+ imported_start_date = site.imported_data.start_date
+ imported_end_date = site.imported_data.end_date
+
+ assert [
+ %{
+ id: 0,
+ source: :universal_analytics,
+ start_date: ^imported_start_date,
+ end_date: ^imported_end_date,
+ status: :completed
+ }
+ ] = Imported.list_all_imports(site)
+ end
+ end
+end
diff --git a/test/plausible/purge_test.exs b/test/plausible/purge_test.exs
index edb7d27fec..a06ba9061f 100644
--- a/test/plausible/purge_test.exs
+++ b/test/plausible/purge_test.exs
@@ -89,6 +89,69 @@ defmodule Plausible.PurgeTest do
end)
end
+ test "delete_imported_stats!/2 deletes legacy imported data only when instructed", %{
+ site: site,
+ site_import1: site_import1,
+ site_import2: site_import2
+ } do
+ populate_stats(site, [
+ build(:pageview),
+ build(:imported_visitors),
+ build(:imported_sources),
+ build(:imported_pages),
+ build(:imported_entry_pages),
+ build(:imported_exit_pages),
+ build(:imported_locations),
+ build(:imported_devices),
+ build(:imported_browsers),
+ build(:imported_operating_systems)
+ ])
+
+ Enum.each(Plausible.Imported.tables(), fn table ->
+ query =
+ from(imported in table,
+ where: imported.site_id == ^site.id and imported.import_id == ^site_import1.id
+ )
+
+ assert await_clickhouse_count(query, 1)
+
+ query =
+ from(imported in table,
+ where: imported.site_id == ^site.id and imported.import_id == ^site_import2.id
+ )
+
+ assert await_clickhouse_count(query, 1)
+
+ query =
+ from(imported in table, where: imported.site_id == ^site.id and imported.import_id == 0)
+
+ assert await_clickhouse_count(query, 1)
+ end)
+
+ assert :ok == Plausible.Purge.delete_imported_stats!(site, 0)
+
+ Enum.each(Plausible.Imported.tables(), fn table ->
+ query =
+ from(imported in table,
+ where: imported.site_id == ^site.id and imported.import_id == ^site_import1.id
+ )
+
+ assert await_clickhouse_count(query, 1)
+
+ query =
+ from(imported in table,
+ where: imported.site_id == ^site.id and imported.import_id == ^site_import2.id
+ )
+
+ assert await_clickhouse_count(query, 1)
+
+ query =
+ from(imported in table, where: imported.site_id == ^site.id and imported.import_id == 0)
+
+ assert await_clickhouse_count(query, 0)
+ end)
+ end
+
test "delete_imported_stats!/1 resets stats_start_date", %{site: site} do
assert :ok == Plausible.Purge.delete_imported_stats!(site)
assert %Plausible.Site{stats_start_date: nil} = Plausible.Repo.reload(site)
diff --git a/test/plausible/site/sites_test.exs b/test/plausible/site/sites_test.exs
index 4898dbfcfe..e623ffdc57 100644
--- a/test/plausible/site/sites_test.exs
+++ b/test/plausible/site/sites_test.exs
@@ -46,7 +46,7 @@ defmodule Plausible.SitesTest do
assert Sites.stats_start_date(site) == nil
end
- test "is date if first pageview if site does have stats" do
+ test "is date if site does have stats" do
site = insert(:site)
populate_stats(site, [
@@ -70,6 +70,33 @@ defmodule Plausible.SitesTest do
end
end
+ describe "native_stats_start_date" do
+ test "is nil if site has no stats" do
+ site = insert(:site)
+
+ assert Sites.native_stats_start_date(site) == nil
+ end
+
+ test "is date if site does have stats" do
+ site = insert(:site)
+
+ populate_stats(site, [
+ build(:pageview)
+ ])
+
+ assert Sites.native_stats_start_date(site) == Timex.today(site.timezone)
+ end
+
+ test "ignores imported stats" do
+ site = insert(:site)
+ insert(:site_import, site: site)
+ {:ok, opts} = add_imported_data(%{site: site})
+ site = Map.new(opts).site
+
+ assert Sites.native_stats_start_date(site) == nil
+ end
+ end
+
describe "has_stats?" do
test "is false if site has no stats" do
site = insert(:site)
diff --git a/test/plausible/stats/clickhouse_test.exs b/test/plausible/stats/clickhouse_test.exs
index 3e36dbd7a1..3b553a4c00 100644
--- a/test/plausible/stats/clickhouse_test.exs
+++ b/test/plausible/stats/clickhouse_test.exs
@@ -254,4 +254,38 @@ defmodule Plausible.Stats.ClickhouseTest do
] = Clickhouse.top_sources_for_spike(site, query, 5, 1)
end
end
+
+ describe "imported_pageview_counts/1" do
+ test "gets pageview counts for each of sites' imports" do
+ site = insert(:site)
+ {:ok, opts} = add_imported_data(%{site: site})
+ site = Map.new(opts).site
+
+ import1 = insert(:site_import, site: site)
+ import2 = insert(:site_import, site: site)
+
+ # legacy import
+ populate_stats(site, [
+ build(:imported_visitors, pageviews: 5),
+ build(:imported_visitors, pageviews: 6)
+ ])
+
+ populate_stats(site, import1.id, [
+ build(:imported_visitors, pageviews: 6),
+ build(:imported_visitors, pageviews: 8)
+ ])
+
+ populate_stats(site, import2.id, [
+ build(:imported_visitors, pageviews: 7),
+ build(:imported_visitors, pageviews: 13)
+ ])
+
+ pageview_counts = Clickhouse.imported_pageview_counts(site)
+
+ assert map_size(pageview_counts) == 3
+ assert pageview_counts[0] == 11
+ assert pageview_counts[import1.id] == 14
+ assert pageview_counts[import2.id] == 20
+ end
+ end
end
diff --git a/test/plausible/imported/imported_test.exs b/test/plausible_web/controllers/api/stats_controller/imported_test.exs
similarity index 99%
rename from test/plausible/imported/imported_test.exs
rename to test/plausible_web/controllers/api/stats_controller/imported_test.exs
index 5084407a08..027db390c2 100644
--- a/test/plausible/imported/imported_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/imported_test.exs
@@ -1,4 +1,4 @@
-defmodule Plausible.ImportedTest do
+defmodule PlausibleWeb.Api.StatsController.ImportedTest do
use PlausibleWeb.ConnCase
use Timex
diff --git a/test/plausible_web/controllers/auth_controller_test.exs b/test/plausible_web/controllers/auth_controller_test.exs
index cb3d4cc1a5..461ccf2e1d 100644
--- a/test/plausible_web/controllers/auth_controller_test.exs
+++ b/test/plausible_web/controllers/auth_controller_test.exs
@@ -1483,7 +1483,8 @@ defmodule PlausibleWeb.AuthControllerTest do
callback_params = %{"error" => "access_denied", "state" => "[#{site.id},\"import\"]"}
conn = get(conn, Routes.auth_path(conn, :google_auth_callback), callback_params)
- assert redirected_to(conn, 302) == Routes.site_path(conn, :settings_general, site.domain)
+ assert redirected_to(conn, 302) ==
+ Routes.site_path(conn, :settings_integrations, site.domain)
assert Phoenix.Flash.get(conn.assigns.flash, :error) =~
"unable to authenticate your Google Analytics"
diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exs
index 538929ac88..c580079ccb 100644
--- a/test/plausible_web/controllers/site_controller_test.exs
+++ b/test/plausible_web/controllers/site_controller_test.exs
@@ -636,6 +636,46 @@ defmodule PlausibleWeb.SiteControllerTest do
end
end
+ describe "GET /:website/settings/imports-exports" do
+ setup [:create_user, :log_in, :create_site]
+
+ test "renders empty imports list", %{conn: conn, site: site} do
+ conn = get(conn, "/#{site.domain}/settings/imports-exports")
+ resp = html_response(conn, 200)
+
+ assert text_of_attr(resp, ~s|a[href]|, "href") =~
+ "https://accounts.google.com/o/oauth2/"
+
+ assert resp =~ "Import Data"
+ assert resp =~ "Existing Imports"
+ assert resp =~ "There are no imports yet"
+ assert resp =~ "Export Data"
+ end
+
+ test "renders imports in import list", %{conn: conn, site: site} do
+ {:ok, opts} = add_imported_data(%{site: site})
+ site = Map.new(opts).site
+
+ _site_import1 = insert(:site_import, site: site, status: SiteImport.pending())
+ _site_import2 = insert(:site_import, site: site, status: SiteImport.importing())
+ site_import3 = insert(:site_import, site: site, status: SiteImport.completed())
+ _site_import4 = insert(:site_import, site: site, status: SiteImport.failed())
+
+ populate_stats(site, site_import3.id, [
+ build(:imported_visitors, pageviews: 77),
+ build(:imported_visitors, pageviews: 21)
+ ])
+
+ conn = get(conn, "/#{site.domain}/settings/imports-exports")
+ resp = html_response(conn, 200)
+
+ buttons = find(resp, ~s|button[data-method="delete"]|)
+ assert length(buttons) == 5
+
+ assert resp =~ "(98 page views)"
+ end
+ end
+
describe "GET /:website/settings/integrations for self-hosting" do
setup [:create_user, :log_in, :create_site]
@@ -1252,7 +1292,8 @@ defmodule PlausibleWeb.SiteControllerTest do
|> get("/#{site.domain}/import/google-analytics/view-id", %{
"access_token" => "token",
"refresh_token" => "foo",
- "expires_at" => "2022-09-22T20:01:37.112777"
+ "expires_at" => "2022-09-22T20:01:37.112777",
+ "legacy" => "true"
})
|> html_response(200)
@@ -1271,7 +1312,8 @@ defmodule PlausibleWeb.SiteControllerTest do
"end_date" => "2022-03-01",
"access_token" => "token",
"refresh_token" => "foo",
- "expires_at" => "2022-09-22T20:01:37.112777"
+ "expires_at" => "2022-09-22T20:01:37.112777",
+ "legacy" => "true"
})
[site_import] = Plausible.Imported.list_all_imports(site)
@@ -1288,10 +1330,11 @@ defmodule PlausibleWeb.SiteControllerTest do
"end_date" => "2022-03-01",
"access_token" => "token",
"refresh_token" => "foo",
- "expires_at" => "2022-09-22T20:01:37.112777"
+ "expires_at" => "2022-09-22T20:01:37.112777",
+ "legacy" => "true"
})
- assert [%{id: import_id}] = Plausible.Imported.list_all_imports(site)
+ assert [%{id: import_id, legacy: true}] = Plausible.Imported.list_all_imports(site)
assert_enqueued(
worker: Plausible.Workers.ImportAnalytics,
@@ -1308,8 +1351,81 @@ defmodule PlausibleWeb.SiteControllerTest do
end
end
- describe "DELETE /:website/settings/:forget_imported" do
- setup [:create_user, :log_in, :create_new_site]
+ describe "DELETE /:website/settings/:forget_import/:import_id" do
+ setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+
+ test "removes site import, associated data and cancels oban job for a particular import", %{
+ conn: conn,
+ user: user,
+ site: site
+ } do
+ {:ok, job} =
+ Plausible.Imported.NoopImporter.new_import(
+ site,
+ user,
+ start_date: ~D[2022-01-01],
+ end_date: Timex.today()
+ )
+
+ %{args: %{import_id: import_id}} = job
+
+ # legacy stats
+ populate_stats(site, [
+ build(:imported_visitors, pageviews: 12)
+ ])
+
+ populate_stats(site, import_id, [
+ build(:imported_visitors, pageviews: 10)
+ ])
+
+ assert [%{id: ^import_id}, %{id: 0}] = Plausible.Imported.list_all_imports(site)
+
+ assert eventually(fn ->
+ count = Plausible.Stats.Clickhouse.imported_pageview_count(site)
+ {count == 22, count}
+ end)
+
+ delete(conn, "/#{site.domain}/settings/forget-import/#{import_id}")
+
+ assert eventually(fn ->
+ count = Plausible.Stats.Clickhouse.imported_pageview_count(site)
+ {count == 12, count}
+ end)
+
+ assert Repo.reload(job).state == "cancelled"
+ end
+
+ test "removes legacy site import along with associated data when instructed", %{
+ conn: conn,
+ site: site
+ } do
+ other_site_import = insert(:site_import, site: site)
+
+ # legacy stats
+ populate_stats(site, [
+ build(:imported_visitors, pageviews: 12)
+ ])
+
+ populate_stats(site, other_site_import.id, [
+ build(:imported_visitors, pageviews: 10)
+ ])
+
+ assert eventually(fn ->
+ count = Plausible.Stats.Clickhouse.imported_pageview_count(site)
+ {count == 22, count}
+ end)
+
+ delete(conn, "/#{site.domain}/settings/forget-import/0")
+
+ assert eventually(fn ->
+ count = Plausible.Stats.Clickhouse.imported_pageview_count(site)
+ {count == 10, count}
+ end)
+ end
+ end
+
+ describe "DELETE /:website/settings/forget_imported" do
+ setup [:create_user, :log_in, :create_new_site, :add_imported_data]
test "removes imported_data field from site", %{conn: conn, site: site} do
delete(conn, "/#{site.domain}/settings/forget-imported")
diff --git a/test/support/factory.ex b/test/support/factory.ex
index a89478603e..afe3182dfa 100644
--- a/test/support/factory.ex
+++ b/test/support/factory.ex
@@ -50,6 +50,20 @@ defmodule Plausible.Factory do
}
end
+ def site_import_factory do
+ today = Date.utc_today()
+
+ %Plausible.Imported.SiteImport{
+ site: build(:site),
+ imported_by: build(:user),
+ start_date: Date.add(today, -200),
+ end_date: today,
+ source: :universal_analytics,
+ status: :completed,
+ legacy: false
+ }
+ end
+
def ch_session_factory do
hostname = sequence(:domain, &"example-#{&1}.com")
diff --git a/test/test_helper.exs b/test/test_helper.exs
index 8d436f5a91..75e4312452 100644
--- a/test/test_helper.exs
+++ b/test/test_helper.exs
@@ -2,6 +2,7 @@
Mox.defmock(Plausible.HTTPClient.Mock, for: Plausible.HTTPClient.Interface)
Application.ensure_all_started(:double)
FunWithFlags.enable(:window_time_on_page)
+FunWithFlags.enable(:imports_exports)
Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual)
if Mix.env() == :small_test do
diff --git a/test/workers/import_analytics_test.exs b/test/workers/import_analytics_test.exs
index fe7f1ad392..7ce1cce45a 100644
--- a/test/workers/import_analytics_test.exs
+++ b/test/workers/import_analytics_test.exs
@@ -229,5 +229,26 @@ defmodule Plausible.Workers.ImportAnalyticsTest do
assert_receive {:notification, :analytics_imports_jobs, %{"transient_fail" => ^import_id}}
end)
end
+
+ test "sends oban notification to calling process on completion when listener setup separately",
+ %{
+ import_opts: import_opts
+ } do
+ Ecto.Adapters.SQL.Sandbox.unboxed_run(Plausible.Repo, fn ->
+ user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))
+ site = insert(:site, members: [user])
+
+ {:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)
+ import_id = job.args[:import_id]
+
+ :ok = Plausible.Imported.Importer.listen()
+
+ job
+ |> Repo.reload!()
+ |> ImportAnalytics.perform()
+
+ assert_receive {:notification, :analytics_imports_jobs, %{"complete" => ^import_id}}
+ end)
+ end
end
end