Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Implement UI for multiple imports (#3727)
Adrian Gruntkowski39aa81a16f6e
diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.exindex 24e6771a3a..085513886d 100644--- a/lib/plausible/google/api.ex+++ b/lib/plausible/google/api.ex@@ -17,9 +17,9 @@ defmodule Plausible.Google.Api doJason.encode!([site_id, redirect_to])end- def import_authorize_url(site_id, redirect_to) do+ def import_authorize_url(site_id, redirect_to, legacy \\ true) do"https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@import_scope}&state=" <>- Jason.encode!([site_id, redirect_to])+ Jason.encode!([site_id, redirect_to, legacy])enddef fetch_verified_properties(auth) dodiff --git a/lib/plausible/imported.ex b/lib/plausible/imported.exindex f141780288..18058f6ec0 100644--- a/lib/plausible/imported.ex+++ b/lib/plausible/imported.ex@@ -40,6 +40,9 @@ defmodule Plausible.Imported do@spec tables() :: [String.t()]def tables, do: @table_names+ @spec max_complete_imports() :: non_neg_integer()+ def max_complete_imports(), do: @max_complete_imports+@spec load_import_data(Site.t()) :: Site.t()def load_import_data(%{import_data_loaded: true} = site), do: site@@ -56,10 +59,24 @@ defmodule Plausible.Imported do}end+ @spec get_import(non_neg_integer()) :: SiteImport.t() | nil+ def get_import(import_id) do+ Repo.get(SiteImport, import_id)+ end++ defdelegate listen(), to: Imported.Importer+@spec list_all_imports(Site.t()) :: [SiteImport.t()]def list_all_imports(site) do- from(i in SiteImport, where: i.site_id == ^site.id)- |> Repo.all()+ imports =+ from(i in SiteImport, where: i.site_id == ^site.id, order_by: [desc: i.inserted_at])+ |> Repo.all()++ if site.imported_data && not Enum.any?(imports, & &1.legacy) do+ imports ++ [SiteImport.from_legacy(site.imported_data)]+ else+ imports+ endend@spec list_complete_import_ids(Site.t()) :: [non_neg_integer()]diff --git a/lib/plausible/imported/importer.ex b/lib/plausible/imported/importer.exindex 60cebea1c1..d196bb678b 100644--- a/lib/plausible/imported/importer.ex+++ b/lib/plausible/imported/importer.ex@@ -155,7 +155,7 @@ defmodule Plausible.Imported.Importer dodef new_import(source, site, user, opts, before_start_fun) doimport_params =opts- |> Keyword.take([:start_date, :end_date])+ |> Keyword.take([:start_date, :end_date, :legacy])|> Keyword.put(:source, source)|> Map.new()@@ -205,11 +205,21 @@ defmodule Plausible.Imported.Importer doOban.Notifier.notify(Oban, @oban_channel, %{event => site_import.id})end+ @doc """+ Allows to explicitly start listening for importer job notifications.++ Listener must explicitly filter out a subset of imports that apply to the given context.+ """+ @spec listen() :: :ok+ def listen() do+ :ok = Oban.Notifier.listen([@oban_channel])+ end+defp schedule_job(site_import, opts) do{listen?, opts} = Keyword.pop(opts, :listen?, false)if listen? do- :ok = Oban.Notifier.listen([@oban_channel])+ :ok = listen()endoptsdiff --git a/lib/plausible/imported/site_import.ex b/lib/plausible/imported/site_import.exindex d8774fcfb1..594fe8db52 100644--- a/lib/plausible/imported/site_import.ex+++ b/lib/plausible/imported/site_import.ex@@ -20,6 +20,7 @@ defmodule Plausible.Imported.SiteImport dofield :end_date, :datefield :source, Ecto.Enum, values: ImportSources.names()field :status, Ecto.Enum, values: @statuses+ field :legacy, :boolean, default: falsebelongs_to :site, Sitebelongs_to :imported_by, User@@ -37,10 +38,29 @@ defmodule Plausible.Imported.SiteImport dodef label(%__MODULE__{source: source}), do: ImportSources.by_name(source).label()def label(%Site.ImportedData{source: source}), do: source+ @spec from_legacy(Site.ImportedData.t()) :: t()+ def from_legacy(%Site.ImportedData{} = data) do+ status =+ case data.status do+ "ok" -> completed()+ "error" -> failed()+ _ -> importing()+ end++ %__MODULE__{+ id: 0,+ legacy: true,+ start_date: data.start_date,+ end_date: data.end_date,+ source: :universal_analytics,+ status: status+ }+ end+@spec create_changeset(Site.t(), User.t(), map()) :: Ecto.Changeset.t()def create_changeset(site, user, params) do%__MODULE__{}- |> cast(params, [:source, :start_date, :end_date])+ |> cast(params, [:source, :start_date, :end_date, :legacy])|> validate_required([:source])|> put_assoc(:site, site)|> put_assoc(:imported_by, user)diff --git a/lib/plausible/imported/universal_analytics.ex b/lib/plausible/imported/universal_analytics.exindex 246be89f4d..1c7a1e093e 100644--- a/lib/plausible/imported/universal_analytics.ex+++ b/lib/plausible/imported/universal_analytics.ex@@ -20,37 +20,43 @@ defmodule Plausible.Imported.UniversalAnalytics do@impl truedef before_start(site_import) do- site = Repo.preload(site_import, :site).site-- site- |> Plausible.Site.start_import(- site_import.start_date,- site_import.end_date,- label()- )- |> Repo.update!()+ if site_import.legacy do+ site = Repo.preload(site_import, :site).site++ site+ |> Plausible.Site.start_import(+ site_import.start_date,+ site_import.end_date,+ label()+ )+ |> Repo.update!()+ end:okend@impl truedef on_success(site_import, _extra_data) do- site = Repo.preload(site_import, :site).site+ if site_import.legacy do+ site = Repo.preload(site_import, :site).site- site- |> Plausible.Site.import_success()- |> Repo.update!()+ site+ |> Plausible.Site.import_success()+ |> Repo.update!()+ end:okend@impl truedef on_failure(site_import) do- site = Repo.preload(site_import, :site).site+ if site_import.legacy do+ site = Repo.preload(site_import, :site).site- site- |> Plausible.Site.import_failure()- |> Repo.update!()+ site+ |> Plausible.Site.import_failure()+ |> Repo.update!()+ end:okenddiff --git a/lib/plausible/purge.ex b/lib/plausible/purge.exindex 9fe11b077e..50fd3ed22e 100644--- a/lib/plausible/purge.ex+++ b/lib/plausible/purge.ex@@ -58,17 +58,19 @@ defmodule Plausible.Purge dodef delete_imported_stats!(%Plausible.Imported.SiteImport{} = site_import) dosite_import = Repo.preload(site_import, :site)+ delete_imported_stats!(site_import.site, site_import.id)+ :ok+ end++ def delete_imported_stats!(%Plausible.Site{} = site, import_id) doEnum.each(Plausible.Imported.tables(), fn table ->sql = "ALTER TABLE #{table} DELETE WHERE site_id = {$0:UInt64} AND import_id = {$1:UInt64}"- Ecto.Adapters.SQL.query!(Plausible.ImportDeletionRepo, sql, [- site_import.site_id,- site_import.id- ])+ Ecto.Adapters.SQL.query!(Plausible.ImportDeletionRepo, sql, [site.id, import_id])end)- Plausible.Sites.clear_stats_start_date!(site_import.site)+ Plausible.Sites.clear_stats_start_date!(site):okenddiff --git a/lib/plausible/sites.ex b/lib/plausible/sites.exindex 1c8ac51e18..12d4235adf 100644--- a/lib/plausible/sites.ex+++ b/lib/plausible/sites.ex@@ -246,7 +246,7 @@ defmodule Plausible.Sites dostart_date =[site.earliest_import_start_date,- Plausible.Stats.Clickhouse.pageview_start_date_local(site)+ native_stats_start_date(site)]|> Enum.reject(&is_nil/1)|> Enum.min(Date, fn -> nil end)@@ -261,6 +261,11 @@ defmodule Plausible.Sites doendend+ @spec native_stats_start_date(Site.t()) :: Date.t() | nil+ def native_stats_start_date(site) do+ Plausible.Stats.Clickhouse.pageview_start_date_local(site)+ end+def has_stats?(site) do!!stats_start_date(site)enddiff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.exindex 782b868320..139e92e99f 100644--- a/lib/plausible/stats/clickhouse.ex+++ b/lib/plausible/stats/clickhouse.ex@@ -39,6 +39,17 @@ defmodule Plausible.Stats.Clickhouse do)end+ @spec imported_pageview_counts(Plausible.Site.t()) :: %{non_neg_integer() => non_neg_integer()}+ def imported_pageview_counts(site) do+ from(i in "imported_visitors",+ where: i.site_id == ^site.id,+ group_by: i.import_id,+ select: {i.import_id, sum(i.pageviews)}+ )+ |> Plausible.ClickhouseRepo.all()+ |> Map.new()+ end+def usage_breakdown([d | _] = domains, date_range) when is_binary(d) doEnum.chunk_every(domains, 300)|> Enum.reduce({0, 0}, fn domains, {pageviews_total, custom_events_total} ->diff --git a/lib/plausible_web/components/generic.ex b/lib/plausible_web/components/generic.exindex e0b792a5a6..1d2a62b06e 100644--- a/lib/plausible_web/components/generic.ex+++ b/lib/plausible_web/components/generic.ex@@ -19,7 +19,18 @@ defmodule PlausibleWeb.Components.Generic do}}+ @button_themes %{+ "primary" => "bg-indigo-600 text-white hover:bg-indigo-700 focus-visible:outline-indigo-600",+ "bright" =>+ "border border-gray-200 bg-gray-100 dark:bg-gray-300 text-gray-800 hover:bg-gray-200 focus-visible:outline-gray-100",+ "danger" =>+ "border border-gray-300 dark:border-gray-500 text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:border-blue-300 active:text-red-800"+ }++ @button_base_class "inline-flex items-center justify-center gap-x-2 rounded-md px-3.5 py-2.5 text-sm font-semibold shadow-sm focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 disabled:bg-gray-400 dark:disabled:text-white dark:disabled:text-gray-400 dark:disabled:bg-gray-700"+attr(:type, :string, default: "button")+ attr(:theme, :string, default: "primary")attr(:class, :string, default: "")attr(:disabled, :boolean, default: false)attr(:rest, :global)@@ -27,12 +38,19 @@ defmodule PlausibleWeb.Components.Generic doslot(:inner_block)def button(assigns) do+ assigns =+ assign(assigns,+ button_base_class: @button_base_class,+ theme_class: @button_themes[assigns.theme]+ )+~H"""<buttontype={@type}disabled={@disabled}class={[- "inline-flex items-center justify-center gap-x-2 rounded-md bg-indigo-600 px-3.5 py-2.5 text-sm font-semibold text-white shadow-sm hover:bg-indigo-700 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600 disabled:bg-gray-400 dark:disabled:text-white dark:disabled:text-gray-400 dark:disabled:bg-gray-700",+ @button_base_class,+ @theme_class,@class]}{@rest}@@ -44,16 +62,32 @@ defmodule PlausibleWeb.Components.Generic doattr(:href, :string, required: true)attr(:class, :string, default: "")+ attr(:theme, :string, default: "primary")+ attr(:disabled, :boolean, default: false)attr(:rest, :global)slot(:inner_block)def button_link(assigns) do+ theme_class =+ if assigns.disabled do+ "bg-gray-400 text-white dark:text-white dark:text-gray-400 dark:bg-gray-700 pointer-events-none cursor-default"+ else+ @button_themes[assigns.theme]+ end++ assigns =+ assign(assigns,+ button_base_class: @button_base_class,+ theme_class: theme_class+ )+~H"""<.linkhref={@href}class={[- "inline-flex items-center justify-center gap-x-2 rounded-md bg-indigo-600 px-3.5 py-2.5 text-sm font-semibold text-white shadow-sm hover:bg-indigo-700 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600 disabled:bg-gray-400 dark:disabled:bg-gray-800",+ @button_base_class,+ @theme_class,@class]}{@rest}diff --git a/lib/plausible_web/controllers/auth_controller.ex b/lib/plausible_web/controllers/auth_controller.exindex fcf9ebee2e..11a68efc52 100644--- a/lib/plausible_web/controllers/auth_controller.ex+++ b/lib/plausible_web/controllers/auth_controller.ex@@ -697,9 +697,24 @@ defmodule PlausibleWeb.AuthController doenddef google_auth_callback(conn, %{"error" => error, "state" => state} = params) do- [site_id, _redirect_to] = Jason.decode!(state)+ [site_id, _redirected_to, legacy] =+ case Jason.decode!(state) do+ [site_id, redirect_to] ->+ [site_id, redirect_to, true]++ [site_id, redirect_to, legacy] ->+ [site_id, redirect_to, legacy]+ end+site = Repo.get(Plausible.Site, site_id)+ redirect_route =+ if legacy do+ Routes.site_path(conn, :settings_integrations, site.domain)+ else+ Routes.site_path(conn, :settings_imports_exports, site.domain)+ end+case error do"access_denied" ->conn@@ -707,7 +722,7 @@ defmodule PlausibleWeb.AuthController do:error,"We were unable to authenticate your Google Analytics account. Please check that you have granted us permission to 'See and download your Google Analytics data' and try again.")- |> redirect(external: Routes.site_path(conn, :settings_general, site.domain))+ |> redirect(external: redirect_route)message when message in ["server_error", "temporarily_unavailable"] ->conn@@ -715,7 +730,7 @@ defmodule PlausibleWeb.AuthController do:error,"We are unable to authenticate your Google Analytics account because Google's authentication service is temporarily unavailable. Please try again in a few moments.")- |> redirect(external: Routes.site_path(conn, :settings_general, site.domain))+ |> redirect(external: redirect_route)_any ->Sentry.capture_message("Google OAuth callback failed. Reason: #{inspect(params)}")@@ -725,13 +740,22 @@ defmodule PlausibleWeb.AuthController do:error,"We were unable to authenticate your Google Analytics account. If the problem persists, please contact support for assistance.")- |> redirect(external: Routes.site_path(conn, :settings_general, site.domain))+ |> redirect(external: redirect_route)endenddef google_auth_callback(conn, %{"code" => code, "state" => state}) dores = Plausible.Google.HTTP.fetch_access_token(code)- [site_id, redirect_to] = Jason.decode!(state)++ [site_id, redirect_to, legacy] =+ case Jason.decode!(state) do+ [site_id, redirect_to] ->+ [site_id, redirect_to, true]++ [site_id, redirect_to, legacy] ->+ [site_id, redirect_to, legacy]+ end+site = Repo.get(Plausible.Site, site_id)expires_at = NaiveDateTime.add(NaiveDateTime.utc_now(), res["expires_in"])@@ -742,7 +766,8 @@ defmodule PlausibleWeb.AuthController doRoutes.site_path(conn, :import_from_google_view_id_form, site.domain,access_token: res["access_token"],refresh_token: res["refresh_token"],- expires_at: NaiveDateTime.to_iso8601(expires_at)+ expires_at: NaiveDateTime.to_iso8601(expires_at),+ legacy: legacy))diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.exindex 1dcf679e8b..51c04c7991 100644--- a/lib/plausible_web/controllers/site_controller.ex+++ b/lib/plausible_web/controllers/site_controller.ex@@ -4,10 +4,12 @@ defmodule PlausibleWeb.SiteController doalias Plausible.Sitesalias Plausible.Billing.Quota- plug PlausibleWeb.RequireAccountPlug+ plug(PlausibleWeb.RequireAccountPlug)- plug PlausibleWeb.AuthorizeSiteAccess,- [:owner, :admin, :super_admin] when action not in [:new, :create_site]+ plug(+ PlausibleWeb.AuthorizeSiteAccess,+ [:owner, :admin, :super_admin] when action not in [:new, :create_site]+ )def new(conn, _params) docurrent_user = conn.assigns[:current_user]@@ -62,10 +64,11 @@ defmodule PlausibleWeb.SiteController dois_first_site =!Repo.exists?(- from sm in Plausible.Site.Membership,+ from(sm in Plausible.Site.Membership,where:sm.user_id == ^user.id andsm.site_id != ^site.id+ ))conn@@ -144,7 +147,7 @@ defmodule PlausibleWeb.SiteController dodef settings_visibility(conn, _params) dosite = conn.assigns[:site]- shared_links = Repo.all(from l in Plausible.Site.SharedLink, where: l.site_id == ^site.id)+ shared_links = Repo.all(from(l in Plausible.Site.SharedLink, where: l.site_id == ^site.id))conn|> render("settings_visibility.html",@@ -267,6 +270,23 @@ defmodule PlausibleWeb.SiteController do)end+ def settings_imports_exports(conn, _params) do+ site = conn.assigns.site++ if FunWithFlags.enabled?(:imports_exports, for: site) do+ conn+ |> render("settings_imports_exports.html",+ site: site,+ dogfood_page_path: "/:dashboard/settings/imports-exports",+ connect_live_socket: true,+ layout: {PlausibleWeb.LayoutView, "site_settings.html"}+ )+ else+ conn+ |> redirect(external: Routes.site_path(conn, :settings, site.domain))+ end+ end+def update_google_auth(conn, %{"google_auth" => attrs}) dosite = conn.assigns[:site] |> Repo.preload(:google_auth)@@ -374,7 +394,7 @@ defmodule PlausibleWeb.SiteController dodef disable_weekly_report(conn, _params) dosite = conn.assigns[:site]- Repo.delete_all(from wr in Plausible.Site.WeeklyReport, where: wr.site_id == ^site.id)+ Repo.delete_all(from(wr in Plausible.Site.WeeklyReport, where: wr.site_id == ^site.id))conn|> put_flash(:success, "You will not receive weekly email reports going forward")@@ -428,7 +448,7 @@ defmodule PlausibleWeb.SiteController dodef disable_monthly_report(conn, _params) dosite = conn.assigns[:site]- Repo.delete_all(from mr in Plausible.Site.MonthlyReport, where: mr.site_id == ^site.id)+ Repo.delete_all(from(mr in Plausible.Site.MonthlyReport, where: mr.site_id == ^site.id))conn|> put_flash(:success, "You will not receive monthly email reports going forward")@@ -488,7 +508,7 @@ defmodule PlausibleWeb.SiteController dodef disable_spike_notification(conn, _params) dosite = conn.assigns[:site]- Repo.delete_all(from mr in Plausible.Site.SpikeNotification, where: mr.site_id == ^site.id)+ Repo.delete_all(from(mr in Plausible.Site.SpikeNotification, where: mr.site_id == ^site.id))conn|> put_flash(:success, "Spike notification disabled")@@ -604,9 +624,10 @@ defmodule PlausibleWeb.SiteController dosite_id = site.idcase Repo.delete_all(- from l in Plausible.Site.SharedLink,+ from(l in Plausible.Site.SharedLink,where: l.slug == ^slug,where: l.site_id == ^site_id+ )) do{1, _} ->conn@@ -624,7 +645,8 @@ defmodule PlausibleWeb.SiteController do"view_id" => view_id,"access_token" => access_token,"refresh_token" => refresh_token,- "expires_at" => expires_at+ "expires_at" => expires_at,+ "legacy" => legacy}) dosite = conn.assigns[:site]@@ -636,6 +658,7 @@ defmodule PlausibleWeb.SiteController doaccess_token: access_token,refresh_token: refresh_token,expires_at: expires_at,+ legacy: legacy,layout: {PlausibleWeb.LayoutView, "focus.html"})end@@ -643,8 +666,16 @@ defmodule PlausibleWeb.SiteController dodef import_from_google_view_id_form(conn, %{"access_token" => access_token,"refresh_token" => refresh_token,- "expires_at" => expires_at+ "expires_at" => expires_at,+ "legacy" => legacy}) do+ redirect_route =+ if legacy == "true" do+ Routes.site_path(conn, :settings_integrations, conn.assigns.site.domain)+ else+ Routes.site_path(conn, :settings_imports_exports, conn.assigns.site.domain)+ end+case Plausible.Google.Api.list_views(access_token) do{:ok, view_ids} ->conn@@ -655,6 +686,7 @@ defmodule PlausibleWeb.SiteController doexpires_at: expires_at,site: conn.assigns.site,view_ids: view_ids,+ legacy: legacy,layout: {PlausibleWeb.LayoutView, "focus.html"})@@ -664,7 +696,7 @@ defmodule PlausibleWeb.SiteController do:error,"We were unable to authenticate your Google Analytics account. Please check that you have granted us permission to 'See and download your Google Analytics data' and try again.")- |> redirect(to: Routes.site_path(conn, :settings_general, conn.assigns.site.domain))+ |> redirect(external: redirect_route){:error, _any} ->conn@@ -672,7 +704,7 @@ defmodule PlausibleWeb.SiteController do:error,"We were unable to list your Google Analytics properties. If the problem persists, please contact support for assistance.")- |> redirect(to: Routes.site_path(conn, :settings_general, conn.assigns.site.domain))+ |> redirect(external: redirect_route)endend@@ -682,7 +714,8 @@ defmodule PlausibleWeb.SiteController do"view_id" => view_id,"access_token" => access_token,"refresh_token" => refresh_token,- "expires_at" => expires_at+ "expires_at" => expires_at,+ "legacy" => legacy}) dosite = conn.assigns[:site]start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token)@@ -701,6 +734,7 @@ defmodule PlausibleWeb.SiteController dosite: site,view_ids: view_ids,selected_view_id_error: "No data found. Nothing to import",+ legacy: legacy,layout: {PlausibleWeb.LayoutView, "focus.html"})@@ -712,7 +746,8 @@ defmodule PlausibleWeb.SiteController doview_id: view_id,access_token: access_token,refresh_token: refresh_token,- expires_at: expires_at+ expires_at: expires_at,+ legacy: legacy))else@@ -722,7 +757,8 @@ defmodule PlausibleWeb.SiteController doview_id: view_id,access_token: access_token,refresh_token: refresh_token,- expires_at: expires_at+ expires_at: expires_at,+ legacy: legacy))end@@ -733,12 +769,14 @@ defmodule PlausibleWeb.SiteController do"view_id" => view_id,"access_token" => access_token,"refresh_token" => refresh_token,- "expires_at" => expires_at+ "expires_at" => expires_at,+ "legacy" => legacy}) dosite = conn.assigns[:site]start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token)- end_date = Plausible.Sites.stats_start_date(site) || Timex.today(site.timezone)++ end_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone){:ok, {view_name, view_id}} = Plausible.Google.Api.get_view(access_token, view_id)@@ -753,6 +791,7 @@ defmodule PlausibleWeb.SiteController doselected_view_id_name: view_name,start_date: start_date,end_date: end_date,+ legacy: legacy,layout: {PlausibleWeb.LayoutView, "focus.html"})end@@ -763,11 +802,19 @@ defmodule PlausibleWeb.SiteController do"end_date" => end_date,"access_token" => access_token,"refresh_token" => refresh_token,- "expires_at" => expires_at+ "expires_at" => expires_at,+ "legacy" => legacy}) dosite = conn.assigns.sitecurrent_user = conn.assigns.current_user+ redirect_route =+ if legacy == "true" do+ Routes.site_path(conn, :settings_integrations, site.domain)+ else+ Routes.site_path(conn, :settings_imports_exports, site.domain)+ end+{:ok, _} =Plausible.Imported.UniversalAnalytics.new_import(site,@@ -777,12 +824,49 @@ defmodule PlausibleWeb.SiteController doend_date: end_date,access_token: access_token,refresh_token: refresh_token,- token_expires_at: expires_at+ token_expires_at: expires_at,+ legacy: legacy == "true")conn|> put_flash(:success, "Import scheduled. An email will be sent when it completes.")- |> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain))+ |> redirect(external: redirect_route)+ end++ def forget_import(conn, %{"import_id" => import_id}) do+ site = conn.assigns.site++ cond do+ import_id == "0" ->+ Plausible.Purge.delete_imported_stats!(site, 0)++ site+ |> Plausible.Site.remove_imported_data()+ |> Repo.update!()++ site_import = Plausible.Imported.get_import(import_id) ->+ Oban.cancel_all_jobs(+ from(j in Oban.Job,+ where:+ j.queue == "analytics_imports" and+ fragment("(? ->> 'import_id')::int", j.args) == ^site_import.id+ )+ )++ Plausible.Purge.delete_imported_stats!(site_import)++ Plausible.Repo.delete!(site_import)++ if site_import.legacy do+ site+ |> Plausible.Site.remove_imported_data()+ |> Repo.update!()+ end+ end++ conn+ |> put_flash(:success, "Imported data has been cleared")+ |> redirect(external: Routes.site_path(conn, :settings_imports_exports, site.domain))enddef forget_imported(conn, _params) do@@ -793,19 +877,13 @@ defmodule PlausibleWeb.SiteController do|> Plausible.Imported.list_all_imports()|> Enum.map(& &1.id)- import_ids =- if site.imported_data do- [0 | import_ids]- else- import_ids- end-if import_ids != [] doOban.cancel_all_jobs(- from j in Oban.Job,+ from(j in Oban.Job,where:j.queue == "analytics_imports" andfragment("(? ->> 'import_id')::int", j.args) in ^import_ids+ ))Plausible.Purge.delete_imported_stats!(site)diff --git a/lib/plausible_web/live/imports_exports_settings.ex b/lib/plausible_web/live/imports_exports_settings.exnew file mode 100644index 0000000000..fc9d170dff--- /dev/null+++ b/lib/plausible_web/live/imports_exports_settings.ex@@ -0,0 +1,144 @@+defmodule PlausibleWeb.Live.ImportsExportsSettings do+ @moduledoc """+ LiveView allowing listing and deleting imports.+ """+ use PlausibleWeb, :live_view+ use Phoenix.HTML++ import PlausibleWeb.Components.Generic+ import PlausibleWeb.TextHelpers++ alias Plausible.Imported+ alias Plausible.Imported.SiteImport+ alias Plausible.Sites++ require Plausible.Imported.SiteImport++ def mount(+ _params,+ %{"domain" => domain, "current_user_id" => user_id},+ socket+ ) do+ socket =+ socket+ |> assign_new(:site, fn ->+ Sites.get_for_user!(user_id, domain, [:owner, :admin, :super_admin])+ end)+ |> assign_new(:site_imports, fn %{site: site} ->+ site+ |> Imported.list_all_imports()+ |> Enum.map(&%{site_import: &1, live_status: &1.status})+ end)+ |> assign_new(:pageview_counts, fn %{site: site} ->+ Plausible.Stats.Clickhouse.imported_pageview_counts(site)+ end)+ |> assign_new(:current_user, fn ->+ Plausible.Repo.get(Plausible.Auth.User, user_id)+ end)++ :ok = Imported.listen()++ {:ok, assign(socket, max_imports: Imported.max_complete_imports())}+ end++ def render(assigns) do+ ~H"""+ <header class="relative border-b border-gray-200 pb-4">+ <h3 class="mt-8 text-md leading-6 font-medium text-gray-900 dark:text-gray-100">+ Existing Imports+ </h3>+ <p class="mt-1 text-sm leading-5 text-gray-500 dark:text-gray-200">+ A maximum of <%= @max_imports %> imports at any time is allowed.+ </p>+ </header>++ <div+ :if={Enum.empty?(@site_imports)}+ class="text-gray-800 dark:text-gray-200 text-center mt-8 mb-12"+ >+ <p>There are no imports yet for this site.</p>+ </div>+ <ul :if={not Enum.empty?(@site_imports)}>+ <li :for={entry <- @site_imports} class="py-4 flex items-center justify-between space-x-4">+ <div class="flex flex-col">+ <p class="text-sm leading-5 font-medium text-gray-900 dark:text-gray-100">+ Import from <%= Plausible.Imported.SiteImport.label(entry.site_import) %>+ <span :if={entry.live_status == SiteImport.completed()} class="text-xs font-normal">+ (<%= Map.get(@pageview_counts, entry.site_import.id, 0) %> page views)+ </span>+ <Heroicons.clock+ :if={entry.live_status == SiteImport.pending()}+ class="inline-block h-6 w-5 text-indigo-600 dark:text-green-600"+ />+ <.spinner+ :if={entry.live_status == SiteImport.importing()}+ class="inline-block h-6 w-5 text-indigo-600 dark:text-green-600"+ />+ <Heroicons.check+ :if={entry.live_status == SiteImport.completed()}+ class="inline-block h-6 w-5 text-indigo-600 dark:text-green-600"+ />+ <Heroicons.exclamation_triangle+ :if={entry.live_status == SiteImport.failed()}+ class="inline-block h-6 w-5 text-indigo-600 dark:text-green-600"+ />+ </p>+ <p class="text-sm leading-5 text-gray-500 dark:text-gray-200">+ From <%= format_date(entry.site_import.start_date) %> to <%= format_date(+ entry.site_import.end_date+ ) %> (created on <%= format_date(+ entry.site_import.inserted_at || entry.site_import.start_date+ ) %>)+ </p>+ </div>+ <.button+ data-to={"/#{URI.encode_www_form(@site.domain)}/settings/forget-import/#{entry.site_import.id}"}+ theme="danger"+ data-method="delete"+ data-csrf={Plug.CSRFProtection.get_csrf_token()}+ class="sm:ml-3 sm:w-auto w-full"+ data-confirm="Are you sure you want to delete this import?"+ >+ <span :if={entry.live_status in [SiteImport.completed(), SiteImport.failed()]}>+ Delete Import+ </span>+ <span :if={entry.live_status not in [SiteImport.completed(), SiteImport.failed()]}>+ Cancel Import+ </span>+ </.button>+ </li>+ </ul>+ """+ end++ def handle_info({:notification, :analytics_imports_jobs, status}, socket) do+ [{status_str, import_id}] = Enum.to_list(status)+ {site_imports, updated?} = update_imports(socket.assigns.site_imports, import_id, status_str)++ pageview_counts =+ if updated? do+ Plausible.Stats.Clickhouse.imported_pageview_counts(socket.assigns.site)+ else+ socket.assigns.pageview_counts+ end++ {:noreply, assign(socket, site_imports: site_imports, pageview_counts: pageview_counts)}+ end++ defp update_imports(site_imports, import_id, status_str) do+ Enum.map_reduce(site_imports, false, fn+ %{site_import: %{id: ^import_id}} = entry, _changed? ->+ new_status =+ case status_str do+ "complete" -> SiteImport.completed()+ "fail" -> SiteImport.failed()+ "transient_fail" -> SiteImport.importing()+ end++ {%{entry | live_status: new_status}, true}++ entry, changed? ->+ {entry, changed?}+ end)+ end+enddiff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.exindex a685701b22..d1c414ac76 100644--- a/lib/plausible_web/router.ex+++ b/lib/plausible_web/router.ex@@ -360,6 +360,7 @@ defmodule PlausibleWeb.Router doget "/:website/settings/danger-zone", SiteController, :settings_danger_zoneget "/:website/settings/integrations", SiteController, :settings_integrationsget "/:website/settings/shields/:shield", SiteController, :settings_shields+ get "/:website/settings/imports-exports", SiteController, :settings_imports_exportsput "/:website/settings/features/visibility/:setting",SiteController,@@ -385,6 +386,7 @@ defmodule PlausibleWeb.Router doget "/:website/import/google-analytics/confirm", SiteController, :import_from_google_confirmpost "/:website/settings/google-import", SiteController, :import_from_googledelete "/:website/settings/forget-imported", SiteController, :forget_imported+ delete "/:website/settings/forget-import/:import_id", SiteController, :forget_importget "/:domain/export", StatsController, :csv_exportget "/:domain/*path", StatsController, :statsdiff --git a/lib/plausible_web/templates/site/import_from_google_confirm.html.eex b/lib/plausible_web/templates/site/import_from_google_confirm.html.eexindex 4e1018faa3..c46459aa94 100644--- a/lib/plausible_web/templates/site/import_from_google_confirm.html.eex+++ b/lib/plausible_web/templates/site/import_from_google_confirm.html.eex@@ -4,6 +4,7 @@<%= hidden_input(f, :access_token, value: @access_token) %><%= hidden_input(f, :refresh_token, value: @refresh_token) %><%= hidden_input(f, :expires_at, value: @expires_at) %>+ <%= hidden_input(f, :legacy, value: @legacy) %><%= case @start_date do %><% {:ok, start_date} -> %>diff --git a/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex b/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eexindex d8f69f2e5e..e5792ed9f9 100644--- a/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex+++ b/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex@@ -22,5 +22,5 @@</p></div>- <%= link("Continue ->", to: Routes.site_path(@conn, :import_from_google_confirm, @site.domain, view_id: @view_id, access_token: @access_token, refresh_token: @refresh_token, expires_at: @expires_at), class: "button mt-6") %>+ <%= link("Continue ->", to: Routes.site_path(@conn, :import_from_google_confirm, @site.domain, view_id: @view_id, access_token: @access_token, refresh_token: @refresh_token, expires_at: @expires_at, legacy: @legacy), class: "button mt-6") %></div>diff --git a/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex b/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eexindex 42099ed0ae..d8b8db4442 100644--- a/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex+++ b/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex@@ -4,6 +4,7 @@<%= hidden_input(f, :access_token, value: @access_token) %><%= hidden_input(f, :refresh_token, value: @refresh_token) %><%= hidden_input(f, :expires_at, value: @expires_at) %>+ <%= hidden_input(f, :legacy, value: @legacy) %><div class="mt-6 text-sm text-gray-500 dark:text-gray-200">Choose the view in your Google Analytics account that will be imported to the <%= @site.domain %> dashboard.diff --git a/lib/plausible_web/templates/site/settings_google_import.html.heex b/lib/plausible_web/templates/site/settings_google_import.html.heexindex acca9718ca..fb223f2982 100644--- a/lib/plausible_web/templates/site/settings_google_import.html.heex+++ b/lib/plausible_web/templates/site/settings_google_import.html.heex@@ -92,7 +92,7 @@<% end %><PlausibleWeb.Components.Google.buttonid="analytics-connect"- to={Plausible.Google.Api.import_authorize_url(@site.id, "import")}+ to={Plausible.Google.Api.import_authorize_url(@site.id, "import", true)}/><% end %><% else %>diff --git a/lib/plausible_web/templates/site/settings_imports_exports.html.heex b/lib/plausible_web/templates/site/settings_imports_exports.html.heexnew file mode 100644index 0000000000..9369143a17--- /dev/null+++ b/lib/plausible_web/templates/site/settings_imports_exports.html.heex@@ -0,0 +1,61 @@+<div class="shadow bg-white dark:bg-gray-800 sm:rounded-md sm:overflow-hidden py-6 px-4 sm:p-6">+ <header class="relative border-b border-gray-200 pb-4">+ <h2 class="text-lg leading-6 font-medium text-gray-900 dark:text-gray-100">+ Import Data+ </h2>+ <p class="mt-1 text-sm leading-5 text-gray-500 dark:text-gray-200">+ Import existing data from external sources. Pick one of the options below to start a new import.+ </p>++ <PlausibleWeb.Components.Generic.docs_info slug="google-analytics-import" />+ </header>++ <div class="mt-5 flex gap-x-4">+ <PlausibleWeb.Components.Generic.button_link+ class="w-36 h-20"+ theme="bright"+ href={Plausible.Google.Api.import_authorize_url(@site.id, "import", false)}+ >+ <img src="/images/icon/universal_analytics_logo.svg" alt="New Universal Analytics import" />+ </PlausibleWeb.Components.Generic.button_link>++ <PlausibleWeb.Components.Generic.button_link+ class="w-36 h-20 opacity-40 cursor-not-allowed"+ theme="bright"+ href=""+ >+ <img+ src="/images/icon/google_analytics_4_logo.svg"+ width="110"+ alt="New Universal Analytics import"+ />+ </PlausibleWeb.Components.Generic.button_link>++ <PlausibleWeb.Components.Generic.button_link+ class="w-36 h-20 opacity-40 cursor-not-allowed"+ theme="bright"+ href=""+ >+ <img class="h-16" src="/images/icon/csv_logo.svg" alt="New CSV import" />+ </PlausibleWeb.Components.Generic.button_link>+ </div>++ <%= live_render(@conn, PlausibleWeb.Live.ImportsExportsSettings,+ session: %{"domain" => @site.domain}+ ) %>++ <header class="relative border-b border-gray-200 pb-4">+ <h2 class="mt-8 text-lg leading-6 font-medium text-gray-900 dark:text-gray-100">+ Export Data+ </h2>+ <p class="mt-1 text-sm leading-5 text-gray-500 dark:text-gray-200">+ Export all your data into CSV format.+ </p>+ </header>++ <div class="mt-4">+ <PlausibleWeb.Components.Generic.button_link disabled href="">+ Export to CSV+ </PlausibleWeb.Components.Generic.button_link>+ </div>+</div>diff --git a/lib/plausible_web/views/layout_view.ex b/lib/plausible_web/views/layout_view.exindex 8bdd199bd6..0b587379d9 100644--- a/lib/plausible_web/views/layout_view.ex+++ b/lib/plausible_web/views/layout_view.ex@@ -59,6 +59,9 @@ defmodule PlausibleWeb.LayoutView doend,%{key: "Custom Properties", value: "properties", icon: :document_text},%{key: "Integrations", value: "integrations", icon: :arrow_path_rounded_square},+ if FunWithFlags.enabled?(:imports_exports, for: conn.assigns.site) do+ %{key: "Imports & Exports", value: "imports-exports", icon: :arrows_up_down}+ end,%{key: "Shields",icon: :shield_exclamation,diff --git a/priv/repo/seeds.exs b/priv/repo/seeds.exsindex 37b0b309ea..e1b499f76c 100644--- a/priv/repo/seeds.exs+++ b/priv/repo/seeds.exs@@ -10,6 +10,8 @@# We recommend using the bang functions (`insert!`, `update!`# and so on) as they will fail if something goes wrong.+FunWithFlags.enable(:imports_exports)+user = Plausible.Factory.insert(:user, email: "[email protected]", password: "plausible")native_stats_range =@@ -18,9 +20,15 @@ native_stats_range =Date.utc_today())-imported_stats_range =+legacy_imported_stats_range =Date.range(Date.add(native_stats_range.first, -360),+ Date.add(native_stats_range.first, -180)+ )++imported_stats_range =+ Date.range(+ Date.add(native_stats_range.first, -180),Date.add(native_stats_range.first, -1))@@ -39,7 +47,7 @@ site =Plausible.Factory.insert(:site,domain: "dummy.site",native_stats_start_at: NaiveDateTime.new!(native_stats_range.first, ~T[00:00:00]),- stats_start_date: NaiveDateTime.new!(imported_stats_range.first, ~T[00:00:00]),+ stats_start_date: NaiveDateTime.new!(legacy_imported_stats_range.first, ~T[00:00:00]),memberships: [Plausible.Factory.build(:site_membership, user: user, role: :owner),Plausible.Factory.build(:site_membership,@@ -250,13 +258,13 @@ end)site =site|> Plausible.Site.start_import(- imported_stats_range.first,- imported_stats_range.last,+ legacy_imported_stats_range.first,+ legacy_imported_stats_range.last,"Google Analytics")|> Plausible.Repo.update!()-imported_stats_range+legacy_imported_stats_range|> Enum.flat_map(fn date ->Enum.flat_map(0..Enum.random(1..500), fn _ ->[@@ -291,3 +299,50 @@ end)site|> Plausible.Site.import_success()|> Plausible.Repo.update!()++site_import =+ site+ |> Plausible.Imported.SiteImport.create_changeset(user, %{+ source: :universal_analytics,+ start_date: imported_stats_range.first,+ end_date: imported_stats_range.last,+ legacy: false+ })+ |> Plausible.Imported.SiteImport.start_changeset()+ |> Plausible.Repo.insert!()++imported_stats_range+|> Enum.flat_map(fn date ->+ Enum.flat_map(0..Enum.random(1..500), fn _ ->+ [+ Plausible.Factory.build(:imported_visitors,+ date: date,+ pageviews: Enum.random(1..20),+ visitors: Enum.random(1..20),+ bounces: Enum.random(1..20),+ visits: Enum.random(1..200),+ visit_duration: Enum.random(1000..10000)+ ),+ Plausible.Factory.build(:imported_sources,+ date: date,+ source: Enum.random(["", "Facebook", "Twitter", "DuckDuckGo", "Google"]),+ visitors: Enum.random(1..20),+ visits: Enum.random(1..200),+ bounces: Enum.random(1..20),+ visit_duration: Enum.random(1000..10000)+ ),+ Plausible.Factory.build(:imported_pages,+ date: date,+ visitors: Enum.random(1..20),+ pageviews: Enum.random(1..20),+ exits: Enum.random(1..20),+ time_on_page: Enum.random(1000..10000)+ )+ ]+ end)+end)+|> then(&Plausible.TestUtils.populate_stats(site, site_import.id, &1))++site_import+|> Plausible.Imported.SiteImport.complete_changeset()+|> Plausible.Repo.update!()diff --git a/priv/static/images/icon/csv_logo.svg b/priv/static/images/icon/csv_logo.svgnew file mode 100644index 0000000000..d36e864e5c--- /dev/null+++ b/priv/static/images/icon/csv_logo.svg@@ -0,0 +1 @@+<svg height="64" viewBox="0 0 56 64" width="56" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="m5.106 0c-2.802 0-5.073 2.272-5.073 5.074v53.841c0 2.803 2.271 5.074 5.073 5.074h45.774c2.801 0 5.074-2.271 5.074-5.074v-38.605l-18.903-20.31h-31.945z" fill="#45b058" fill-rule="evenodd"/><path d="m20.306 43.197c.126.144.198.324.198.522 0 .378-.306.72-.703.72-.18 0-.378-.072-.504-.234-.702-.846-1.891-1.387-3.007-1.387-2.629 0-4.627 2.017-4.627 4.88 0 2.845 1.999 4.879 4.627 4.879 1.134 0 2.25-.486 3.007-1.369.125-.144.324-.233.504-.233.415 0 .703.359.703.738 0 .18-.072.36-.198.504-.937.972-2.215 1.693-4.015 1.693-3.457 0-6.176-2.521-6.176-6.212s2.719-6.212 6.176-6.212c1.8.001 3.096.721 4.015 1.711zm6.802 10.714c-1.782 0-3.187-.594-4.213-1.495-.162-.144-.234-.342-.234-.54 0-.361.27-.757.702-.757.144 0 .306.036.432.144.828.739 1.98 1.314 3.367 1.314 2.143 0 2.827-1.152 2.827-2.071 0-3.097-7.112-1.386-7.112-5.672 0-1.98 1.764-3.331 4.123-3.331 1.548 0 2.881.467 3.853 1.278.162.144.252.342.252.54 0 .36-.306.72-.703.72-.144 0-.306-.054-.432-.162-.882-.72-1.98-1.044-3.079-1.044-1.44 0-2.467.774-2.467 1.909 0 2.701 7.112 1.152 7.112 5.636.001 1.748-1.187 3.531-4.428 3.531zm16.994-11.254-4.159 10.335c-.198.486-.685.81-1.188.81h-.036c-.522 0-1.008-.324-1.207-.81l-4.142-10.335c-.036-.09-.054-.18-.054-.288 0-.36.323-.793.81-.793.306 0 .594.18.72.486l3.889 9.992 3.889-9.992c.108-.288.396-.486.72-.486.468 0 .81.378.81.793.001.09-.017.198-.052.288z" fill="#fff"/><g clip-rule="evenodd" fill-rule="evenodd"><path d="m56.001 20.357v1h-12.8s-6.312-1.26-6.128-6.707c0 0 .208 5.707 6.003 5.707z" fill="#349c42"/><path d="m37.098.006v14.561c0 1.656 1.104 5.791 6.104 5.791h12.8l-18.904-20.352z" fill="#fff" opacity=".5"/></g></svg>\ No newline at end of filediff --git a/priv/static/images/icon/google_analytics_4_logo.svg b/priv/static/images/icon/google_analytics_4_logo.svgnew file mode 100644index 0000000000..951af735ff--- /dev/null+++ b/priv/static/images/icon/google_analytics_4_logo.svg@@ -0,0 +1 @@+<?xml version="1.0" encoding="UTF-8"?> <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="265.361" height="128.733" viewBox="0 0 265.361 128.733"><defs><clipPath id="clip-path"><rect width="265.361" height="128.733" fill="none"></rect></clipPath></defs><g id="Repeat_Grid_1" data-name="Repeat Grid 1" clip-path="url(#clip-path)"><g id="Group_51" data-name="Group 51" transform="translate(-595.244 -549.85)"><rect id="Rectangle_3" data-name="Rectangle 3" width="29.944" height="124.055" rx="14.972" transform="translate(673.313 549.85)" fill="#ffa800"></rect><rect id="Rectangle_4" data-name="Rectangle 4" width="29.944" height="77.689" rx="14.972" transform="translate(633.922 596.215)" fill="#f46d01"></rect><circle id="Ellipse_2" data-name="Ellipse 2" cx="14.616" cy="14.616" r="14.616" transform="translate(595.244 644.673)" fill="#f46d01"></circle><path id="Path_12" data-name="Path 12" d="M18.259-10.974H11.377V-8.8h4.247v6.231a8.8,8.8,0,0,1-3.6.558c-4.867,0-8.06-3.131-8.06-8.432,0-5.239,3.317-8.4,8.4-8.4a10.83,10.83,0,0,1,4.619.9l.651-2.2a12.917,12.917,0,0,0-5.208-.961c-7.006,0-11.284,4.557-11.315,10.788A10.749,10.749,0,0,0,4.03-2.542,10.74,10.74,0,0,0,11.842.217,19.465,19.465,0,0,0,18.259-.93ZM28.8-15.345c-4.154,0-7.44,2.945-7.44,7.967,0,4.743,3.131,7.719,7.192,7.719,3.627,0,7.471-2.418,7.471-7.967C36.022-12.214,33.108-15.345,28.8-15.345ZM28.737-13.3c3.224,0,4.5,3.224,4.5,5.766,0,3.379-1.953,5.828-4.557,5.828-2.666,0-4.557-2.48-4.557-5.766C24.118-10.323,25.513-13.3,28.737-13.3Zm17.081-2.046c-4.154,0-7.44,2.945-7.44,7.967,0,4.743,3.131,7.719,7.192,7.719,3.627,0,7.471-2.418,7.471-7.967C53.041-12.214,50.127-15.345,45.818-15.345ZM45.756-13.3c3.224,0,4.495,3.224,4.495,5.766,0,3.379-1.953,5.828-4.557,5.828-2.666,0-4.557-2.48-4.557-5.766C41.137-10.323,42.532-13.3,45.756-13.3ZM67.022-15,66.9-12.741h-.062a4.99,4.99,0,0,0-4.681-2.6c-3.441,0-6.758,2.883-6.758,7.967,0,4.154,2.666,7.316,6.386,7.316a5.356,5.356,0,0,0,4.774-2.511h.062V-.9c0,3.782-2.046,5.239-4.836,5.239a8.461,8.461,0,0,1-4.4-1.178L56.7,5.239a9.684,9.684,0,0,0,4.991,1.24,7.954,7.954,0,0,0,5.456-1.8c1.457-1.3,2.139-3.41,2.139-6.851v-8.711c0-1.829.062-3.038.124-4.123Zm-.434,8.618a4.746,4.746,0,0,1-.217,1.457,3.988,3.988,0,0,1-3.782,2.79c-2.945,0-4.433-2.449-4.433-5.394,0-3.472,1.86-5.673,4.464-5.673a3.93,3.93,0,0,1,3.813,2.883,3.885,3.885,0,0,1,.155,1.24ZM73.811,0h2.728V-22.01H73.811ZM93.186-7.006a10.289,10.289,0,0,0,.093-1.271c0-2.759-1.3-7.068-6.2-7.068-4.371,0-7.037,3.565-7.037,8.091S82.8.31,87.42.31a12.153,12.153,0,0,0,4.991-.93l-.465-1.953a9.865,9.865,0,0,1-4.154.775c-2.728,0-5.084-1.519-5.146-5.208ZM82.677-8.959c.217-1.891,1.426-4.433,4.185-4.433,3.069,0,3.813,2.7,3.782,4.433ZM13.144,21.428,15.376,28h2.883L11.129,7.106H7.874L.775,28h2.79l2.17-6.572ZM6.293,19.32l2.046-6.045c.4-1.271.744-2.542,1.054-3.782h.062c.31,1.209.62,2.449,1.085,3.813l2.046,6.014ZM21.142,28H23.87V18.979a3.922,3.922,0,0,1,.186-1.271,3.876,3.876,0,0,1,3.658-2.79c2.573,0,3.472,2.015,3.472,4.433V28h2.728V19.041c0-5.146-3.224-6.386-5.3-6.386a5.517,5.517,0,0,0-4.96,2.821h-.062L23.436,13H21.018c.093,1.24.124,2.511.124,4.061Zm27.993,0a21.125,21.125,0,0,1-.248-3.6V18.793c0-3.007-1.116-6.138-5.7-6.138a9.312,9.312,0,0,0-4.929,1.333l.62,1.8A7.476,7.476,0,0,1,42.78,14.67c3.069,0,3.41,2.232,3.41,3.472v.31c-5.8-.031-9.021,1.953-9.021,5.58a4.275,4.275,0,0,0,4.588,4.309,5.588,5.588,0,0,0,4.588-2.232h.093L46.655,28Zm-2.883-5.053a2.847,2.847,0,0,1-.155.868,3.733,3.733,0,0,1-3.627,2.511,2.414,2.414,0,0,1-2.573-2.6c0-2.914,3.379-3.441,6.355-3.379ZM53.289,28h2.728V5.99H53.289Zm5.332-15L64.17,26.822a2.379,2.379,0,0,1,.186.713,1.958,1.958,0,0,1-.217.651,8.271,8.271,0,0,1-2.294,3.038,7.216,7.216,0,0,1-2.387,1.333l.682,2.294a7.392,7.392,0,0,0,3.348-1.767c1.86-1.612,3.193-4.247,5.146-9.393L72.726,13H69.843l-2.976,8.8c-.372,1.085-.682,2.232-.961,3.131h-.062c-.248-.9-.62-2.077-.961-3.069L61.6,13ZM75.826,9.4V13H73.5v2.077h2.325v8.184c0,1.767.279,3.1,1.054,3.906a3.78,3.78,0,0,0,2.945,1.147,6.678,6.678,0,0,0,2.418-.372l-.124-2.046a6.069,6.069,0,0,1-1.581.186c-1.519,0-2.046-1.054-2.046-2.914V15.073H82.4V13H78.492V8.687ZM88.195,28V13H85.467V28ZM86.831,7.075A1.662,1.662,0,0,0,85.126,8.78a1.69,1.69,0,0,0,3.379,0A1.619,1.619,0,0,0,86.831,7.075ZM102.951,25.4a7.75,7.75,0,0,1-3.348.744c-2.976,0-5.208-2.139-5.208-5.611,0-3.131,1.86-5.673,5.3-5.673a6.326,6.326,0,0,1,3.162.713l.62-2.108a9.119,9.119,0,0,0-3.782-.775,7.624,7.624,0,0,0-8.06,7.967c0,4.588,2.945,7.657,7.471,7.657a10.048,10.048,0,0,0,4.309-.868Zm2.6,1.891a8.557,8.557,0,0,0,4.216,1.023c3.565,0,5.611-1.86,5.611-4.495,0-2.232-1.333-3.534-3.937-4.526-1.953-.744-2.852-1.3-2.852-2.542,0-1.116.9-2.046,2.511-2.046a5.839,5.839,0,0,1,3.069.868l.682-1.984a7.489,7.489,0,0,0-3.689-.93c-3.224,0-5.177,1.984-5.177,4.4,0,1.8,1.271,3.286,3.968,4.247,2.015.744,2.79,1.457,2.79,2.759,0,1.24-.93,2.232-2.914,2.232a7.149,7.149,0,0,1-3.6-1.085ZM135.594,28V22.513h2.821V20.374h-2.821V7.85h-2.945l-8.99,12.865v1.8h9.362V28Zm-9.176-7.626v-.062l4.929-6.851c.558-.93,1.054-1.8,1.674-2.976h.093q-.093,1.581-.093,3.162v6.727Z" transform="translate(722.19 643.732)" fill="gray"></path></g></g></svg>\ No newline at end of filediff --git a/priv/static/images/icon/universal_analytics_logo.svg b/priv/static/images/icon/universal_analytics_logo.svgnew file mode 100644index 0000000000..5ce2f3df3e--- /dev/null+++ b/priv/static/images/icon/universal_analytics_logo.svg@@ -0,0 +1 @@+<svg xmlns="http://www.w3.org/2000/svg" viewBox="26 -29 120 60" width="120" height="60"><linearGradient id="A" gradientUnits="userSpaceOnUse" x1="173.867" y1="-72.65" x2="216.749" y2="-72.65"><stop offset="0" stop-color="#e96f0b"/><stop offset="1" stop-color="#f37901"/></linearGradient><g fill-rule="evenodd"><path d="M47.33 16H36.226c-1.364 0-2.468-1.104-2.468-2.468V8.208c0-1.364 1.104-2.468 2.468-2.468h7.013v-7.792c0-1.364 1.104-2.468 2.468-2.468H53.5v20.455h-6.17z" fill="#ffc517"/><path d="M61.3 16h-7.8v-27.532c0-1.364 1.104-2.468 2.468-2.468h5.325c1.364 0 2.468 1.104 2.468 2.468v25.065C63.693 14.896 62.6 16 61.3 16z" fill="#f57e02"/><path d="M216.708-72.65v32.646c0 5.713-4.625 10.338-10.338 10.338h-32.646v-85.968l42.984 42.984z" fill="url(#A)" transform="matrix(.238687 0 0 .238687 11.968065 23.081216)"/></g><g fill="#757575"><path d="M74.093 7.228l2.388 6.27h-4.627l2.24-6.27zm-.896-2.1l-5.224 13.73h1.94l1.343-3.73h5.82l1.343 3.73h1.94l-5.224-13.73h-1.94zm28.2 13.742h1.8V5.138h-1.8v13.73zm-18.35-8.07c.597-.896 1.8-1.642 2.985-1.642 2.388 0 3.582 1.642 3.582 4.03v5.82h-1.8v-5.522c0-1.94-1.045-2.687-2.388-2.687-1.493 0-2.537 1.493-2.537 2.836v5.224h-1.8V9.32h1.8l.15 1.493zm8.06 5.234c0-2.1 1.94-3.284 4.18-3.284 1.343 0 2.24.3 2.537.597v-.3c0-1.493-1.194-2.24-2.388-2.24-1.045 0-1.94.448-2.24 1.343l-1.642-.746c.3-.896 1.493-2.24 3.88-2.24 2.24 0 4.18 1.343 4.18 4.03v5.672h-1.642v-1.343h-.15c-.448.746-1.493 1.642-3.134 1.642-1.94 0-3.582-1.194-3.582-3.134m6.716-1.194s-.746-.597-2.24-.597c-1.8 0-2.537 1.045-2.537 1.8 0 1.045 1.045 1.493 1.94 1.493 1.343 0 2.836-1.194 2.836-2.687"/><path d="M105.735 23.05l2.1-4.925-3.73-8.657h1.8l2.836 6.567 2.836-6.567h1.8l-5.82 13.582h-1.8zm26.27-10.45c-.448-1.194-1.493-1.94-2.537-1.94-1.493 0-2.836 1.343-2.836 3.284s1.343 3.284 2.836 3.284c1.045 0 2.1-.746 2.537-1.8l1.493.896c-.746 1.642-2.24 2.687-4.03 2.687-2.537 0-4.627-2.24-4.627-5.075 0-2.985 2.1-5.075 4.627-5.075 1.8 0 3.284 1.045 4.03 2.687l-1.493 1.045z" fill-rule="evenodd"/><path d="M138.422 19.168c2.388 0 3.582-1.343 3.582-2.985 0-3.582-5.224-2.24-5.224-4.328 0-.746.597-1.194 1.642-1.194s2.1.448 2.537 1.194l1.045-1.045c-.597-.746-2.24-1.8-3.73-1.8-2.24 0-3.433 1.343-3.433 2.985 0 3.433 5.373 2.24 5.373 4.03 0 .896-.597 1.493-1.8 1.493s-1.8-.746-2.388-1.493L134.7 17.08c.896.896 2.24 2.1 3.73 2.1zm-17.015-.298h1.8V9.318h-1.8v9.552z"/><path d="M122.153 4.7c.746 0 1.194.597 1.194 1.194 0 .746-.597 1.194-1.194 1.194s-1.194-.597-1.194-1.194.597-1.194 1.194-1.194zm-2.537 12.687l.3 1.343h-1.8c-1.642 0-2.388-1.194-2.388-2.985V11.1h-1.8V9.3h1.8V6.482h1.8v2.836h2.1v1.8h-2.1v4.925c0 1.343 1.045 1.343 2.1 1.343z" fill-rule="evenodd"/><path d="M77.048-10.042v2.24h5.224c-.15 1.194-.597 2.1-1.194 2.687-.746.746-1.94 1.642-4.03 1.642-3.134 0-5.672-2.537-5.672-5.82 0-3.134 2.537-5.82 5.672-5.82 1.8 0 2.985.746 3.88 1.493l1.493-1.493c-1.343-1.194-2.985-2.24-5.373-2.24-4.328 0-8.06 3.582-8.06 7.9s3.73 7.9 8.06 7.9c2.388 0 4.18-.746 5.522-2.24 1.493-1.493 1.94-3.433 1.94-5.075 0-.448 0-1.045-.15-1.343h-7.313zm13.282-1.8c-2.836 0-5.075 2.1-5.075 5.075s2.24 5.075 5.075 5.075a5.03 5.03 0 0 0 5.075-5.075c0-2.985-2.24-5.075-5.075-5.075zm0 8.2c-1.493 0-2.836-1.343-2.836-3.134S88.837-9.9 90.33-9.9s2.837 1.2 2.837 3.13-1.343 3.134-2.836 3.134zm24.47-7.156c-.597-.597-1.493-1.194-2.836-1.194-2.537 0-4.776 2.24-4.776 5.075s2.24 5.075 4.776 5.075c1.194 0 2.24-.597 2.687-1.194h.15v.746c0 1.94-1.045 2.985-2.687 2.985-1.343 0-2.24-1.045-2.537-1.8l-1.94.746c.597 1.343 2.1 2.985 4.478 2.985 2.687 0 4.925-1.493 4.925-5.373V-12h-2.1v1.194zm-2.537 7.164c-1.493 0-2.687-1.343-2.687-3.134s1.194-3.134 2.687-3.134 2.687 1.343 2.687 3.134-1.194 3.134-2.687 3.134zm-10.897-8.2c-2.836 0-5.075 2.1-5.075 5.075s2.24 5.075 5.075 5.075a5.03 5.03 0 0 0 5.075-5.075c.15-2.985-2.24-5.075-5.075-5.075zm0 8.2c-1.493 0-2.836-1.343-2.836-3.134s1.343-3.134 2.836-3.134 2.836 1.194 2.836 3.134-1.343 3.134-2.836 3.134zM118.7-17.206h2.24v15.522h-2.24v-15.522zm8.358 13.582c-1.194 0-1.94-.597-2.537-1.493l6.866-2.836-.3-.597c-.448-1.194-1.8-3.284-4.328-3.284-2.687 0-4.776 2.1-4.776 5.075a5.03 5.03 0 0 0 5.075 5.075c2.388 0 3.73-1.493 4.328-2.24l-1.8-1.194c-.597.896-1.343 1.493-2.537 1.493zm-.15-6.27c.896 0 1.642.448 1.94 1.194l-4.627 1.94c0-2.24 1.493-3.134 2.687-3.134z"/></g></svg>\ No newline at end of filediff --git a/test/plausible/imported/universal_analytics_test.exs b/test/plausible/imported/universal_analytics_test.exsindex 80ad9caab6..2d878fbc69 100644--- a/test/plausible/imported/universal_analytics_test.exs+++ b/test/plausible/imported/universal_analytics_test.exs@@ -20,7 +20,8 @@ defmodule Plausible.Imported.UniversalAnalyticsTest doend_date: "2024-01-02",access_token: "access123",refresh_token: "refresh123",- token_expires_at: expires_at+ token_expires_at: expires_at,+ legacy: true)assert %Oban.Job{@@ -42,7 +43,8 @@ defmodule Plausible.Imported.UniversalAnalyticsTest dosource: :universal_analytics,start_date: ~D[2023-10-01],end_date: ~D[2024-01-02],- status: SiteImport.pending()+ status: SiteImport.pending(),+ legacy: true}] = Plausible.Imported.list_all_imports(site)@@ -61,6 +63,35 @@ defmodule Plausible.Imported.UniversalAnalyticsTest doassert opts[:date_range] == Date.range(~D[2023-10-01], ~D[2024-01-02])assert opts[:auth] == {"access123", "refresh123", expires_at}end++ test "creates SiteImport with legacy flag set to false when instructed", %{+ user: user,+ site: site+ } do+ expires_at = NaiveDateTime.to_iso8601(NaiveDateTime.utc_now())++ assert {:ok, job} =+ UniversalAnalytics.new_import(site, user,+ view_id: 123,+ start_date: "2023-10-01",+ end_date: "2024-01-02",+ access_token: "access123",+ refresh_token: "refresh123",+ token_expires_at: expires_at,+ legacy: false+ )++ assert %Oban.Job{args: %{"import_id" => import_id}} = Repo.reload!(job)++ assert [+ %{+ id: ^import_id,+ legacy: false+ }+ ] = Plausible.Imported.list_all_imports(site)++ assert %{imported_data: nil} = Repo.reload!(site)+ endenddescribe "import_data/2" dodiff --git a/test/plausible/imported_test.exs b/test/plausible/imported_test.exsnew file mode 100644index 0000000000..ee98f5c37e--- /dev/null+++ b/test/plausible/imported_test.exs@@ -0,0 +1,62 @@+defmodule Plausible.ImportedTest do+ use Plausible.DataCase+ use Plausible++ alias Plausible.Imported++ describe "list_all_imports/1" do+ test "returns empty when there are no imports" do+ site = insert(:site)++ assert Imported.list_all_imports(site) == []+ end++ test "returns imports in various states" do+ site = insert(:site)++ _rogue_import = insert(:site_import)++ import1 = insert(:site_import, site: site, status: :pending)+ import2 = insert(:site_import, site: site, status: :importing)+ import3 = insert(:site_import, site: site, status: :completed)+ import4 = insert(:site_import, site: site, status: :failed)++ assert [%{id: id1}, %{id: id2}, %{id: id3}, %{id: id4}] = Imported.list_all_imports(site)++ ids = [id1, id2, id3, id4]++ assert import1.id in ids+ assert import2.id in ids+ assert import3.id in ids+ assert import4.id in ids+ end++ test "returns one legacy import when present with respective site import entry" do+ site = insert(:site)+ {:ok, opts} = add_imported_data(%{site: site})+ site = Map.new(opts).site+ site_import = insert(:site_import, site: site, legacy: true)+ site_import_id = site_import.id++ assert [%{id: ^site_import_id}] = Imported.list_all_imports(site)+ end++ test "returns legacy import without respective site import entry" do+ site = insert(:site)+ {:ok, opts} = add_imported_data(%{site: site})+ site = Map.new(opts).site+ imported_start_date = site.imported_data.start_date+ imported_end_date = site.imported_data.end_date++ assert [+ %{+ id: 0,+ source: :universal_analytics,+ start_date: ^imported_start_date,+ end_date: ^imported_end_date,+ status: :completed+ }+ ] = Imported.list_all_imports(site)+ end+ end+enddiff --git a/test/plausible/purge_test.exs b/test/plausible/purge_test.exsindex edb7d27fec..a06ba9061f 100644--- a/test/plausible/purge_test.exs+++ b/test/plausible/purge_test.exs@@ -89,6 +89,69 @@ defmodule Plausible.PurgeTest doend)end+ test "delete_imported_stats!/2 deletes legacy imported data only when instructed", %{+ site: site,+ site_import1: site_import1,+ site_import2: site_import2+ } do+ populate_stats(site, [+ build(:pageview),+ build(:imported_visitors),+ build(:imported_sources),+ build(:imported_pages),+ build(:imported_entry_pages),+ build(:imported_exit_pages),+ build(:imported_locations),+ build(:imported_devices),+ build(:imported_browsers),+ build(:imported_operating_systems)+ ])++ Enum.each(Plausible.Imported.tables(), fn table ->+ query =+ from(imported in table,+ where: imported.site_id == ^site.id and imported.import_id == ^site_import1.id+ )++ assert await_clickhouse_count(query, 1)++ query =+ from(imported in table,+ where: imported.site_id == ^site.id and imported.import_id == ^site_import2.id+ )++ assert await_clickhouse_count(query, 1)++ query =+ from(imported in table, where: imported.site_id == ^site.id and imported.import_id == 0)++ assert await_clickhouse_count(query, 1)+ end)++ assert :ok == Plausible.Purge.delete_imported_stats!(site, 0)++ Enum.each(Plausible.Imported.tables(), fn table ->+ query =+ from(imported in table,+ where: imported.site_id == ^site.id and imported.import_id == ^site_import1.id+ )++ assert await_clickhouse_count(query, 1)++ query =+ from(imported in table,+ where: imported.site_id == ^site.id and imported.import_id == ^site_import2.id+ )++ assert await_clickhouse_count(query, 1)++ query =+ from(imported in table, where: imported.site_id == ^site.id and imported.import_id == 0)++ assert await_clickhouse_count(query, 0)+ end)+ end+test "delete_imported_stats!/1 resets stats_start_date", %{site: site} doassert :ok == Plausible.Purge.delete_imported_stats!(site)assert %Plausible.Site{stats_start_date: nil} = Plausible.Repo.reload(site)diff --git a/test/plausible/site/sites_test.exs b/test/plausible/site/sites_test.exsindex 4898dbfcfe..e623ffdc57 100644--- a/test/plausible/site/sites_test.exs+++ b/test/plausible/site/sites_test.exs@@ -46,7 +46,7 @@ defmodule Plausible.SitesTest doassert Sites.stats_start_date(site) == nilend- test "is date if first pageview if site does have stats" do+ test "is date if site does have stats" dosite = insert(:site)populate_stats(site, [@@ -70,6 +70,33 @@ defmodule Plausible.SitesTest doendend+ describe "native_stats_start_date" do+ test "is nil if site has no stats" do+ site = insert(:site)++ assert Sites.native_stats_start_date(site) == nil+ end++ test "is date if site does have stats" do+ site = insert(:site)++ populate_stats(site, [+ build(:pageview)+ ])++ assert Sites.native_stats_start_date(site) == Timex.today(site.timezone)+ end++ test "ignores imported stats" do+ site = insert(:site)+ insert(:site_import, site: site)+ {:ok, opts} = add_imported_data(%{site: site})+ site = Map.new(opts).site++ assert Sites.native_stats_start_date(site) == nil+ end+ end+describe "has_stats?" dotest "is false if site has no stats" dosite = insert(:site)diff --git a/test/plausible/stats/clickhouse_test.exs b/test/plausible/stats/clickhouse_test.exsindex 3e36dbd7a1..3b553a4c00 100644--- a/test/plausible/stats/clickhouse_test.exs+++ b/test/plausible/stats/clickhouse_test.exs@@ -254,4 +254,38 @@ defmodule Plausible.Stats.ClickhouseTest do] = Clickhouse.top_sources_for_spike(site, query, 5, 1)endend++ describe "imported_pageview_counts/1" do+ test "gets pageview counts for each of sites' imports" do+ site = insert(:site)+ {:ok, opts} = add_imported_data(%{site: site})+ site = Map.new(opts).site++ import1 = insert(:site_import, site: site)+ import2 = insert(:site_import, site: site)++ # legacy import+ populate_stats(site, [+ build(:imported_visitors, pageviews: 5),+ build(:imported_visitors, pageviews: 6)+ ])++ populate_stats(site, import1.id, [+ build(:imported_visitors, pageviews: 6),+ build(:imported_visitors, pageviews: 8)+ ])++ populate_stats(site, import2.id, [+ build(:imported_visitors, pageviews: 7),+ build(:imported_visitors, pageviews: 13)+ ])++ pageview_counts = Clickhouse.imported_pageview_counts(site)++ assert map_size(pageview_counts) == 3+ assert pageview_counts[0] == 11+ assert pageview_counts[import1.id] == 14+ assert pageview_counts[import2.id] == 20+ end+ endenddiff --git a/test/plausible/imported/imported_test.exs b/test/plausible_web/controllers/api/stats_controller/imported_test.exssimilarity index 99%rename from test/plausible/imported/imported_test.exsrename to test/plausible_web/controllers/api/stats_controller/imported_test.exsindex 5084407a08..027db390c2 100644--- a/test/plausible/imported/imported_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/imported_test.exs@@ -1,4 +1,4 @@-defmodule Plausible.ImportedTest do+defmodule PlausibleWeb.Api.StatsController.ImportedTest douse PlausibleWeb.ConnCaseuse Timexdiff --git a/test/plausible_web/controllers/auth_controller_test.exs b/test/plausible_web/controllers/auth_controller_test.exsindex cb3d4cc1a5..461ccf2e1d 100644--- a/test/plausible_web/controllers/auth_controller_test.exs+++ b/test/plausible_web/controllers/auth_controller_test.exs@@ -1483,7 +1483,8 @@ defmodule PlausibleWeb.AuthControllerTest docallback_params = %{"error" => "access_denied", "state" => "[#{site.id},\"import\"]"}conn = get(conn, Routes.auth_path(conn, :google_auth_callback), callback_params)- assert redirected_to(conn, 302) == Routes.site_path(conn, :settings_general, site.domain)+ assert redirected_to(conn, 302) ==+ Routes.site_path(conn, :settings_integrations, site.domain)assert Phoenix.Flash.get(conn.assigns.flash, :error) =~"unable to authenticate your Google Analytics"diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exsindex 538929ac88..c580079ccb 100644--- a/test/plausible_web/controllers/site_controller_test.exs+++ b/test/plausible_web/controllers/site_controller_test.exs@@ -636,6 +636,46 @@ defmodule PlausibleWeb.SiteControllerTest doendend+ describe "GET /:website/settings/imports-exports" do+ setup [:create_user, :log_in, :create_site]++ test "renders empty imports list", %{conn: conn, site: site} do+ conn = get(conn, "/#{site.domain}/settings/imports-exports")+ resp = html_response(conn, 200)++ assert text_of_attr(resp, ~s|a[href]|, "href") =~+ "https://accounts.google.com/o/oauth2/"++ assert resp =~ "Import Data"+ assert resp =~ "Existing Imports"+ assert resp =~ "There are no imports yet"+ assert resp =~ "Export Data"+ end++ test "renders imports in import list", %{conn: conn, site: site} do+ {:ok, opts} = add_imported_data(%{site: site})+ site = Map.new(opts).site++ _site_import1 = insert(:site_import, site: site, status: SiteImport.pending())+ _site_import2 = insert(:site_import, site: site, status: SiteImport.importing())+ site_import3 = insert(:site_import, site: site, status: SiteImport.completed())+ _site_import4 = insert(:site_import, site: site, status: SiteImport.failed())++ populate_stats(site, site_import3.id, [+ build(:imported_visitors, pageviews: 77),+ build(:imported_visitors, pageviews: 21)+ ])++ conn = get(conn, "/#{site.domain}/settings/imports-exports")+ resp = html_response(conn, 200)++ buttons = find(resp, ~s|button[data-method="delete"]|)+ assert length(buttons) == 5++ assert resp =~ "(98 page views)"+ end+ end+describe "GET /:website/settings/integrations for self-hosting" dosetup [:create_user, :log_in, :create_site]@@ -1252,7 +1292,8 @@ defmodule PlausibleWeb.SiteControllerTest do|> get("/#{site.domain}/import/google-analytics/view-id", %{"access_token" => "token","refresh_token" => "foo",- "expires_at" => "2022-09-22T20:01:37.112777"+ "expires_at" => "2022-09-22T20:01:37.112777",+ "legacy" => "true"})|> html_response(200)@@ -1271,7 +1312,8 @@ defmodule PlausibleWeb.SiteControllerTest do"end_date" => "2022-03-01","access_token" => "token","refresh_token" => "foo",- "expires_at" => "2022-09-22T20:01:37.112777"+ "expires_at" => "2022-09-22T20:01:37.112777",+ "legacy" => "true"})[site_import] = Plausible.Imported.list_all_imports(site)@@ -1288,10 +1330,11 @@ defmodule PlausibleWeb.SiteControllerTest do"end_date" => "2022-03-01","access_token" => "token","refresh_token" => "foo",- "expires_at" => "2022-09-22T20:01:37.112777"+ "expires_at" => "2022-09-22T20:01:37.112777",+ "legacy" => "true"})- assert [%{id: import_id}] = Plausible.Imported.list_all_imports(site)+ assert [%{id: import_id, legacy: true}] = Plausible.Imported.list_all_imports(site)assert_enqueued(worker: Plausible.Workers.ImportAnalytics,@@ -1308,8 +1351,81 @@ defmodule PlausibleWeb.SiteControllerTest doendend- describe "DELETE /:website/settings/:forget_imported" do- setup [:create_user, :log_in, :create_new_site]+ describe "DELETE /:website/settings/:forget_import/:import_id" do+ setup [:create_user, :log_in, :create_new_site, :add_imported_data]++ test "removes site import, associated data and cancels oban job for a particular import", %{+ conn: conn,+ user: user,+ site: site+ } do+ {:ok, job} =+ Plausible.Imported.NoopImporter.new_import(+ site,+ user,+ start_date: ~D[2022-01-01],+ end_date: Timex.today()+ )++ %{args: %{import_id: import_id}} = job++ # legacy stats+ populate_stats(site, [+ build(:imported_visitors, pageviews: 12)+ ])++ populate_stats(site, import_id, [+ build(:imported_visitors, pageviews: 10)+ ])++ assert [%{id: ^import_id}, %{id: 0}] = Plausible.Imported.list_all_imports(site)++ assert eventually(fn ->+ count = Plausible.Stats.Clickhouse.imported_pageview_count(site)+ {count == 22, count}+ end)++ delete(conn, "/#{site.domain}/settings/forget-import/#{import_id}")++ assert eventually(fn ->+ count = Plausible.Stats.Clickhouse.imported_pageview_count(site)+ {count == 12, count}+ end)++ assert Repo.reload(job).state == "cancelled"+ end++ test "removes legacy site import along with associated data when instructed", %{+ conn: conn,+ site: site+ } do+ other_site_import = insert(:site_import, site: site)++ # legacy stats+ populate_stats(site, [+ build(:imported_visitors, pageviews: 12)+ ])++ populate_stats(site, other_site_import.id, [+ build(:imported_visitors, pageviews: 10)+ ])++ assert eventually(fn ->+ count = Plausible.Stats.Clickhouse.imported_pageview_count(site)+ {count == 22, count}+ end)++ delete(conn, "/#{site.domain}/settings/forget-import/0")++ assert eventually(fn ->+ count = Plausible.Stats.Clickhouse.imported_pageview_count(site)+ {count == 10, count}+ end)+ end+ end++ describe "DELETE /:website/settings/forget_imported" do+ setup [:create_user, :log_in, :create_new_site, :add_imported_data]test "removes imported_data field from site", %{conn: conn, site: site} dodelete(conn, "/#{site.domain}/settings/forget-imported")diff --git a/test/support/factory.ex b/test/support/factory.exindex a89478603e..afe3182dfa 100644--- a/test/support/factory.ex+++ b/test/support/factory.ex@@ -50,6 +50,20 @@ defmodule Plausible.Factory do}end+ def site_import_factory do+ today = Date.utc_today()++ %Plausible.Imported.SiteImport{+ site: build(:site),+ imported_by: build(:user),+ start_date: Date.add(today, -200),+ end_date: today,+ source: :universal_analytics,+ status: :completed,+ legacy: false+ }+ end+def ch_session_factory dohostname = sequence(:domain, &"example-#{&1}.com")diff --git a/test/test_helper.exs b/test/test_helper.exsindex 8d436f5a91..75e4312452 100644--- a/test/test_helper.exs+++ b/test/test_helper.exs@@ -2,6 +2,7 @@Mox.defmock(Plausible.HTTPClient.Mock, for: Plausible.HTTPClient.Interface)Application.ensure_all_started(:double)FunWithFlags.enable(:window_time_on_page)+FunWithFlags.enable(:imports_exports)Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual)if Mix.env() == :small_test dodiff --git a/test/workers/import_analytics_test.exs b/test/workers/import_analytics_test.exsindex fe7f1ad392..7ce1cce45a 100644--- a/test/workers/import_analytics_test.exs+++ b/test/workers/import_analytics_test.exs@@ -229,5 +229,26 @@ defmodule Plausible.Workers.ImportAnalyticsTest doassert_receive {:notification, :analytics_imports_jobs, %{"transient_fail" => ^import_id}}end)end++ test "sends oban notification to calling process on completion when listener setup separately",+ %{+ import_opts: import_opts+ } do+ Ecto.Adapters.SQL.Sandbox.unboxed_run(Plausible.Repo, fn ->+ user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))+ site = insert(:site, members: [user])++ {:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)+ import_id = job.args[:import_id]++ :ok = Plausible.Imported.Importer.listen()++ job+ |> Repo.reload!()+ |> ImportAnalytics.perform()++ assert_receive {:notification, :analytics_imports_jobs, %{"complete" => ^import_id}}+ end)+ endendend