Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Remove references to `site.imported_data` (#4006)
Adrian Gruntkowskic10580777ea8
diff --git a/lib/plausible/data_migration/site_imports.ex b/lib/plausible/data_migration/site_imports.exindex 205de86e17..b5e5f9cbd1 100644--- a/lib/plausible/data_migration/site_imports.ex+++ b/lib/plausible/data_migration/site_imports.ex@@ -58,7 +58,7 @@ defmodule Plausible.DataMigration.SiteImports doparams =site.imported_data- |> Imported.SiteImport.from_legacy()+ |> from_legacy()|> Map.put(:site_id, site.id)|> Map.take([:legacy, :start_date, :end_date, :source, :status, :site_id])@@ -216,4 +216,22 @@ defmodule Plausible.DataMigration.SiteImports doselect: %{max_date: fragment("max(?)", q.date)})end++ defp from_legacy(%Site.ImportedData{} = data) do+ status =+ case data.status do+ "ok" -> SiteImport.completed()+ "error" -> SiteImport.failed()+ _ -> SiteImport.importing()+ end++ %SiteImport{+ id: 0,+ legacy: true,+ start_date: data.start_date,+ end_date: data.end_date,+ source: :universal_analytics,+ status: status+ }+ endenddiff --git a/lib/plausible/imported.ex b/lib/plausible/imported.exindex 100667e480..c5f94e27a3 100644--- a/lib/plausible/imported.ex+++ b/lib/plausible/imported.ex@@ -67,20 +67,23 @@ defmodule Plausible.Imported doRepo.get_by(SiteImport, id: import_id, site_id: site.id)end+ @spec get_legacy_import(Site.t()) :: SiteImport.t() | nil+ def get_legacy_import(site) do+ from(i in SiteImport,+ where: i.site_id == ^site.id and i.legacy == true,+ order_by: [desc: i.updated_at],+ limit: 1+ )+ |> Repo.one()+ end+defdelegate listen(), to: Imported.Importer@spec list_all_imports(Site.t(), atom()) :: [SiteImport.t()]def list_all_imports(site, status \\ nil) do- imports =- from(i in SiteImport, where: i.site_id == ^site.id, order_by: [desc: i.inserted_at])- |> maybe_filter_by_status(status)- |> Repo.all()-- if site.imported_data && not Enum.any?(imports, & &1.legacy) do- imports ++ [SiteImport.from_legacy(site.imported_data)]- else- imports- end+ from(i in SiteImport, where: i.site_id == ^site.id, order_by: [desc: i.inserted_at])+ |> maybe_filter_by_status(status)+ |> Repo.all()end@spec other_imports_in_progress?(SiteImport.t()) :: boolean()@@ -113,7 +116,7 @@ defmodule Plausible.Imported doids = Enum.map(ids, &elem(&1, 1))# account for legacy imports as well- if has_legacy? || (site.imported_data && site.imported_data.status == "ok") do+ if has_legacy? do[0 | ids]elseids@@ -132,23 +135,7 @@ defmodule Plausible.Imported do)|> Repo.one()- dates = dates || %{start_date: nil, end_date: nil}-- if site.imported_data && site.imported_data.status == "ok" do- start_date =- [dates.start_date, site.imported_data.start_date]- |> Enum.reject(&is_nil/1)- |> Enum.min(Date, fn -> nil end)-- end_date =- [dates.end_date, site.imported_data.end_date]- |> Enum.reject(&is_nil/1)- |> Enum.max(Date, fn -> nil end)-- %{start_date: start_date, end_date: end_date}- else- dates- end+ dates || %{start_date: nil, end_date: nil}end@spec delete_imports_for_site(Site.t()) :: :okdiff --git a/lib/plausible/imported/site_import.ex b/lib/plausible/imported/site_import.exindex 9090bba41d..effcaae239 100644--- a/lib/plausible/imported/site_import.ex+++ b/lib/plausible/imported/site_import.ex@@ -33,34 +33,11 @@ defmodule Plausible.Imported.SiteImport dodefmacro unquote(status)(), do: unquote(status)end- @spec label(t() | Site.ImportedData.t()) :: String.t()- def label(%__MODULE__{source: source, label: label}) do+ @spec label(t()) :: String.t()+ def label(%{source: source, label: label}) dobuild_label(ImportSources.by_name(source).label(), label)end- # NOTE: this is necessary for backwards compatibility- # with legacy imports- def label(%Site.ImportedData{source: source}), do: build_label(source, nil)-- @spec from_legacy(Site.ImportedData.t()) :: t()- def from_legacy(%Site.ImportedData{} = data) do- status =- case data.status do- "ok" -> completed()- "error" -> failed()- _ -> importing()- end-- %__MODULE__{- id: 0,- legacy: true,- start_date: data.start_date,- end_date: data.end_date,- source: :universal_analytics,- status: status- }- end-@spec create_changeset(Site.t(), User.t(), map()) :: Ecto.Changeset.t()def create_changeset(site, user, params) do%__MODULE__{}diff --git a/lib/plausible/imported/universal_analytics.ex b/lib/plausible/imported/universal_analytics.exindex 1d0dcb2e8a..031d7a56c4 100644--- a/lib/plausible/imported/universal_analytics.ex+++ b/lib/plausible/imported/universal_analytics.ex@@ -5,8 +5,6 @@ defmodule Plausible.Imported.UniversalAnalytics douse Plausible.Imported.Importer- alias Plausible.Repo-@missing_values ["(none)", "(not set)", "(not provided)", "(other)"]@impl true@@ -18,49 +16,6 @@ defmodule Plausible.Imported.UniversalAnalytics do@impl truedef email_template(), do: "google_analytics_import.html"- @impl true- def before_start(site_import) do- if site_import.legacy do- site = Repo.preload(site_import, :site).site-- site- |> Plausible.Site.start_import(- site_import.start_date,- site_import.end_date,- label()- )- |> Repo.update!()- end-- :ok- end-- @impl true- def on_success(site_import, _extra_data) do- if site_import.legacy do- site = Repo.preload(site_import, :site).site-- site- |> Plausible.Site.import_success()- |> Repo.update!()- end-- :ok- end-- @impl true- def on_failure(site_import) do- if site_import.legacy do- site = Repo.preload(site_import, :site).site-- site- |> Plausible.Site.import_failure()- |> Repo.update!()- end-- :ok- end-@impl truedef parse_args(%{"view_id" => view_id, "start_date" => start_date, "end_date" => end_date} = argsdiff --git a/lib/plausible/purge.ex b/lib/plausible/purge.exindex 50fd3ed22e..aa072bade8 100644--- a/lib/plausible/purge.ex+++ b/lib/plausible/purge.ex@@ -60,6 +60,10 @@ defmodule Plausible.Purge dosite_import = Repo.preload(site_import, :site)delete_imported_stats!(site_import.site, site_import.id)+ if site_import.legacy do+ delete_imported_stats!(site_import.site, 0)+ end+:okenddiff --git a/lib/plausible/site.ex b/lib/plausible/site.exindex 6085bb5d08..45a5f47f79 100644--- a/lib/plausible/site.ex+++ b/lib/plausible/site.ex@@ -29,6 +29,7 @@ defmodule Plausible.Site dofield :domain_changed_from, :stringfield :domain_changed_at, :naive_datetime+ # NOTE: needed by `SiteImports` data migration scriptembeds_one :imported_data, Plausible.Site.ImportedData, on_replace: :updatemany_to_many :members, User, join_through: Plausible.Site.Membership@@ -153,31 +154,6 @@ defmodule Plausible.Site dochange(site, native_stats_start_at: val)end- def start_import(site, start_date, end_date, imported_source, status \\ "importing") do- change(site,- imported_data: %{- start_date: start_date,- end_date: end_date,- source: imported_source,- status: status- }- )- end-- def import_success(site) do- change(site,- imported_data: %{status: "ok"}- )- end-- def import_failure(site) do- change(site, imported_data: %{status: "error"})- end-- def remove_imported_data(site) do- change(site, imported_data: nil)- end-defp clean_domain(changeset) doclean_domain =(get_field(changeset, :domain) || "")diff --git a/lib/plausible/site/imported_data.ex b/lib/plausible/site/imported_data.exindex bec7e571e6..710cc80d44 100644--- a/lib/plausible/site/imported_data.ex+++ b/lib/plausible/site/imported_data.ex@@ -1,6 +1,8 @@defmodule Plausible.Site.ImportedData do@moduledoc """Embedded schema for analytics imports++ NOTE: needed by `SiteImports` data migration script"""use Ecto.Schemadiff --git a/lib/plausible_web/components/settings.ex b/lib/plausible_web/components/settings.exindex 34f457479c..ade376c1e0 100644--- a/lib/plausible_web/components/settings.ex+++ b/lib/plausible_web/components/settings.ex@@ -7,6 +7,10 @@ defmodule PlausibleWeb.Components.Settings doimport PlausibleWeb.Components.Generic+ alias Plausible.Imported.SiteImport++ require Plausible.Imported.SiteImport+embed_templates("../templates/site/settings_search_console.html")embed_templates("../templates/site/settings_google_import.html")enddiff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.exindex 06350bdeee..d71e9359a3 100644--- a/lib/plausible_web/controllers/site_controller.ex+++ b/lib/plausible_web/controllers/site_controller.ex@@ -237,8 +237,10 @@ defmodule PlausibleWeb.SiteController doPlausible.Google.API.fetch_verified_properties(site.google_auth)end+ legacy_import = Plausible.Imported.get_legacy_import(site)+imported_pageviews =- if site.imported_data do+ if legacy_import doPlausible.Stats.Clickhouse.imported_pageview_count(site)else0@@ -249,6 +251,7 @@ defmodule PlausibleWeb.SiteController doconn|> render("settings_integrations.html",site: site,+ legacy_import: legacy_import,imported_pageviews: imported_pageviews,has_plugins_tokens?: has_plugins_tokens?,search_console_domains: search_console_domains,@@ -646,34 +649,18 @@ defmodule PlausibleWeb.SiteController dodef forget_import(conn, %{"import_id" => import_id}) dosite = conn.assigns.site- cond do- import_id == "0" ->- Plausible.Purge.delete_imported_stats!(site, 0)-- site- |> Plausible.Site.remove_imported_data()- |> Repo.update!()-- site_import = Plausible.Imported.get_import(site, import_id) ->- Oban.cancel_all_jobs(- from(j in Oban.Job,- where:- j.queue == "analytics_imports" and- fragment("(? ->> 'import_id')::int", j.args) == ^site_import.id- )+ if site_import = Plausible.Imported.get_import(site, import_id) do+ Oban.cancel_all_jobs(+ from(j in Oban.Job,+ where:+ j.queue == "analytics_imports" and+ fragment("(? ->> 'import_id')::int", j.args) == ^site_import.id)+ )- Plausible.Purge.delete_imported_stats!(site_import)-- Plausible.Repo.delete!(site_import)-- if site_import.legacy do- Plausible.Purge.delete_imported_stats!(site, 0)+ Plausible.Purge.delete_imported_stats!(site_import)- site- |> Plausible.Site.remove_imported_data()- |> Repo.update!()- end+ Plausible.Repo.delete!(site_import)endconn@@ -701,10 +688,6 @@ defmodule PlausibleWeb.SiteController doPlausible.Purge.delete_imported_stats!(site)Plausible.Imported.delete_imports_for_site(site)-- site- |> Plausible.Site.remove_imported_data()- |> Repo.update!()endconndiff --git a/lib/plausible_web/live/imports_exports_settings.ex b/lib/plausible_web/live/imports_exports_settings.exindex 4e0851a75d..c1b83ed721 100644--- a/lib/plausible_web/live/imports_exports_settings.ex+++ b/lib/plausible_web/live/imports_exports_settings.ex@@ -137,7 +137,7 @@ defmodule PlausibleWeb.Live.ImportsExportsSettings do<%= Plausible.Imported.SiteImport.label(entry.site_import) %><span :if={entry.live_status == SiteImport.completed()} class="text-xs font-normal">(<%= PlausibleWeb.StatsView.large_number_format(- Map.get(@pageview_counts, entry.site_import.id, 0)+ pageview_count(entry.site_import, @pageview_counts)) %> page views)</span></p>@@ -183,6 +183,16 @@ defmodule PlausibleWeb.Live.ImportsExportsSettings do{:noreply, assign(socket, site_imports: site_imports, pageview_counts: pageview_counts)}end+ defp pageview_count(site_import, pageview_counts) do+ count = Map.get(pageview_counts, site_import.id, 0)++ if site_import.legacy do+ count + Map.get(pageview_counts, 0, 0)+ else+ count+ end+ end+defp update_imports(site_imports, import_id, status_str) doEnum.map_reduce(site_imports, false, fn%{site_import: %{id: ^import_id}} = entry, _changed? ->diff --git a/lib/plausible_web/templates/site/settings_google_import.html.heex b/lib/plausible_web/templates/site/settings_google_import.html.heexindex c7d76b99c8..bb6d2c4d6b 100644--- a/lib/plausible_web/templates/site/settings_google_import.html.heex+++ b/lib/plausible_web/templates/site/settings_google_import.html.heex@@ -11,11 +11,11 @@<%= if Keyword.get(Application.get_env(:plausible, :google), :client_id) do %><%= cond do %>- <% @site.imported_data && @site.imported_data.status == "importing" -> %>+ <% @legacy_import && @legacy_import.status in [SiteImport.pending(), SiteImport.importing()] -> %><li class="py-4 flex items-center justify-between space-x-4"><div class="flex flex-col"><p class="text-sm leading-5 font-medium text-gray-900 dark:text-gray-100">- Import from <%= @site.imported_data.source %>+ Import from Google Analytics<svgclass="animate-spin -mr-1 ml-1 h-4 w-4 inline text-indigo-600"xmlns="http://www.w3.org/2000/svg"@@ -40,8 +40,8 @@</svg></p><p class="text-sm leading-5 text-gray-500 dark:text-gray-200">- From <%= PlausibleWeb.EmailView.date_format(@site.imported_data.start_date) %> to <%= PlausibleWeb.EmailView.date_format(- @site.imported_data.end_date+ From <%= PlausibleWeb.EmailView.date_format(@legacy_import.start_date) %> to <%= PlausibleWeb.EmailView.date_format(+ @legacy_import.end_date) %></p></div>@@ -52,11 +52,11 @@"inline-block mt-4 px-4 py-2 border border-gray-300 dark:border-gray-500 text-sm leading-5 font-medium rounded-md text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:outline-none focus:border-blue-300 focus:ring active:text-red-800 active:bg-gray-50 transition ease-in-out duration-150") %></li>- <% @site.imported_data && @site.imported_data.status == "ok" -> %>+ <% @legacy_import && @legacy_import.status == SiteImport.completed() -> %><li class="py-4 flex items-center justify-between space-x-4"><div class="flex flex-col"><p class="text-sm leading-5 font-medium text-gray-900 dark:text-gray-100">- Import from <%= @site.imported_data.source %>+ Import from Google Analytics<svgclass="h-4 w-4 inline ml-1 -mt-1 text-green-600"xmlns="http://www.w3.org/2000/svg"@@ -69,8 +69,8 @@</svg></p><p class="text-sm leading-5 text-gray-500 dark:text-gray-200">- From <%= PlausibleWeb.EmailView.date_format(@site.imported_data.start_date) %> to <%= PlausibleWeb.EmailView.date_format(- @site.imported_data.end_date+ From <%= PlausibleWeb.EmailView.date_format(@legacy_import.start_date) %> to <%= PlausibleWeb.EmailView.date_format(+ @legacy_import.end_date) %></p></div>@@ -85,7 +85,7 @@) %></li><% true -> %>- <%= if @site.imported_data && @site.imported_data.status == "error" do %>+ <%= if @legacy_import && @legacy_import.status == SiteImport.failed() do %><div class="text-sm mt-2 text-gray-900 dark:text-gray-100">Your latest import has failed. You can try importing again by clicking the button below. If you try multiple times and the import keeps failing, please contact support.</div>diff --git a/lib/plausible_web/templates/site/settings_integrations.html.heex b/lib/plausible_web/templates/site/settings_integrations.html.heexindex adb56e0dde..bc10f7eef9 100644--- a/lib/plausible_web/templates/site/settings_integrations.html.heex+++ b/lib/plausible_web/templates/site/settings_integrations.html.heex@@ -7,6 +7,7 @@<PlausibleWeb.Components.Settings.settings_google_importsite={@site}imported_pageviews={@imported_pageviews}+ legacy_import={@legacy_import}/><% end %>diff --git a/lib/plausible_web/views/site_view.ex b/lib/plausible_web/views/site_view.exindex a3ae89f776..b394dbf4c0 100644--- a/lib/plausible_web/views/site_view.ex+++ b/lib/plausible_web/views/site_view.ex@@ -2,6 +2,10 @@ defmodule PlausibleWeb.SiteView douse PlausibleWeb, :viewuse Plausible+ alias Plausible.Imported.SiteImport++ require Plausible.Imported.SiteImport+def plausible_url doPlausibleWeb.Endpoint.url()enddiff --git a/priv/repo/seeds.exs b/priv/repo/seeds.exsindex db634bcfd6..75b0891762 100644--- a/priv/repo/seeds.exs+++ b/priv/repo/seeds.exs@@ -256,51 +256,6 @@ native_stats_rangeend)|> Plausible.TestUtils.populate_stats()-site =- site- |> Plausible.Site.start_import(- legacy_imported_stats_range.first,- legacy_imported_stats_range.last,- "Google Analytics"- )- |> Plausible.Repo.update!()--legacy_imported_stats_range-|> Enum.flat_map(fn date ->- Enum.flat_map(0..Enum.random(1..500), fn _ ->- [- Plausible.Factory.build(:imported_visitors,- date: date,- pageviews: Enum.random(1..20),- visitors: Enum.random(1..20),- bounces: Enum.random(1..20),- visits: Enum.random(1..200),- visit_duration: Enum.random(1000..10000)- ),- Plausible.Factory.build(:imported_sources,- date: date,- source: Enum.random(["", "Facebook", "Twitter", "DuckDuckGo", "Google"]),- visitors: Enum.random(1..20),- visits: Enum.random(1..200),- bounces: Enum.random(1..20),- visit_duration: Enum.random(1000..10000)- ),- Plausible.Factory.build(:imported_pages,- date: date,- visitors: Enum.random(1..20),- pageviews: Enum.random(1..20),- exits: Enum.random(1..20),- time_on_page: Enum.random(1000..10000)- )- ]- end)-end)-|> then(&Plausible.TestUtils.populate_stats(site, &1))--site-|> Plausible.Site.import_success()-|> Plausible.Repo.update!()-site_import =site|> Plausible.Imported.SiteImport.create_changeset(user, %{diff --git a/test/plausible/data_migration/site_imports_test.exs b/test/plausible/data_migration/site_imports_test.exsindex 83e586265f..397a5684c5 100644--- a/test/plausible/data_migration/site_imports_test.exs+++ b/test/plausible/data_migration/site_imports_test.exs@@ -6,7 +6,6 @@ defmodule Plausible.DataMigration.SiteImportsTest doalias Plausible.DataMigration.SiteImportsalias Plausible.Importedalias Plausible.Repo- alias Plausible.Sitedescribe "run/1" dotest "runs for empty dataset" do@@ -30,7 +29,7 @@ defmodule Plausible.DataMigration.SiteImportsTest dotest "adds site import entry when it's missing and adjusts end date" dosite =insert(:site)- |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")+ |> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")|> Repo.update!()populate_stats(site, 0, [@@ -54,7 +53,7 @@ defmodule Plausible.DataMigration.SiteImportsTest dotest "runs in dry mode without making any persistent changes" dosite =insert(:site)- |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")+ |> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")|> Repo.update!()populate_stats(site, 0, [@@ -71,15 +70,14 @@ defmodule Plausible.DataMigration.SiteImportsTest dosite = Repo.reload!(site)- assert [%{id: id, legacy: true}] = Imported.list_all_imports(site)- assert id == 0+ assert [] = Imported.list_all_imports(site)assert site.imported_data.end_date == ~D[2021-01-08]endtest "does not set end date to latter than the current one" dosite =insert(:site)- |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")+ |> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")|> Repo.update!()populate_stats(site, 0, [@@ -103,7 +101,7 @@ defmodule Plausible.DataMigration.SiteImportsTest dotest "removes site import when there are no stats" dosite =insert(:site)- |> Site.start_import(~D[2021-01-02], ~D[2020-02-02], "Google Analytics", "ok")+ |> start_import(~D[2021-01-02], ~D[2020-02-02], "Google Analytics", "ok")|> Repo.update!()assert capture_io(fn ->@@ -118,7 +116,7 @@ defmodule Plausible.DataMigration.SiteImportsTest dotest "leaves site and imports unchanged if everything fits" dosite =insert(:site)- |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")+ |> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")|> Repo.update!()existing_import =@@ -150,7 +148,7 @@ defmodule Plausible.DataMigration.SiteImportsTest dotest "only considers sites with completed site imports or 'ok' imported data" dosite1 =insert(:site)- |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")+ |> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")|> Repo.update!()existing_import1 =@@ -164,7 +162,7 @@ defmodule Plausible.DataMigration.SiteImportsTest dosite2 =insert(:site)- |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")+ |> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")|> Repo.update!()existing_import2 =@@ -178,12 +176,12 @@ defmodule Plausible.DataMigration.SiteImportsTest dosite3 =insert(:site)- |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")+ |> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")|> Repo.update!()site4 =insert(:site)- |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")+ |> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")|> Repo.update!()_existing_import3 =@@ -272,4 +270,15 @@ defmodule Plausible.DataMigration.SiteImportsTest doendendend++ defp start_import(site, start_date, end_date, imported_source, status) do+ change(site,+ imported_data: %{+ start_date: start_date,+ end_date: end_date,+ source: imported_source,+ status: status+ }+ )+ endenddiff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exsindex 9d7168f011..3e6bb55aa1 100644--- a/test/plausible/imported/csv_importer_test.exs+++ b/test/plausible/imported/csv_importer_test.exs@@ -70,8 +70,6 @@ defmodule Plausible.Imported.CSVImporterTest do}] = Plausible.Imported.list_all_imports(site)- assert %{imported_data: nil} = Repo.reload!(site)-assert CSVImporter.parse_args(args) == [uploads: uploads,storage: on_full_build(do: "s3", else: "local")diff --git a/test/plausible/imported/universal_analytics_test.exs b/test/plausible/imported/universal_analytics_test.exsindex 4fe543fc1b..aa2e53531b 100644--- a/test/plausible/imported/universal_analytics_test.exs+++ b/test/plausible/imported/universal_analytics_test.exs@@ -50,15 +50,6 @@ defmodule Plausible.Imported.UniversalAnalyticsTest do}] = Plausible.Imported.list_all_imports(site)- assert %{- imported_data: %{- source: "Google Analytics",- start_date: ~D[2023-10-01],- end_date: ~D[2024-01-02],- status: "importing"- }- } = Repo.reload!(site)-assert opts = [_ | _] = UniversalAnalytics.parse_args(args)assert opts[:view_id] == "123"@@ -91,8 +82,6 @@ defmodule Plausible.Imported.UniversalAnalyticsTest dolegacy: false}] = Plausible.Imported.list_all_imports(site)-- assert %{imported_data: nil} = Repo.reload!(site)endenddiff --git a/test/plausible/imported_test.exs b/test/plausible/imported_test.exsindex 37b989795a..49666f6523 100644--- a/test/plausible/imported_test.exs+++ b/test/plausible/imported_test.exs@@ -30,34 +30,6 @@ defmodule Plausible.ImportedTest doassert import3.id in idsassert import4.id in idsend-- test "returns one legacy import when present with respective site import entry" do- site = insert(:site)- {:ok, opts} = add_imported_data(%{site: site})- site = Map.new(opts).site- site_import = insert(:site_import, site: site, legacy: true)- site_import_id = site_import.id-- assert [%{id: ^site_import_id}] = Imported.list_all_imports(site)- end-- test "returns legacy import without respective site import entry" do- site = insert(:site)- {:ok, opts} = add_imported_data(%{site: site})- site = Map.new(opts).site- imported_start_date = site.imported_data.start_date- imported_end_date = site.imported_data.end_date-- assert [- %{- id: 0,- source: :universal_analytics,- start_date: ^imported_start_date,- end_date: ^imported_end_date,- status: :completed- }- ] = Imported.list_all_imports(site)- endenddescribe "get_imports_date_range/1" do@@ -68,15 +40,7 @@ defmodule Plausible.ImportedTest doendtest "returns empty when only incomplete or failed imports are present" do- site =- insert(:site)- |> Plausible.Site.start_import(- ~D[2020-04-01],- ~D[2022-06-22],- "Google Analytics",- "error"- )- |> Repo.update!()+ site = insert(:site)_import1 = insert(:site_import, site: site, status: :pending)_import2 = insert(:site_import, site: site, status: :importing)@@ -87,15 +51,7 @@ defmodule Plausible.ImportedTest doendtest "returns start and end dates considering all imports" do- site =- insert(:site)- |> Plausible.Site.start_import(- ~D[2020-04-01],- ~D[2022-06-22],- "Google Analytics",- "ok"- )- |> Repo.update!()+ site = insert(:site)_import1 =insert(:site_import,@@ -114,7 +70,7 @@ defmodule Plausible.ImportedTest dostatus: :completed)- assert %{start_date: ~D[2020-04-01], end_date: ~D[2024-01-08]} =+ assert %{start_date: ~D[2020-04-02], end_date: ~D[2024-01-08]} =Imported.get_imports_date_range(site)endenddiff --git a/test/plausible/site/sites_test.exs b/test/plausible/site/sites_test.exsindex e623ffdc57..b7c5d837df 100644--- a/test/plausible/site/sites_test.exs+++ b/test/plausible/site/sites_test.exs@@ -90,8 +90,6 @@ defmodule Plausible.SitesTest dotest "ignores imported stats" dosite = insert(:site)insert(:site_import, site: site)- {:ok, opts} = add_imported_data(%{site: site})- site = Map.new(opts).siteassert Sites.native_stats_start_date(site) == nilenddiff --git a/test/plausible/stats/clickhouse_test.exs b/test/plausible/stats/clickhouse_test.exsindex 3b553a4c00..7175d719f4 100644--- a/test/plausible/stats/clickhouse_test.exs+++ b/test/plausible/stats/clickhouse_test.exs@@ -258,8 +258,6 @@ defmodule Plausible.Stats.ClickhouseTest dodescribe "imported_pageview_counts/1" dotest "gets pageview counts for each of sites' imports" dosite = insert(:site)- {:ok, opts} = add_imported_data(%{site: site})- site = Map.new(opts).siteimport1 = insert(:site_import, site: site)import2 = insert(:site_import, site: site)diff --git a/test/plausible/stats/comparisons_test.exs b/test/plausible/stats/comparisons_test.exsindex d0875bf2db..9f72a35f96 100644--- a/test/plausible/stats/comparisons_test.exs+++ b/test/plausible/stats/comparisons_test.exs@@ -203,7 +203,7 @@ defmodule Plausible.Stats.ComparisonsTest doenddescribe "include_imported" do- setup [:create_user, :create_new_site, :add_imported_data]+ setup [:create_user, :create_new_site, :create_site_import]test "defaults to source_query.include_imported", %{site: site} doquery = Query.from(site, %{"period" => "day", "date" => "2023-01-01"})diff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exsindex 672638da9c..7ff466b583 100644--- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs@@ -473,10 +473,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doenddescribe "with imported data" do- setup :add_imported_data+ setup :create_site_import- test "does not count imported stats unless specified", %{conn: conn, site: site} do- populate_stats(site, [+ test "does not count imported stats unless specified", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [build(:imported_visitors, date: ~D[2023-01-01]),build(:pageview, timestamp: ~N[2023-01-01 00:00:00])])@@ -501,8 +505,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do}end- test "counts imported stats when comparing with previous period", %{conn: conn, site: site} do- populate_stats(site, [+ test "counts imported stats when comparing with previous period", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [build(:imported_visitors,visits: 2,bounces: 1,@@ -540,8 +548,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do}end- test "ignores imported data when filters are applied", %{conn: conn, site: site} do- populate_stats(site, [+ test "ignores imported data when filters are applied", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [build(:imported_visitors, date: ~D[2023-01-01]),build(:imported_sources, date: ~D[2023-01-01]),build(:pageview,@@ -566,8 +578,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do}end- test "events metric with imported data is disallowed", %{conn: conn, site: site} do- populate_stats(site, [+ test "events metric with imported data is disallowed", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [build(:imported_visitors, date: ~D[2023-01-01])])diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exsindex 8d91cf080f..178c7e9328 100644--- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs@@ -330,12 +330,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "breaks down all metrics by visit:referrer with imported data", %{conn: conn, site: site} do- site =- site- |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")- |> Plausible.Repo.update!()+ site_import =+ insert(:site_import,+ site: site,+ start_date: ~D[2005-01-01],+ end_date: Timex.today(),+ source: :universal_analytics+ )- populate_stats(site, [+ populate_stats(site, site_import.id, [build(:pageview, referrer: "site.com", timestamp: ~N[2021-01-01 00:00:00]),build(:pageview, referrer: "site.com/1", timestamp: ~N[2021-01-01 00:00:00]),build(:imported_sources,@@ -520,12 +523,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "breaks down all metrics by visit:utm_source with imported data", %{conn: conn, site: site} do- site =- site- |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")- |> Plausible.Repo.update!()+ site_import =+ insert(:site_import,+ site: site,+ start_date: ~D[2005-01-01],+ end_date: Timex.today(),+ source: :universal_analytics+ )- populate_stats(site, [+ populate_stats(site, site_import.id, [build(:pageview, utm_source: "SomeUTMSource", timestamp: ~N[2021-01-01 00:00:00]),build(:pageview, utm_source: "SomeUTMSource-1", timestamp: ~N[2021-01-01 00:00:00]),build(:imported_sources,@@ -842,12 +848,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest dotest "pageviews breakdown by event:page - imported data having pageviews=0 and visitors=n should be bypassed",%{conn: conn, site: site} do- site =- site- |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")- |> Plausible.Repo.update!()+ site_import =+ insert(:site_import,+ site: site,+ start_date: ~D[2005-01-01],+ end_date: Timex.today(),+ source: :universal_analytics+ )- populate_stats(site, [+ populate_stats(site, site_import.id, [build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),build(:pageview,@@ -2378,12 +2387,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest dodescribe "metrics" dotest "returns time_on_page with imported data", %{conn: conn, site: site} do- site =- site- |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")- |> Plausible.Repo.update!()+ site_import =+ insert(:site_import,+ site: site,+ start_date: ~D[2005-01-01],+ end_date: Timex.today(),+ source: :universal_analytics+ )- populate_stats(site, [+ populate_stats(site, site_import.id, [build(:imported_pages, page: "/A", time_on_page: 40, date: ~D[2021-01-01]),build(:imported_pages, page: "/A", time_on_page: 110, date: ~D[2021-01-01]),build(:imported_pages, page: "/B", time_on_page: 499, date: ~D[2021-01-01]),diff --git a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs b/test/plausible_web/controllers/api/stats_controller/browsers_test.exsindex 8f0486fa37..bd504c50ba 100644--- a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs@@ -2,7 +2,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest douse PlausibleWeb.ConnCasedescribe "GET /api/stats/:domain/browsers" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_site_import]test "returns top browsers by unique visitors", %{conn: conn, site: site} dopopulate_stats(site, [@@ -109,8 +109,12 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do]end- test "returns top browsers including imported data", %{conn: conn, site: site} do- populate_stats(site, [+ test "returns top browsers including imported data", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [build(:pageview, browser: "Chrome"),build(:imported_browsers, browser: "Chrome"),build(:imported_browsers, browser: "Firefox"),@@ -133,9 +137,10 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest dotest "skips breakdown when visitors=0 (possibly due to 'Enable Users Metric' in GA)", %{conn: conn,- site: site+ site: site,+ site_import: site_import} do- populate_stats(site, [+ populate_stats(site, site_import.id, [build(:imported_browsers, browser: "Chrome", visitors: 0, visits: 14),build(:imported_browsers, browser: "Firefox", visitors: 0, visits: 14),build(:imported_browsers,@@ -169,9 +174,10 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest dotest "select empty imported_browsers as (not set), merging with the native (not set)", %{conn: conn,- site: site+ site: site,+ site_import: site_import} do- populate_stats(site, [+ populate_stats(site, site_import.id, [build(:pageview, user_id: 123),build(:imported_browsers, visitors: 1),build(:imported_visitors, visitors: 1)@@ -186,7 +192,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest doenddescribe "GET /api/stats/:domain/browser-versions" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "returns correct conversion_rate when browser_version clashes across browsers", %{conn: conn,diff --git a/test/plausible_web/controllers/api/stats_controller/cities_test.exs b/test/plausible_web/controllers/api/stats_controller/cities_test.exsindex cb0b4f4661..903d6d625e 100644--- a/test/plausible_web/controllers/api/stats_controller/cities_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/cities_test.exs@@ -32,7 +32,7 @@ defmodule PlausibleWeb.Api.StatsController.CitiesTest do])end- setup [:create_user, :log_in, :create_new_site, :add_imported_data, :seed]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import, :seed]test "returns top cities by new visitors", %{conn: conn, site: site} doconn = get(conn, "/api/stats/#{site.domain}/cities?period=day")diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exsindex 1451233e6b..c594f660ea 100644--- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs@@ -631,12 +631,14 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest doconn: conn,site: site} do- site =- site- |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")- |> Plausible.Repo.update!()+ site_import =+ insert(:site_import,+ start_date: ~D[2005-01-01],+ end_date: Timex.today(),+ source: :universal_analytics+ )- populate_stats(site, [+ populate_stats(site, site_import.id, [build(:pageview, pathname: "/"),build(:pageview, pathname: "/another"),build(:pageview, pathname: "/blog/post-1"),diff --git a/test/plausible_web/controllers/api/stats_controller/countries_test.exs b/test/plausible_web/controllers/api/stats_controller/countries_test.exsindex 245e3a1f73..de616c28d1 100644--- a/test/plausible_web/controllers/api/stats_controller/countries_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/countries_test.exs@@ -2,7 +2,7 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest douse PlausibleWeb.ConnCasedescribe "GET /api/stats/:domain/countries" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "returns top countries by new visitors", %{conn: conn, site: site} dopopulate_stats(site, [diff --git a/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs b/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exsindex 25e76077bd..400e6d26ce 100644--- a/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs@@ -2,7 +2,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest douse PlausibleWeb.ConnCasedescribe "GET /api/stats/:domain/custom-prop-values/:prop_key" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "returns breakdown by a custom property", %{conn: conn, site: site} doprop_key = "parim_s6ber"diff --git a/test/plausible_web/controllers/api/stats_controller/imported_test.exs b/test/plausible_web/controllers/api/stats_controller/imported_test.exsindex e2f5c93c33..d460733e72 100644--- a/test/plausible_web/controllers/api/stats_controller/imported_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/imported_test.exs@@ -10,43 +10,30 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do|> then(&Plausible.Imported.Buffer.insert_all(table_name, &1))end- for import_type <- [:legacy, :new_and_legacy, :new] do+ for import_type <- [:new_and_legacy, :new] dodescribe "Parse and import third party data fetched from Google Analytics as #{import_type} import" do- if import_type == :new do- setup [:create_user, :log_in, :create_new_site]- else- setup [:create_user, :log_in, :create_new_site, :add_imported_data]- end+ setup [:create_user, :log_in, :create_new_site]setup %{user: user, site: site} do- if unquote(import_type) in [:new, :new_and_legacy] do- import_params =- if unquote(import_type) == :new_and_legacy do- site.imported_data- |> Map.from_struct()- |> Map.put(:source, :universal_analytics)- else- %{- source: :universal_analytics,- start_date: ~D[2005-01-01],- end_date: Timex.today()- }- end-- site_import =- site- |> Plausible.Imported.SiteImport.create_changeset(- user,- import_params- )- |> Plausible.Repo.insert!()- |> Plausible.Imported.SiteImport.complete_changeset()- |> Plausible.Repo.update!()-- {:ok, %{import_id: site_import.id}}- else- {:ok, %{import_id: 0}}- end+ import_params =+ %{+ source: :universal_analytics,+ start_date: ~D[2005-01-01],+ end_date: Timex.today(),+ legacy: unquote(import_type) == :new_and_legacy+ }++ site_import =+ site+ |> Plausible.Imported.SiteImport.create_changeset(+ user,+ import_params+ )+ |> Plausible.Repo.insert!()+ |> Plausible.Imported.SiteImport.complete_changeset()+ |> Plausible.Repo.update!()++ {:ok, %{import_id: site_import.id}}endtest "Visitors data imported from Google Analytics", %{diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exsindex deeff48857..b3cc866684 100644--- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs@@ -4,7 +4,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do@user_id 123describe "GET /api/stats/main-graph - plot" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "displays pageviews for the last 30 minutes in realtime graph", %{conn: conn, site: site} dopopulate_stats(site, [@@ -368,7 +368,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doenddescribe "GET /api/stats/main-graph - pageviews plot" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "displays pageviews for a month", %{conn: conn, site: site} dopopulate_stats(site, [@@ -556,7 +556,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doenddescribe "GET /api/stats/main-graph - bounce_rate plot" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "displays bounce_rate for a month", %{conn: conn, site: site} dopopulate_stats(site, [@@ -620,7 +620,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doenddescribe "GET /api/stats/main-graph - visit_duration plot" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "displays visit_duration for a month", %{conn: conn, site: site} dopopulate_stats(site, [@@ -915,7 +915,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doenddescribe "GET /api/stats/main-graph - comparisons" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "returns past month stats when period=30d and comparison=previous_period", %{conn: conn,@@ -1007,17 +1007,6 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest dobuild(:pageview, timestamp: ~N[2021-01-01 00:00:00])])- site- |> Ecto.Changeset.change(- imported_data: %{- start_date: ~D[2005-01-01],- end_date: ~D[2020-01-31],- source: "Google Analytics",- status: "ok"- }- )- |> Plausible.Repo.update!()-conn =get(conn,@@ -1064,17 +1053,6 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest dobuild(:pageview, timestamp: ~N[2021-01-01 00:00:00])])- site- |> Ecto.Changeset.change(- imported_data: %{- start_date: ~D[2005-01-01],- end_date: ~D[2020-01-31],- source: "Google Analytics",- status: "ok"- }- )- |> Plausible.Repo.update!()-conn =get(conn,@@ -1125,7 +1103,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do@tag :full_build_onlydescribe "GET /api/stats/main-graph - total_revenue plot" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "plots total_revenue for a month", %{conn: conn, site: site} doinsert(:goal, site: site, event_name: "Payment", currency: "USD")@@ -1205,7 +1183,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do@tag :full_build_onlydescribe "GET /api/stats/main-graph - average_revenue plot" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "plots total_revenue for a month", %{conn: conn, site: site} doinsert(:goal, site: site, event_name: "Payment", currency: "USD")diff --git a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exsindex 6a2f9d8127..d383fca24b 100644--- a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs@@ -2,7 +2,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest douse PlausibleWeb.ConnCasedescribe "GET /api/stats/:domain/operating-systems" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "returns operating systems by unique visitors", %{conn: conn, site: site} dopopulate_stats(site, [@@ -211,7 +211,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest doenddescribe "GET /api/stats/:domain/operating-system-versions" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "returns top OS versions by unique visitors", %{conn: conn, site: site} dopopulate_stats(site, [diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exsindex be2259ee8c..ea9ccfe6ae 100644--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs@@ -4,7 +4,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do@user_id 123describe "GET /api/stats/:domain/pages" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "returns top pages by visitors", %{conn: conn, site: site} dopopulate_stats(site, [@@ -1202,7 +1202,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest doenddescribe "GET /api/stats/:domain/entry-pages" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "returns top entry pages by visitors", %{conn: conn, site: site} dopopulate_stats(site, [@@ -1555,7 +1555,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest doenddescribe "GET /api/stats/:domain/exit-pages" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "returns top exit pages by visitors", %{conn: conn, site: site} dopopulate_stats(site, [diff --git a/test/plausible_web/controllers/api/stats_controller/regions_test.exs b/test/plausible_web/controllers/api/stats_controller/regions_test.exsindex c148cb2b39..23ac063bab 100644--- a/test/plausible_web/controllers/api/stats_controller/regions_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/regions_test.exs@@ -32,7 +32,7 @@ defmodule PlausibleWeb.Api.StatsController.RegionsTest do])end- setup [:create_user, :log_in, :create_new_site, :add_imported_data, :seed]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import, :seed]test "returns top cities by new visitors", %{conn: conn, site: site} doconn = get(conn, "/api/stats/#{site.domain}/regions?period=day")diff --git a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exsindex a25520be2f..128d1293df 100644--- a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs@@ -2,7 +2,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest douse PlausibleWeb.ConnCasedescribe "GET /api/stats/:domain/screen-sizes" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "returns screen sizes by new visitors", %{conn: conn, site: site} dopopulate_stats(site, [diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exsindex 8304732ee9..4d4481c4c0 100644--- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs@@ -4,7 +4,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do@user_id 123describe "GET /api/stats/:domain/sources" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "returns top sources by unique user ids", %{conn: conn, site: site} dopopulate_stats(site, [@@ -544,7 +544,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest doenddescribe "GET /api/stats/:domain/utm_mediums" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "returns top utm_mediums by unique user ids", %{conn: conn, site: site} dopopulate_stats(site, [@@ -696,7 +696,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest doenddescribe "GET /api/stats/:domain/utm_campaigns" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "returns top utm_campaigns by unique user ids", %{conn: conn, site: site} dopopulate_stats(site, [@@ -856,7 +856,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest doenddescribe "GET /api/stats/:domain/utm_sources" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "returns top utm_sources by unique user ids", %{conn: conn, site: site} dopopulate_stats(site, [@@ -904,7 +904,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest doenddescribe "GET /api/stats/:domain/utm_terms" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "returns top utm_terms by unique user ids", %{conn: conn, site: site} dopopulate_stats(site, [@@ -1064,7 +1064,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest doenddescribe "GET /api/stats/:domain/utm_contents" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "returns top utm_contents by unique user ids", %{conn: conn, site: site} dopopulate_stats(site, [diff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exsindex 911e099e80..46c7fb1f0e 100644--- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs@@ -463,7 +463,7 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest doenddescribe "GET /api/stats/top-stats - with imported data" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "returns divisible metrics as 0 when no stats exist", %{site: site,@@ -1267,7 +1267,7 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest doenddescribe "GET /api/stats/top-stats - with comparisons" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "does not return comparisons by default", %{site: site, conn: conn} dopopulate_stats(site, [@@ -1352,17 +1352,6 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest dobuild(:pageview, timestamp: ~N[2021-01-01 00:00:00])])- site- |> Ecto.Changeset.change(- imported_data: %{- start_date: ~D[2005-01-01],- end_date: ~D[2020-01-31],- source: "Google Analytics",- status: "ok"- }- )- |> Plausible.Repo.update!()-conn =get(conn,@@ -1393,17 +1382,6 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest dobuild(:pageview, timestamp: ~N[2021-01-01 00:00:00])])- site- |> Ecto.Changeset.change(- imported_data: %{- start_date: ~D[2005-01-01],- end_date: ~D[2020-01-31],- source: "Google Analytics",- status: "ok"- }- )- |> Plausible.Repo.update!()-conn =get(conn,diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exsindex 0ed74bc184..d6a876ff27 100644--- a/test/plausible_web/controllers/site_controller_test.exs+++ b/test/plausible_web/controllers/site_controller_test.exs@@ -655,9 +655,6 @@ defmodule PlausibleWeb.SiteControllerTest doendtest "renders imports in import list", %{conn: conn, site: site} do- {:ok, opts} = add_imported_data(%{site: site})- site = Map.new(opts).site-_site_import1 = insert(:site_import, site: site, status: SiteImport.pending())_site_import2 = insert(:site_import, site: site, status: SiteImport.importing())@@ -675,7 +672,7 @@ defmodule PlausibleWeb.SiteControllerTest doresp = html_response(conn, 200)buttons = find(resp, ~s|button[data-method="delete"]|)- assert length(buttons) == 5+ assert length(buttons) == 4assert resp =~ "Google Analytics (123456)"assert resp =~ "(98 page views)"@@ -693,6 +690,20 @@ defmodule PlausibleWeb.SiteControllerTest do"Maximum of #{Plausible.Imported.max_complete_imports()} imports is reached."end+ test "considers older legacy imports when showing pageview count", %{conn: conn, site: site} do+ _site_import =+ insert(:site_import, site: site, legacy: true, status: SiteImport.completed())++ populate_stats(site, [+ build(:imported_visitors, pageviews: 77),+ build(:imported_visitors, pageviews: 21)+ ])++ conn = get(conn, "/#{site.domain}/settings/imports-exports")++ assert html_response(conn, 200) =~ "(98 page views)"+ end+test "disables import buttons when there's import in progress", %{conn: conn, site: site} do_site_import1 = insert(:site_import, site: site, status: SiteImport.completed())_site_import2 = insert(:site_import, site: site, status: SiteImport.importing())@@ -1312,7 +1323,7 @@ defmodule PlausibleWeb.SiteControllerTest doenddescribe "DELETE /:website/settings/:forget_import/:import_id" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]test "removes site import, associated data and cancels oban job for a particular import", %{conn: conn,@@ -1338,7 +1349,11 @@ defmodule PlausibleWeb.SiteControllerTest dobuild(:imported_visitors, pageviews: 10)])- assert [%{id: ^import_id}, %{id: 0}] = Plausible.Imported.list_all_imports(site)+ imports = Plausible.Imported.list_all_imports(site)++ assert Enum.find(imports, &(&1.id == import_id))++ site = Plausible.Imported.load_import_data(site)assert eventually(fn ->count = Plausible.Stats.Clickhouse.imported_pageview_count(site)@@ -1355,9 +1370,10 @@ defmodule PlausibleWeb.SiteControllerTest doassert Repo.reload(job).state == "cancelled"end- test "removes legacy site import along with associated data when instructed", %{+ test "removes all legacy site import data when instructed", %{conn: conn,- site: site+ site: site,+ site_import: legacy_site_import} doother_site_import = insert(:site_import, site: site)@@ -1375,7 +1391,7 @@ defmodule PlausibleWeb.SiteControllerTest do{count == 22, count}end)- delete(conn, "/#{site.domain}/settings/forget-import/0")+ delete(conn, "/#{site.domain}/settings/forget-import/#{legacy_site_import.id}")assert eventually(fn ->count = Plausible.Stats.Clickhouse.imported_pageview_count(site)@@ -1385,13 +1401,7 @@ defmodule PlausibleWeb.SiteControllerTest doenddescribe "DELETE /:website/settings/forget_imported" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]-- test "removes imported_data field from site", %{conn: conn, site: site} do- delete(conn, "/#{site.domain}/settings/forget-imported")-- assert Repo.reload(site).imported_data == nil- end+ setup [:create_user, :log_in, :create_new_site]test "removes actual imported data from Clickhouse", %{conn: conn, user: user, site: site} doPlausible.Imported.NoopImporter.new_import(diff --git a/test/support/test_utils.ex b/test/support/test_utils.exindex 7ac7e579cf..4f3da9d647 100644--- a/test/support/test_utils.ex+++ b/test/support/test_utils.ex@@ -47,13 +47,21 @@ defmodule Plausible.TestUtils do{:ok, site: site}end- def add_imported_data(%{site: site}) do- site =- site- |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")- |> Repo.update!()+ def create_legacy_site_import(%{site: site}) do+ create_site_import(%{site: site, create_legacy_import?: true})+ end- {:ok, site: site}+ def create_site_import(%{site: site} = opts) do+ site_import =+ Factory.insert(:site_import,+ site: site,+ start_date: ~D[2005-01-01],+ end_date: Timex.today(),+ source: :universal_analytics,+ legacy: opts[:create_legacy_import?] == true+ )++ {:ok, site_import: site_import}enddef create_new_site(%{user: user}) do