Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Remove references to `site.imported_data` (#4006)

Adrian Gruntkowskic10580777ea8
diff --git a/lib/plausible/data_migration/site_imports.ex b/lib/plausible/data_migration/site_imports.ex
index 205de86e17..b5e5f9cbd1 100644
--- a/lib/plausible/data_migration/site_imports.ex
+++ b/lib/plausible/data_migration/site_imports.ex
@@ -58,7 +58,7 @@ defmodule Plausible.DataMigration.SiteImports do
params =
site.imported_data
- |> Imported.SiteImport.from_legacy()
+ |> from_legacy()
|> Map.put(:site_id, site.id)
|> Map.take([:legacy, :start_date, :end_date, :source, :status, :site_id])
@@ -216,4 +216,22 @@ defmodule Plausible.DataMigration.SiteImports do
select: %{max_date: fragment("max(?)", q.date)}
)
end
+
+ defp from_legacy(%Site.ImportedData{} = data) do
+ status =
+ case data.status do
+ "ok" -> SiteImport.completed()
+ "error" -> SiteImport.failed()
+ _ -> SiteImport.importing()
+ end
+
+ %SiteImport{
+ id: 0,
+ legacy: true,
+ start_date: data.start_date,
+ end_date: data.end_date,
+ source: :universal_analytics,
+ status: status
+ }
+ end
end
diff --git a/lib/plausible/imported.ex b/lib/plausible/imported.ex
index 100667e480..c5f94e27a3 100644
--- a/lib/plausible/imported.ex
+++ b/lib/plausible/imported.ex
@@ -67,20 +67,23 @@ defmodule Plausible.Imported do
Repo.get_by(SiteImport, id: import_id, site_id: site.id)
end
+ @spec get_legacy_import(Site.t()) :: SiteImport.t() | nil
+ def get_legacy_import(site) do
+ from(i in SiteImport,
+ where: i.site_id == ^site.id and i.legacy == true,
+ order_by: [desc: i.updated_at],
+ limit: 1
+ )
+ |> Repo.one()
+ end
+
defdelegate listen(), to: Imported.Importer
@spec list_all_imports(Site.t(), atom()) :: [SiteImport.t()]
def list_all_imports(site, status \\ nil) do
- imports =
- from(i in SiteImport, where: i.site_id == ^site.id, order_by: [desc: i.inserted_at])
- |> maybe_filter_by_status(status)
- |> Repo.all()
-
- if site.imported_data && not Enum.any?(imports, & &1.legacy) do
- imports ++ [SiteImport.from_legacy(site.imported_data)]
- else
- imports
- end
+ from(i in SiteImport, where: i.site_id == ^site.id, order_by: [desc: i.inserted_at])
+ |> maybe_filter_by_status(status)
+ |> Repo.all()
end
@spec other_imports_in_progress?(SiteImport.t()) :: boolean()
@@ -113,7 +116,7 @@ defmodule Plausible.Imported do
ids = Enum.map(ids, &elem(&1, 1))
# account for legacy imports as well
- if has_legacy? || (site.imported_data && site.imported_data.status == "ok") do
+ if has_legacy? do
[0 | ids]
else
ids
@@ -132,23 +135,7 @@ defmodule Plausible.Imported do
)
|> Repo.one()
- dates = dates || %{start_date: nil, end_date: nil}
-
- if site.imported_data && site.imported_data.status == "ok" do
- start_date =
- [dates.start_date, site.imported_data.start_date]
- |> Enum.reject(&is_nil/1)
- |> Enum.min(Date, fn -> nil end)
-
- end_date =
- [dates.end_date, site.imported_data.end_date]
- |> Enum.reject(&is_nil/1)
- |> Enum.max(Date, fn -> nil end)
-
- %{start_date: start_date, end_date: end_date}
- else
- dates
- end
+ dates || %{start_date: nil, end_date: nil}
end
@spec delete_imports_for_site(Site.t()) :: :ok
diff --git a/lib/plausible/imported/site_import.ex b/lib/plausible/imported/site_import.ex
index 9090bba41d..effcaae239 100644
--- a/lib/plausible/imported/site_import.ex
+++ b/lib/plausible/imported/site_import.ex
@@ -33,34 +33,11 @@ defmodule Plausible.Imported.SiteImport do
defmacro unquote(status)(), do: unquote(status)
end
- @spec label(t() | Site.ImportedData.t()) :: String.t()
- def label(%__MODULE__{source: source, label: label}) do
+ @spec label(t()) :: String.t()
+ def label(%{source: source, label: label}) do
build_label(ImportSources.by_name(source).label(), label)
end
- # NOTE: this is necessary for backwards compatibility
- # with legacy imports
- def label(%Site.ImportedData{source: source}), do: build_label(source, nil)
-
- @spec from_legacy(Site.ImportedData.t()) :: t()
- def from_legacy(%Site.ImportedData{} = data) do
- status =
- case data.status do
- "ok" -> completed()
- "error" -> failed()
- _ -> importing()
- end
-
- %__MODULE__{
- id: 0,
- legacy: true,
- start_date: data.start_date,
- end_date: data.end_date,
- source: :universal_analytics,
- status: status
- }
- end
-
@spec create_changeset(Site.t(), User.t(), map()) :: Ecto.Changeset.t()
def create_changeset(site, user, params) do
%__MODULE__{}
diff --git a/lib/plausible/imported/universal_analytics.ex b/lib/plausible/imported/universal_analytics.ex
index 1d0dcb2e8a..031d7a56c4 100644
--- a/lib/plausible/imported/universal_analytics.ex
+++ b/lib/plausible/imported/universal_analytics.ex
@@ -5,8 +5,6 @@ defmodule Plausible.Imported.UniversalAnalytics do
use Plausible.Imported.Importer
- alias Plausible.Repo
-
@missing_values ["(none)", "(not set)", "(not provided)", "(other)"]
@impl true
@@ -18,49 +16,6 @@ defmodule Plausible.Imported.UniversalAnalytics do
@impl true
def email_template(), do: "google_analytics_import.html"
- @impl true
- def before_start(site_import) do
- if site_import.legacy do
- site = Repo.preload(site_import, :site).site
-
- site
- |> Plausible.Site.start_import(
- site_import.start_date,
- site_import.end_date,
- label()
- )
- |> Repo.update!()
- end
-
- :ok
- end
-
- @impl true
- def on_success(site_import, _extra_data) do
- if site_import.legacy do
- site = Repo.preload(site_import, :site).site
-
- site
- |> Plausible.Site.import_success()
- |> Repo.update!()
- end
-
- :ok
- end
-
- @impl true
- def on_failure(site_import) do
- if site_import.legacy do
- site = Repo.preload(site_import, :site).site
-
- site
- |> Plausible.Site.import_failure()
- |> Repo.update!()
- end
-
- :ok
- end
-
@impl true
def parse_args(
%{"view_id" => view_id, "start_date" => start_date, "end_date" => end_date} = args
diff --git a/lib/plausible/purge.ex b/lib/plausible/purge.ex
index 50fd3ed22e..aa072bade8 100644
--- a/lib/plausible/purge.ex
+++ b/lib/plausible/purge.ex
@@ -60,6 +60,10 @@ defmodule Plausible.Purge do
site_import = Repo.preload(site_import, :site)
delete_imported_stats!(site_import.site, site_import.id)
+ if site_import.legacy do
+ delete_imported_stats!(site_import.site, 0)
+ end
+
:ok
end
diff --git a/lib/plausible/site.ex b/lib/plausible/site.ex
index 6085bb5d08..45a5f47f79 100644
--- a/lib/plausible/site.ex
+++ b/lib/plausible/site.ex
@@ -29,6 +29,7 @@ defmodule Plausible.Site do
field :domain_changed_from, :string
field :domain_changed_at, :naive_datetime
+ # NOTE: needed by `SiteImports` data migration script
embeds_one :imported_data, Plausible.Site.ImportedData, on_replace: :update
many_to_many :members, User, join_through: Plausible.Site.Membership
@@ -153,31 +154,6 @@ defmodule Plausible.Site do
change(site, native_stats_start_at: val)
end
- def start_import(site, start_date, end_date, imported_source, status \\ "importing") do
- change(site,
- imported_data: %{
- start_date: start_date,
- end_date: end_date,
- source: imported_source,
- status: status
- }
- )
- end
-
- def import_success(site) do
- change(site,
- imported_data: %{status: "ok"}
- )
- end
-
- def import_failure(site) do
- change(site, imported_data: %{status: "error"})
- end
-
- def remove_imported_data(site) do
- change(site, imported_data: nil)
- end
-
defp clean_domain(changeset) do
clean_domain =
(get_field(changeset, :domain) || "")
diff --git a/lib/plausible/site/imported_data.ex b/lib/plausible/site/imported_data.ex
index bec7e571e6..710cc80d44 100644
--- a/lib/plausible/site/imported_data.ex
+++ b/lib/plausible/site/imported_data.ex
@@ -1,6 +1,8 @@
defmodule Plausible.Site.ImportedData do
@moduledoc """
Embedded schema for analytics imports
+
+ NOTE: needed by `SiteImports` data migration script
"""
use Ecto.Schema
diff --git a/lib/plausible_web/components/settings.ex b/lib/plausible_web/components/settings.ex
index 34f457479c..ade376c1e0 100644
--- a/lib/plausible_web/components/settings.ex
+++ b/lib/plausible_web/components/settings.ex
@@ -7,6 +7,10 @@ defmodule PlausibleWeb.Components.Settings do
import PlausibleWeb.Components.Generic
+ alias Plausible.Imported.SiteImport
+
+ require Plausible.Imported.SiteImport
+
embed_templates("../templates/site/settings_search_console.html")
embed_templates("../templates/site/settings_google_import.html")
end
diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex
index 06350bdeee..d71e9359a3 100644
--- a/lib/plausible_web/controllers/site_controller.ex
+++ b/lib/plausible_web/controllers/site_controller.ex
@@ -237,8 +237,10 @@ defmodule PlausibleWeb.SiteController do
Plausible.Google.API.fetch_verified_properties(site.google_auth)
end
+ legacy_import = Plausible.Imported.get_legacy_import(site)
+
imported_pageviews =
- if site.imported_data do
+ if legacy_import do
Plausible.Stats.Clickhouse.imported_pageview_count(site)
else
0
@@ -249,6 +251,7 @@ defmodule PlausibleWeb.SiteController do
conn
|> render("settings_integrations.html",
site: site,
+ legacy_import: legacy_import,
imported_pageviews: imported_pageviews,
has_plugins_tokens?: has_plugins_tokens?,
search_console_domains: search_console_domains,
@@ -646,34 +649,18 @@ defmodule PlausibleWeb.SiteController do
def forget_import(conn, %{"import_id" => import_id}) do
site = conn.assigns.site
- cond do
- import_id == "0" ->
- Plausible.Purge.delete_imported_stats!(site, 0)
-
- site
- |> Plausible.Site.remove_imported_data()
- |> Repo.update!()
-
- site_import = Plausible.Imported.get_import(site, import_id) ->
- Oban.cancel_all_jobs(
- from(j in Oban.Job,
- where:
- j.queue == "analytics_imports" and
- fragment("(? ->> 'import_id')::int", j.args) == ^site_import.id
- )
+ if site_import = Plausible.Imported.get_import(site, import_id) do
+ Oban.cancel_all_jobs(
+ from(j in Oban.Job,
+ where:
+ j.queue == "analytics_imports" and
+ fragment("(? ->> 'import_id')::int", j.args) == ^site_import.id
)
+ )
- Plausible.Purge.delete_imported_stats!(site_import)
-
- Plausible.Repo.delete!(site_import)
-
- if site_import.legacy do
- Plausible.Purge.delete_imported_stats!(site, 0)
+ Plausible.Purge.delete_imported_stats!(site_import)
- site
- |> Plausible.Site.remove_imported_data()
- |> Repo.update!()
- end
+ Plausible.Repo.delete!(site_import)
end
conn
@@ -701,10 +688,6 @@ defmodule PlausibleWeb.SiteController do
Plausible.Purge.delete_imported_stats!(site)
Plausible.Imported.delete_imports_for_site(site)
-
- site
- |> Plausible.Site.remove_imported_data()
- |> Repo.update!()
end
conn
diff --git a/lib/plausible_web/live/imports_exports_settings.ex b/lib/plausible_web/live/imports_exports_settings.ex
index 4e0851a75d..c1b83ed721 100644
--- a/lib/plausible_web/live/imports_exports_settings.ex
+++ b/lib/plausible_web/live/imports_exports_settings.ex
@@ -137,7 +137,7 @@ defmodule PlausibleWeb.Live.ImportsExportsSettings do
<%= Plausible.Imported.SiteImport.label(entry.site_import) %>
<span :if={entry.live_status == SiteImport.completed()} class="text-xs font-normal">
(<%= PlausibleWeb.StatsView.large_number_format(
- Map.get(@pageview_counts, entry.site_import.id, 0)
+ pageview_count(entry.site_import, @pageview_counts)
) %> page views)
</span>
</p>
@@ -183,6 +183,16 @@ defmodule PlausibleWeb.Live.ImportsExportsSettings do
{:noreply, assign(socket, site_imports: site_imports, pageview_counts: pageview_counts)}
end
+ defp pageview_count(site_import, pageview_counts) do
+ count = Map.get(pageview_counts, site_import.id, 0)
+
+ if site_import.legacy do
+ count + Map.get(pageview_counts, 0, 0)
+ else
+ count
+ end
+ end
+
defp update_imports(site_imports, import_id, status_str) do
Enum.map_reduce(site_imports, false, fn
%{site_import: %{id: ^import_id}} = entry, _changed? ->
diff --git a/lib/plausible_web/templates/site/settings_google_import.html.heex b/lib/plausible_web/templates/site/settings_google_import.html.heex
index c7d76b99c8..bb6d2c4d6b 100644
--- a/lib/plausible_web/templates/site/settings_google_import.html.heex
+++ b/lib/plausible_web/templates/site/settings_google_import.html.heex
@@ -11,11 +11,11 @@
<%= if Keyword.get(Application.get_env(:plausible, :google), :client_id) do %>
<%= cond do %>
- <% @site.imported_data && @site.imported_data.status == "importing" -> %>
+ <% @legacy_import && @legacy_import.status in [SiteImport.pending(), SiteImport.importing()] -> %>
<li class="py-4 flex items-center justify-between space-x-4">
<div class="flex flex-col">
<p class="text-sm leading-5 font-medium text-gray-900 dark:text-gray-100">
- Import from <%= @site.imported_data.source %>
+ Import from Google Analytics
<svg
class="animate-spin -mr-1 ml-1 h-4 w-4 inline text-indigo-600"
xmlns="http://www.w3.org/2000/svg"
@@ -40,8 +40,8 @@
</svg>
</p>
<p class="text-sm leading-5 text-gray-500 dark:text-gray-200">
- From <%= PlausibleWeb.EmailView.date_format(@site.imported_data.start_date) %> to <%= PlausibleWeb.EmailView.date_format(
- @site.imported_data.end_date
+ From <%= PlausibleWeb.EmailView.date_format(@legacy_import.start_date) %> to <%= PlausibleWeb.EmailView.date_format(
+ @legacy_import.end_date
) %>
</p>
</div>
@@ -52,11 +52,11 @@
"inline-block mt-4 px-4 py-2 border border-gray-300 dark:border-gray-500 text-sm leading-5 font-medium rounded-md text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:outline-none focus:border-blue-300 focus:ring active:text-red-800 active:bg-gray-50 transition ease-in-out duration-150"
) %>
</li>
- <% @site.imported_data && @site.imported_data.status == "ok" -> %>
+ <% @legacy_import && @legacy_import.status == SiteImport.completed() -> %>
<li class="py-4 flex items-center justify-between space-x-4">
<div class="flex flex-col">
<p class="text-sm leading-5 font-medium text-gray-900 dark:text-gray-100">
- Import from <%= @site.imported_data.source %>
+ Import from Google Analytics
<svg
class="h-4 w-4 inline ml-1 -mt-1 text-green-600"
xmlns="http://www.w3.org/2000/svg"
@@ -69,8 +69,8 @@
</svg>
</p>
<p class="text-sm leading-5 text-gray-500 dark:text-gray-200">
- From <%= PlausibleWeb.EmailView.date_format(@site.imported_data.start_date) %> to <%= PlausibleWeb.EmailView.date_format(
- @site.imported_data.end_date
+ From <%= PlausibleWeb.EmailView.date_format(@legacy_import.start_date) %> to <%= PlausibleWeb.EmailView.date_format(
+ @legacy_import.end_date
) %>
</p>
</div>
@@ -85,7 +85,7 @@
) %>
</li>
<% true -> %>
- <%= if @site.imported_data && @site.imported_data.status == "error" do %>
+ <%= if @legacy_import && @legacy_import.status == SiteImport.failed() do %>
<div class="text-sm mt-2 text-gray-900 dark:text-gray-100">
Your latest import has failed. You can try importing again by clicking the button below. If you try multiple times and the import keeps failing, please contact support.
</div>
diff --git a/lib/plausible_web/templates/site/settings_integrations.html.heex b/lib/plausible_web/templates/site/settings_integrations.html.heex
index adb56e0dde..bc10f7eef9 100644
--- a/lib/plausible_web/templates/site/settings_integrations.html.heex
+++ b/lib/plausible_web/templates/site/settings_integrations.html.heex
@@ -7,6 +7,7 @@
<PlausibleWeb.Components.Settings.settings_google_import
site={@site}
imported_pageviews={@imported_pageviews}
+ legacy_import={@legacy_import}
/>
<% end %>
diff --git a/lib/plausible_web/views/site_view.ex b/lib/plausible_web/views/site_view.ex
index a3ae89f776..b394dbf4c0 100644
--- a/lib/plausible_web/views/site_view.ex
+++ b/lib/plausible_web/views/site_view.ex
@@ -2,6 +2,10 @@ defmodule PlausibleWeb.SiteView do
use PlausibleWeb, :view
use Plausible
+ alias Plausible.Imported.SiteImport
+
+ require Plausible.Imported.SiteImport
+
def plausible_url do
PlausibleWeb.Endpoint.url()
end
diff --git a/priv/repo/seeds.exs b/priv/repo/seeds.exs
index db634bcfd6..75b0891762 100644
--- a/priv/repo/seeds.exs
+++ b/priv/repo/seeds.exs
@@ -256,51 +256,6 @@ native_stats_range
end)
|> Plausible.TestUtils.populate_stats()
-site =
- site
- |> Plausible.Site.start_import(
- legacy_imported_stats_range.first,
- legacy_imported_stats_range.last,
- "Google Analytics"
- )
- |> Plausible.Repo.update!()
-
-legacy_imported_stats_range
-|> Enum.flat_map(fn date ->
- Enum.flat_map(0..Enum.random(1..500), fn _ ->
- [
- Plausible.Factory.build(:imported_visitors,
- date: date,
- pageviews: Enum.random(1..20),
- visitors: Enum.random(1..20),
- bounces: Enum.random(1..20),
- visits: Enum.random(1..200),
- visit_duration: Enum.random(1000..10000)
- ),
- Plausible.Factory.build(:imported_sources,
- date: date,
- source: Enum.random(["", "Facebook", "Twitter", "DuckDuckGo", "Google"]),
- visitors: Enum.random(1..20),
- visits: Enum.random(1..200),
- bounces: Enum.random(1..20),
- visit_duration: Enum.random(1000..10000)
- ),
- Plausible.Factory.build(:imported_pages,
- date: date,
- visitors: Enum.random(1..20),
- pageviews: Enum.random(1..20),
- exits: Enum.random(1..20),
- time_on_page: Enum.random(1000..10000)
- )
- ]
- end)
-end)
-|> then(&Plausible.TestUtils.populate_stats(site, &1))
-
-site
-|> Plausible.Site.import_success()
-|> Plausible.Repo.update!()
-
site_import =
site
|> Plausible.Imported.SiteImport.create_changeset(user, %{
diff --git a/test/plausible/data_migration/site_imports_test.exs b/test/plausible/data_migration/site_imports_test.exs
index 83e586265f..397a5684c5 100644
--- a/test/plausible/data_migration/site_imports_test.exs
+++ b/test/plausible/data_migration/site_imports_test.exs
@@ -6,7 +6,6 @@ defmodule Plausible.DataMigration.SiteImportsTest do
alias Plausible.DataMigration.SiteImports
alias Plausible.Imported
alias Plausible.Repo
- alias Plausible.Site
describe "run/1" do
test "runs for empty dataset" do
@@ -30,7 +29,7 @@ defmodule Plausible.DataMigration.SiteImportsTest do
test "adds site import entry when it's missing and adjusts end date" do
site =
insert(:site)
- |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
+ |> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|> Repo.update!()
populate_stats(site, 0, [
@@ -54,7 +53,7 @@ defmodule Plausible.DataMigration.SiteImportsTest do
test "runs in dry mode without making any persistent changes" do
site =
insert(:site)
- |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
+ |> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|> Repo.update!()
populate_stats(site, 0, [
@@ -71,15 +70,14 @@ defmodule Plausible.DataMigration.SiteImportsTest do
site = Repo.reload!(site)
- assert [%{id: id, legacy: true}] = Imported.list_all_imports(site)
- assert id == 0
+ assert [] = Imported.list_all_imports(site)
assert site.imported_data.end_date == ~D[2021-01-08]
end
test "does not set end date to latter than the current one" do
site =
insert(:site)
- |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
+ |> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|> Repo.update!()
populate_stats(site, 0, [
@@ -103,7 +101,7 @@ defmodule Plausible.DataMigration.SiteImportsTest do
test "removes site import when there are no stats" do
site =
insert(:site)
- |> Site.start_import(~D[2021-01-02], ~D[2020-02-02], "Google Analytics", "ok")
+ |> start_import(~D[2021-01-02], ~D[2020-02-02], "Google Analytics", "ok")
|> Repo.update!()
assert capture_io(fn ->
@@ -118,7 +116,7 @@ defmodule Plausible.DataMigration.SiteImportsTest do
test "leaves site and imports unchanged if everything fits" do
site =
insert(:site)
- |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
+ |> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|> Repo.update!()
existing_import =
@@ -150,7 +148,7 @@ defmodule Plausible.DataMigration.SiteImportsTest do
test "only considers sites with completed site imports or 'ok' imported data" do
site1 =
insert(:site)
- |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")
+ |> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")
|> Repo.update!()
existing_import1 =
@@ -164,7 +162,7 @@ defmodule Plausible.DataMigration.SiteImportsTest do
site2 =
insert(:site)
- |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
+ |> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|> Repo.update!()
existing_import2 =
@@ -178,12 +176,12 @@ defmodule Plausible.DataMigration.SiteImportsTest do
site3 =
insert(:site)
- |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")
+ |> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")
|> Repo.update!()
site4 =
insert(:site)
- |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")
+ |> start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")
|> Repo.update!()
_existing_import3 =
@@ -272,4 +270,15 @@ defmodule Plausible.DataMigration.SiteImportsTest do
end
end
end
+
+ defp start_import(site, start_date, end_date, imported_source, status) do
+ change(site,
+ imported_data: %{
+ start_date: start_date,
+ end_date: end_date,
+ source: imported_source,
+ status: status
+ }
+ )
+ end
end
diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exs
index 9d7168f011..3e6bb55aa1 100644
--- a/test/plausible/imported/csv_importer_test.exs
+++ b/test/plausible/imported/csv_importer_test.exs
@@ -70,8 +70,6 @@ defmodule Plausible.Imported.CSVImporterTest do
}
] = Plausible.Imported.list_all_imports(site)
- assert %{imported_data: nil} = Repo.reload!(site)
-
assert CSVImporter.parse_args(args) == [
uploads: uploads,
storage: on_full_build(do: "s3", else: "local")
diff --git a/test/plausible/imported/universal_analytics_test.exs b/test/plausible/imported/universal_analytics_test.exs
index 4fe543fc1b..aa2e53531b 100644
--- a/test/plausible/imported/universal_analytics_test.exs
+++ b/test/plausible/imported/universal_analytics_test.exs
@@ -50,15 +50,6 @@ defmodule Plausible.Imported.UniversalAnalyticsTest do
}
] = Plausible.Imported.list_all_imports(site)
- assert %{
- imported_data: %{
- source: "Google Analytics",
- start_date: ~D[2023-10-01],
- end_date: ~D[2024-01-02],
- status: "importing"
- }
- } = Repo.reload!(site)
-
assert opts = [_ | _] = UniversalAnalytics.parse_args(args)
assert opts[:view_id] == "123"
@@ -91,8 +82,6 @@ defmodule Plausible.Imported.UniversalAnalyticsTest do
legacy: false
}
] = Plausible.Imported.list_all_imports(site)
-
- assert %{imported_data: nil} = Repo.reload!(site)
end
end
diff --git a/test/plausible/imported_test.exs b/test/plausible/imported_test.exs
index 37b989795a..49666f6523 100644
--- a/test/plausible/imported_test.exs
+++ b/test/plausible/imported_test.exs
@@ -30,34 +30,6 @@ defmodule Plausible.ImportedTest do
assert import3.id in ids
assert import4.id in ids
end
-
- test "returns one legacy import when present with respective site import entry" do
- site = insert(:site)
- {:ok, opts} = add_imported_data(%{site: site})
- site = Map.new(opts).site
- site_import = insert(:site_import, site: site, legacy: true)
- site_import_id = site_import.id
-
- assert [%{id: ^site_import_id}] = Imported.list_all_imports(site)
- end
-
- test "returns legacy import without respective site import entry" do
- site = insert(:site)
- {:ok, opts} = add_imported_data(%{site: site})
- site = Map.new(opts).site
- imported_start_date = site.imported_data.start_date
- imported_end_date = site.imported_data.end_date
-
- assert [
- %{
- id: 0,
- source: :universal_analytics,
- start_date: ^imported_start_date,
- end_date: ^imported_end_date,
- status: :completed
- }
- ] = Imported.list_all_imports(site)
- end
end
describe "get_imports_date_range/1" do
@@ -68,15 +40,7 @@ defmodule Plausible.ImportedTest do
end
test "returns empty when only incomplete or failed imports are present" do
- site =
- insert(:site)
- |> Plausible.Site.start_import(
- ~D[2020-04-01],
- ~D[2022-06-22],
- "Google Analytics",
- "error"
- )
- |> Repo.update!()
+ site = insert(:site)
_import1 = insert(:site_import, site: site, status: :pending)
_import2 = insert(:site_import, site: site, status: :importing)
@@ -87,15 +51,7 @@ defmodule Plausible.ImportedTest do
end
test "returns start and end dates considering all imports" do
- site =
- insert(:site)
- |> Plausible.Site.start_import(
- ~D[2020-04-01],
- ~D[2022-06-22],
- "Google Analytics",
- "ok"
- )
- |> Repo.update!()
+ site = insert(:site)
_import1 =
insert(:site_import,
@@ -114,7 +70,7 @@ defmodule Plausible.ImportedTest do
status: :completed
)
- assert %{start_date: ~D[2020-04-01], end_date: ~D[2024-01-08]} =
+ assert %{start_date: ~D[2020-04-02], end_date: ~D[2024-01-08]} =
Imported.get_imports_date_range(site)
end
end
diff --git a/test/plausible/site/sites_test.exs b/test/plausible/site/sites_test.exs
index e623ffdc57..b7c5d837df 100644
--- a/test/plausible/site/sites_test.exs
+++ b/test/plausible/site/sites_test.exs
@@ -90,8 +90,6 @@ defmodule Plausible.SitesTest do
test "ignores imported stats" do
site = insert(:site)
insert(:site_import, site: site)
- {:ok, opts} = add_imported_data(%{site: site})
- site = Map.new(opts).site
assert Sites.native_stats_start_date(site) == nil
end
diff --git a/test/plausible/stats/clickhouse_test.exs b/test/plausible/stats/clickhouse_test.exs
index 3b553a4c00..7175d719f4 100644
--- a/test/plausible/stats/clickhouse_test.exs
+++ b/test/plausible/stats/clickhouse_test.exs
@@ -258,8 +258,6 @@ defmodule Plausible.Stats.ClickhouseTest do
describe "imported_pageview_counts/1" do
test "gets pageview counts for each of sites' imports" do
site = insert(:site)
- {:ok, opts} = add_imported_data(%{site: site})
- site = Map.new(opts).site
import1 = insert(:site_import, site: site)
import2 = insert(:site_import, site: site)
diff --git a/test/plausible/stats/comparisons_test.exs b/test/plausible/stats/comparisons_test.exs
index d0875bf2db..9f72a35f96 100644
--- a/test/plausible/stats/comparisons_test.exs
+++ b/test/plausible/stats/comparisons_test.exs
@@ -203,7 +203,7 @@ defmodule Plausible.Stats.ComparisonsTest do
end
describe "include_imported" do
- setup [:create_user, :create_new_site, :add_imported_data]
+ setup [:create_user, :create_new_site, :create_site_import]
test "defaults to source_query.include_imported", %{site: site} do
query = Query.from(site, %{"period" => "day", "date" => "2023-01-01"})
diff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
index 672638da9c..7ff466b583 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
@@ -473,10 +473,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
describe "with imported data" do
- setup :add_imported_data
+ setup :create_site_import
- test "does not count imported stats unless specified", %{conn: conn, site: site} do
- populate_stats(site, [
+ test "does not count imported stats unless specified", %{
+ conn: conn,
+ site: site,
+ site_import: site_import
+ } do
+ populate_stats(site, site_import.id, [
build(:imported_visitors, date: ~D[2023-01-01]),
build(:pageview, timestamp: ~N[2023-01-01 00:00:00])
])
@@ -501,8 +505,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
}
end
- test "counts imported stats when comparing with previous period", %{conn: conn, site: site} do
- populate_stats(site, [
+ test "counts imported stats when comparing with previous period", %{
+ conn: conn,
+ site: site,
+ site_import: site_import
+ } do
+ populate_stats(site, site_import.id, [
build(:imported_visitors,
visits: 2,
bounces: 1,
@@ -540,8 +548,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
}
end
- test "ignores imported data when filters are applied", %{conn: conn, site: site} do
- populate_stats(site, [
+ test "ignores imported data when filters are applied", %{
+ conn: conn,
+ site: site,
+ site_import: site_import
+ } do
+ populate_stats(site, site_import.id, [
build(:imported_visitors, date: ~D[2023-01-01]),
build(:imported_sources, date: ~D[2023-01-01]),
build(:pageview,
@@ -566,8 +578,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
}
end
- test "events metric with imported data is disallowed", %{conn: conn, site: site} do
- populate_stats(site, [
+ test "events metric with imported data is disallowed", %{
+ conn: conn,
+ site: site,
+ site_import: site_import
+ } do
+ populate_stats(site, site_import.id, [
build(:imported_visitors, date: ~D[2023-01-01])
])
diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
index 8d91cf080f..178c7e9328 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
@@ -330,12 +330,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breaks down all metrics by visit:referrer with imported data", %{conn: conn, site: site} do
- site =
- site
- |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")
- |> Plausible.Repo.update!()
+ site_import =
+ insert(:site_import,
+ site: site,
+ start_date: ~D[2005-01-01],
+ end_date: Timex.today(),
+ source: :universal_analytics
+ )
- populate_stats(site, [
+ populate_stats(site, site_import.id, [
build(:pageview, referrer: "site.com", timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, referrer: "site.com/1", timestamp: ~N[2021-01-01 00:00:00]),
build(:imported_sources,
@@ -520,12 +523,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breaks down all metrics by visit:utm_source with imported data", %{conn: conn, site: site} do
- site =
- site
- |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")
- |> Plausible.Repo.update!()
+ site_import =
+ insert(:site_import,
+ site: site,
+ start_date: ~D[2005-01-01],
+ end_date: Timex.today(),
+ source: :universal_analytics
+ )
- populate_stats(site, [
+ populate_stats(site, site_import.id, [
build(:pageview, utm_source: "SomeUTMSource", timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, utm_source: "SomeUTMSource-1", timestamp: ~N[2021-01-01 00:00:00]),
build(:imported_sources,
@@ -842,12 +848,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
test "pageviews breakdown by event:page - imported data having pageviews=0 and visitors=n should be bypassed",
%{conn: conn, site: site} do
- site =
- site
- |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")
- |> Plausible.Repo.update!()
+ site_import =
+ insert(:site_import,
+ site: site,
+ start_date: ~D[2005-01-01],
+ end_date: Timex.today(),
+ source: :universal_analytics
+ )
- populate_stats(site, [
+ populate_stats(site, site_import.id, [
build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview,
@@ -2378,12 +2387,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
describe "metrics" do
test "returns time_on_page with imported data", %{conn: conn, site: site} do
- site =
- site
- |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")
- |> Plausible.Repo.update!()
+ site_import =
+ insert(:site_import,
+ site: site,
+ start_date: ~D[2005-01-01],
+ end_date: Timex.today(),
+ source: :universal_analytics
+ )
- populate_stats(site, [
+ populate_stats(site, site_import.id, [
build(:imported_pages, page: "/A", time_on_page: 40, date: ~D[2021-01-01]),
build(:imported_pages, page: "/A", time_on_page: 110, date: ~D[2021-01-01]),
build(:imported_pages, page: "/B", time_on_page: 499, date: ~D[2021-01-01]),
diff --git a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs
index 8f0486fa37..bd504c50ba 100644
--- a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs
@@ -2,7 +2,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
use PlausibleWeb.ConnCase
describe "GET /api/stats/:domain/browsers" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_site_import]
test "returns top browsers by unique visitors", %{conn: conn, site: site} do
populate_stats(site, [
@@ -109,8 +109,12 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
]
end
- test "returns top browsers including imported data", %{conn: conn, site: site} do
- populate_stats(site, [
+ test "returns top browsers including imported data", %{
+ conn: conn,
+ site: site,
+ site_import: site_import
+ } do
+ populate_stats(site, site_import.id, [
build(:pageview, browser: "Chrome"),
build(:imported_browsers, browser: "Chrome"),
build(:imported_browsers, browser: "Firefox"),
@@ -133,9 +137,10 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
test "skips breakdown when visitors=0 (possibly due to 'Enable Users Metric' in GA)", %{
conn: conn,
- site: site
+ site: site,
+ site_import: site_import
} do
- populate_stats(site, [
+ populate_stats(site, site_import.id, [
build(:imported_browsers, browser: "Chrome", visitors: 0, visits: 14),
build(:imported_browsers, browser: "Firefox", visitors: 0, visits: 14),
build(:imported_browsers,
@@ -169,9 +174,10 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
test "select empty imported_browsers as (not set), merging with the native (not set)", %{
conn: conn,
- site: site
+ site: site,
+ site_import: site_import
} do
- populate_stats(site, [
+ populate_stats(site, site_import.id, [
build(:pageview, user_id: 123),
build(:imported_browsers, visitors: 1),
build(:imported_visitors, visitors: 1)
@@ -186,7 +192,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
end
describe "GET /api/stats/:domain/browser-versions" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns correct conversion_rate when browser_version clashes across browsers", %{
conn: conn,
diff --git a/test/plausible_web/controllers/api/stats_controller/cities_test.exs b/test/plausible_web/controllers/api/stats_controller/cities_test.exs
index cb0b4f4661..903d6d625e 100644
--- a/test/plausible_web/controllers/api/stats_controller/cities_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/cities_test.exs
@@ -32,7 +32,7 @@ defmodule PlausibleWeb.Api.StatsController.CitiesTest do
])
end
- setup [:create_user, :log_in, :create_new_site, :add_imported_data, :seed]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import, :seed]
test "returns top cities by new visitors", %{conn: conn, site: site} do
conn = get(conn, "/api/stats/#{site.domain}/cities?period=day")
diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
index 1451233e6b..c594f660ea 100644
--- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
@@ -631,12 +631,14 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
conn: conn,
site: site
} do
- site =
- site
- |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")
- |> Plausible.Repo.update!()
+ site_import =
+ insert(:site_import,
+ start_date: ~D[2005-01-01],
+ end_date: Timex.today(),
+ source: :universal_analytics
+ )
- populate_stats(site, [
+ populate_stats(site, site_import.id, [
build(:pageview, pathname: "/"),
build(:pageview, pathname: "/another"),
build(:pageview, pathname: "/blog/post-1"),
diff --git a/test/plausible_web/controllers/api/stats_controller/countries_test.exs b/test/plausible_web/controllers/api/stats_controller/countries_test.exs
index 245e3a1f73..de616c28d1 100644
--- a/test/plausible_web/controllers/api/stats_controller/countries_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/countries_test.exs
@@ -2,7 +2,7 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do
use PlausibleWeb.ConnCase
describe "GET /api/stats/:domain/countries" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns top countries by new visitors", %{conn: conn, site: site} do
populate_stats(site, [
diff --git a/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs b/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs
index 25e76077bd..400e6d26ce 100644
--- a/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs
@@ -2,7 +2,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
use PlausibleWeb.ConnCase
describe "GET /api/stats/:domain/custom-prop-values/:prop_key" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns breakdown by a custom property", %{conn: conn, site: site} do
prop_key = "parim_s6ber"
diff --git a/test/plausible_web/controllers/api/stats_controller/imported_test.exs b/test/plausible_web/controllers/api/stats_controller/imported_test.exs
index e2f5c93c33..d460733e72 100644
--- a/test/plausible_web/controllers/api/stats_controller/imported_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/imported_test.exs
@@ -10,43 +10,30 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
|> then(&Plausible.Imported.Buffer.insert_all(table_name, &1))
end
- for import_type <- [:legacy, :new_and_legacy, :new] do
+ for import_type <- [:new_and_legacy, :new] do
describe "Parse and import third party data fetched from Google Analytics as #{import_type} import" do
- if import_type == :new do
- setup [:create_user, :log_in, :create_new_site]
- else
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
- end
+ setup [:create_user, :log_in, :create_new_site]
setup %{user: user, site: site} do
- if unquote(import_type) in [:new, :new_and_legacy] do
- import_params =
- if unquote(import_type) == :new_and_legacy do
- site.imported_data
- |> Map.from_struct()
- |> Map.put(:source, :universal_analytics)
- else
- %{
- source: :universal_analytics,
- start_date: ~D[2005-01-01],
- end_date: Timex.today()
- }
- end
-
- site_import =
- site
- |> Plausible.Imported.SiteImport.create_changeset(
- user,
- import_params
- )
- |> Plausible.Repo.insert!()
- |> Plausible.Imported.SiteImport.complete_changeset()
- |> Plausible.Repo.update!()
-
- {:ok, %{import_id: site_import.id}}
- else
- {:ok, %{import_id: 0}}
- end
+ import_params =
+ %{
+ source: :universal_analytics,
+ start_date: ~D[2005-01-01],
+ end_date: Timex.today(),
+ legacy: unquote(import_type) == :new_and_legacy
+ }
+
+ site_import =
+ site
+ |> Plausible.Imported.SiteImport.create_changeset(
+ user,
+ import_params
+ )
+ |> Plausible.Repo.insert!()
+ |> Plausible.Imported.SiteImport.complete_changeset()
+ |> Plausible.Repo.update!()
+
+ {:ok, %{import_id: site_import.id}}
end
test "Visitors data imported from Google Analytics", %{
diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
index deeff48857..b3cc866684 100644
--- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
@@ -4,7 +4,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
@user_id 123
describe "GET /api/stats/main-graph - plot" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "displays pageviews for the last 30 minutes in realtime graph", %{conn: conn, site: site} do
populate_stats(site, [
@@ -368,7 +368,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
end
describe "GET /api/stats/main-graph - pageviews plot" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "displays pageviews for a month", %{conn: conn, site: site} do
populate_stats(site, [
@@ -556,7 +556,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
end
describe "GET /api/stats/main-graph - bounce_rate plot" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "displays bounce_rate for a month", %{conn: conn, site: site} do
populate_stats(site, [
@@ -620,7 +620,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
end
describe "GET /api/stats/main-graph - visit_duration plot" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "displays visit_duration for a month", %{conn: conn, site: site} do
populate_stats(site, [
@@ -915,7 +915,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
end
describe "GET /api/stats/main-graph - comparisons" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns past month stats when period=30d and comparison=previous_period", %{
conn: conn,
@@ -1007,17 +1007,6 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
- site
- |> Ecto.Changeset.change(
- imported_data: %{
- start_date: ~D[2005-01-01],
- end_date: ~D[2020-01-31],
- source: "Google Analytics",
- status: "ok"
- }
- )
- |> Plausible.Repo.update!()
-
conn =
get(
conn,
@@ -1064,17 +1053,6 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
- site
- |> Ecto.Changeset.change(
- imported_data: %{
- start_date: ~D[2005-01-01],
- end_date: ~D[2020-01-31],
- source: "Google Analytics",
- status: "ok"
- }
- )
- |> Plausible.Repo.update!()
-
conn =
get(
conn,
@@ -1125,7 +1103,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
@tag :full_build_only
describe "GET /api/stats/main-graph - total_revenue plot" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "plots total_revenue for a month", %{conn: conn, site: site} do
insert(:goal, site: site, event_name: "Payment", currency: "USD")
@@ -1205,7 +1183,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
@tag :full_build_only
describe "GET /api/stats/main-graph - average_revenue plot" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "plots total_revenue for a month", %{conn: conn, site: site} do
insert(:goal, site: site, event_name: "Payment", currency: "USD")
diff --git a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs
index 6a2f9d8127..d383fca24b 100644
--- a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs
@@ -2,7 +2,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do
use PlausibleWeb.ConnCase
describe "GET /api/stats/:domain/operating-systems" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns operating systems by unique visitors", %{conn: conn, site: site} do
populate_stats(site, [
@@ -211,7 +211,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do
end
describe "GET /api/stats/:domain/operating-system-versions" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns top OS versions by unique visitors", %{conn: conn, site: site} do
populate_stats(site, [
diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs
index be2259ee8c..ea9ccfe6ae 100644
--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs
@@ -4,7 +4,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
@user_id 123
describe "GET /api/stats/:domain/pages" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns top pages by visitors", %{conn: conn, site: site} do
populate_stats(site, [
@@ -1202,7 +1202,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
end
describe "GET /api/stats/:domain/entry-pages" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns top entry pages by visitors", %{conn: conn, site: site} do
populate_stats(site, [
@@ -1555,7 +1555,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
end
describe "GET /api/stats/:domain/exit-pages" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns top exit pages by visitors", %{conn: conn, site: site} do
populate_stats(site, [
diff --git a/test/plausible_web/controllers/api/stats_controller/regions_test.exs b/test/plausible_web/controllers/api/stats_controller/regions_test.exs
index c148cb2b39..23ac063bab 100644
--- a/test/plausible_web/controllers/api/stats_controller/regions_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/regions_test.exs
@@ -32,7 +32,7 @@ defmodule PlausibleWeb.Api.StatsController.RegionsTest do
])
end
- setup [:create_user, :log_in, :create_new_site, :add_imported_data, :seed]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import, :seed]
test "returns top cities by new visitors", %{conn: conn, site: site} do
conn = get(conn, "/api/stats/#{site.domain}/regions?period=day")
diff --git a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs
index a25520be2f..128d1293df 100644
--- a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs
@@ -2,7 +2,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do
use PlausibleWeb.ConnCase
describe "GET /api/stats/:domain/screen-sizes" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns screen sizes by new visitors", %{conn: conn, site: site} do
populate_stats(site, [
diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exs
index 8304732ee9..4d4481c4c0 100644
--- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs
@@ -4,7 +4,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
@user_id 123
describe "GET /api/stats/:domain/sources" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns top sources by unique user ids", %{conn: conn, site: site} do
populate_stats(site, [
@@ -544,7 +544,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
end
describe "GET /api/stats/:domain/utm_mediums" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns top utm_mediums by unique user ids", %{conn: conn, site: site} do
populate_stats(site, [
@@ -696,7 +696,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
end
describe "GET /api/stats/:domain/utm_campaigns" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns top utm_campaigns by unique user ids", %{conn: conn, site: site} do
populate_stats(site, [
@@ -856,7 +856,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
end
describe "GET /api/stats/:domain/utm_sources" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns top utm_sources by unique user ids", %{conn: conn, site: site} do
populate_stats(site, [
@@ -904,7 +904,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
end
describe "GET /api/stats/:domain/utm_terms" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns top utm_terms by unique user ids", %{conn: conn, site: site} do
populate_stats(site, [
@@ -1064,7 +1064,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
end
describe "GET /api/stats/:domain/utm_contents" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns top utm_contents by unique user ids", %{conn: conn, site: site} do
populate_stats(site, [
diff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
index 911e099e80..46c7fb1f0e 100644
--- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
@@ -463,7 +463,7 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
end
describe "GET /api/stats/top-stats - with imported data" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "returns divisible metrics as 0 when no stats exist", %{
site: site,
@@ -1267,7 +1267,7 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
end
describe "GET /api/stats/top-stats - with comparisons" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "does not return comparisons by default", %{site: site, conn: conn} do
populate_stats(site, [
@@ -1352,17 +1352,6 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
- site
- |> Ecto.Changeset.change(
- imported_data: %{
- start_date: ~D[2005-01-01],
- end_date: ~D[2020-01-31],
- source: "Google Analytics",
- status: "ok"
- }
- )
- |> Plausible.Repo.update!()
-
conn =
get(
conn,
@@ -1393,17 +1382,6 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
- site
- |> Ecto.Changeset.change(
- imported_data: %{
- start_date: ~D[2005-01-01],
- end_date: ~D[2020-01-31],
- source: "Google Analytics",
- status: "ok"
- }
- )
- |> Plausible.Repo.update!()
-
conn =
get(
conn,
diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exs
index 0ed74bc184..d6a876ff27 100644
--- a/test/plausible_web/controllers/site_controller_test.exs
+++ b/test/plausible_web/controllers/site_controller_test.exs
@@ -655,9 +655,6 @@ defmodule PlausibleWeb.SiteControllerTest do
end
test "renders imports in import list", %{conn: conn, site: site} do
- {:ok, opts} = add_imported_data(%{site: site})
- site = Map.new(opts).site
-
_site_import1 = insert(:site_import, site: site, status: SiteImport.pending())
_site_import2 = insert(:site_import, site: site, status: SiteImport.importing())
@@ -675,7 +672,7 @@ defmodule PlausibleWeb.SiteControllerTest do
resp = html_response(conn, 200)
buttons = find(resp, ~s|button[data-method="delete"]|)
- assert length(buttons) == 5
+ assert length(buttons) == 4
assert resp =~ "Google Analytics (123456)"
assert resp =~ "(98 page views)"
@@ -693,6 +690,20 @@ defmodule PlausibleWeb.SiteControllerTest do
"Maximum of #{Plausible.Imported.max_complete_imports()} imports is reached."
end
+ test "considers older legacy imports when showing pageview count", %{conn: conn, site: site} do
+ _site_import =
+ insert(:site_import, site: site, legacy: true, status: SiteImport.completed())
+
+ populate_stats(site, [
+ build(:imported_visitors, pageviews: 77),
+ build(:imported_visitors, pageviews: 21)
+ ])
+
+ conn = get(conn, "/#{site.domain}/settings/imports-exports")
+
+ assert html_response(conn, 200) =~ "(98 page views)"
+ end
+
test "disables import buttons when there's import in progress", %{conn: conn, site: site} do
_site_import1 = insert(:site_import, site: site, status: SiteImport.completed())
_site_import2 = insert(:site_import, site: site, status: SiteImport.importing())
@@ -1312,7 +1323,7 @@ defmodule PlausibleWeb.SiteControllerTest do
end
describe "DELETE /:website/settings/:forget_import/:import_id" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ setup [:create_user, :log_in, :create_new_site, :create_legacy_site_import]
test "removes site import, associated data and cancels oban job for a particular import", %{
conn: conn,
@@ -1338,7 +1349,11 @@ defmodule PlausibleWeb.SiteControllerTest do
build(:imported_visitors, pageviews: 10)
])
- assert [%{id: ^import_id}, %{id: 0}] = Plausible.Imported.list_all_imports(site)
+ imports = Plausible.Imported.list_all_imports(site)
+
+ assert Enum.find(imports, &(&1.id == import_id))
+
+ site = Plausible.Imported.load_import_data(site)
assert eventually(fn ->
count = Plausible.Stats.Clickhouse.imported_pageview_count(site)
@@ -1355,9 +1370,10 @@ defmodule PlausibleWeb.SiteControllerTest do
assert Repo.reload(job).state == "cancelled"
end
- test "removes legacy site import along with associated data when instructed", %{
+ test "removes all legacy site import data when instructed", %{
conn: conn,
- site: site
+ site: site,
+ site_import: legacy_site_import
} do
other_site_import = insert(:site_import, site: site)
@@ -1375,7 +1391,7 @@ defmodule PlausibleWeb.SiteControllerTest do
{count == 22, count}
end)
- delete(conn, "/#{site.domain}/settings/forget-import/0")
+ delete(conn, "/#{site.domain}/settings/forget-import/#{legacy_site_import.id}")
assert eventually(fn ->
count = Plausible.Stats.Clickhouse.imported_pageview_count(site)
@@ -1385,13 +1401,7 @@ defmodule PlausibleWeb.SiteControllerTest do
end
describe "DELETE /:website/settings/forget_imported" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
-
- test "removes imported_data field from site", %{conn: conn, site: site} do
- delete(conn, "/#{site.domain}/settings/forget-imported")
-
- assert Repo.reload(site).imported_data == nil
- end
+ setup [:create_user, :log_in, :create_new_site]
test "removes actual imported data from Clickhouse", %{conn: conn, user: user, site: site} do
Plausible.Imported.NoopImporter.new_import(
diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex
index 7ac7e579cf..4f3da9d647 100644
--- a/test/support/test_utils.ex
+++ b/test/support/test_utils.ex
@@ -47,13 +47,21 @@ defmodule Plausible.TestUtils do
{:ok, site: site}
end
- def add_imported_data(%{site: site}) do
- site =
- site
- |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")
- |> Repo.update!()
+ def create_legacy_site_import(%{site: site}) do
+ create_site_import(%{site: site, create_legacy_import?: true})
+ end
- {:ok, site: site}
+ def create_site_import(%{site: site} = opts) do
+ site_import =
+ Factory.insert(:site_import,
+ site: site,
+ start_date: ~D[2005-01-01],
+ end_date: Timex.today(),
+ source: :universal_analytics,
+ legacy: opts[:create_legacy_import?] == true
+ )
+
+ {:ok, site_import: site_import}
end
def create_new_site(%{user: user}) do