Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Add multiple imports per site (#3724)
Adrian Gruntkowskif8b4d5066ab1
diff --git a/config/runtime.exs b/config/runtime.exsindex 90c5cce767..ee5e4cff6b 100644--- a/config/runtime.exs+++ b/config/runtime.exs@@ -524,8 +524,6 @@ base_queues = [check_stats_emails: 1,site_setup_emails: 1,clean_invitations: 1,- # NOTE: to be removed once #3700 is released- google_analytics_imports: 1,analytics_imports: 1,domain_change_transition: 1,check_accept_traffic_until: 1diff --git a/lib/oban_error_reporter.ex b/lib/oban_error_reporter.exindex 171b7d5f49..d9c3b46795 100644--- a/lib/oban_error_reporter.ex+++ b/lib/oban_error_reporter.ex@@ -22,32 +22,31 @@ defmodule ObanErrorReporter doSentry.capture_exception(meta.reason, stacktrace: meta.stacktrace, extra: extra)end- # NOTE: To be cleaned up once #3700 is released- @analytics_queues ["analytics_imports", "google_analytics_imports"]-defp on_job_exception(%Oban.Job{- queue: queue,- args: %{"site_id" => site_id, "source" => source},+ queue: "analytics_imports",+ args: %{"import_id" => import_id},state: "executing",attempt: attempt,max_attempts: max_attempts})- when queue in @analytics_queues and attempt >= max_attempts do- site = Plausible.Repo.get(Plausible.Site, site_id)+ when attempt >= max_attempts do+ site_import = Plausible.Repo.get(Plausible.Imported.SiteImport, import_id)- if site do- Plausible.Workers.ImportAnalytics.import_failed(source, site)+ if site_import do+ Plausible.Workers.ImportAnalytics.import_fail(site_import)endenddefp on_job_exception(%Oban.Job{- queue: queue,- args: %{"site_id" => site_id},+ queue: "analytics_imports",+ args: %{"import_id" => import_id},state: "executing"- })- when queue in @analytics_queues do- site = Plausible.Repo.get(Plausible.Site, site_id)- Plausible.Purge.delete_imported_stats!(site)+ }) do+ site_import = Plausible.Repo.get(Plausible.Imported.SiteImport, import_id)++ if site_import do+ Plausible.Workers.ImportAnalytics.import_fail_transient(site_import)+ endenddefp on_job_exception(_job), do: :ignorediff --git a/lib/plausible/imported.ex b/lib/plausible/imported.exnew file mode 100644index 0000000000..82623fbd2d--- /dev/null+++ b/lib/plausible/imported.ex@@ -0,0 +1,110 @@+defmodule Plausible.Imported do+ @moduledoc """+ Context for managing site statistics imports.++ Currently following importers are implemented:++ * `Plausible.Imported.UniversalAnalytics` - existing mechanism, for legacy Google+ analytics formerly known as "Google Analytics"+ * `Plausible.Imported.NoopImporter` - importer stub, used mainly for testing purposes+ * `Plausible.Imported.CSVImporter` - a placeholder stub for CSV importer that will+ be added soon++ For more information on implementing importers, see `Plausible.Imported.Importer`.+ """++ import Ecto.Query++ alias Plausible.Imported+ alias Plausible.Imported.SiteImport+ alias Plausible.Repo+ alias Plausible.Site++ require Plausible.Imported.SiteImport++ @tables [+ Imported.Visitor,+ Imported.Source,+ Imported.Page,+ Imported.EntryPage,+ Imported.ExitPage,+ Imported.Location,+ Imported.Device,+ Imported.Browser,+ Imported.OperatingSystem+ ]++ @table_names Enum.map(@tables, & &1.__schema__(:source))+ # Maximum number of complete imports to account for when querying stats+ @max_complete_imports 5++ @spec tables() :: [String.t()]+ def tables, do: @table_names++ @spec load_import_data(Site.t()) :: Site.t()+ def load_import_data(%{import_data_loaded: true} = site), do: site++ def load_import_data(site) do+ complete_import_ids = list_complete_import_ids(site)+ earliest_import = get_earliest_import(site) || %{}++ %{+ site+ | import_data_loaded: true,+ earliest_import_start_date: Map.get(earliest_import, :start_date),+ earliest_import_end_date: Map.get(earliest_import, :end_date),+ complete_import_ids: complete_import_ids+ }+ end++ @spec list_all_imports(Site.t()) :: [SiteImport.t()]+ def list_all_imports(site) do+ from(i in SiteImport, where: i.site_id == ^site.id)+ |> Repo.all()+ end++ @spec list_complete_import_ids(Site.t()) :: [non_neg_integer()]+ def list_complete_import_ids(site) do+ ids =+ from(i in SiteImport,+ where: i.site_id == ^site.id and i.status == ^SiteImport.completed(),+ select: i.id,+ limit: @max_complete_imports+ )+ |> Repo.all()++ # account for legacy imports as well+ if site.imported_data && site.imported_data.status == "ok" do+ [0 | ids]+ else+ ids+ end+ end++ @spec get_earliest_import(Site.t()) :: SiteImport.t() | nil+ def get_earliest_import(site) do+ first_import =+ from(i in SiteImport,+ where: i.site_id == ^site.id and i.status == ^SiteImport.completed(),+ order_by: i.start_date,+ limit: 1+ )+ |> Repo.one()++ legacy_import =+ if site.imported_data && site.imported_data.status == "ok" do+ site.imported_data+ end++ [legacy_import, first_import]+ |> Enum.reject(&is_nil/1)+ |> Enum.min_by(& &1.start_date, Date, fn -> nil end)+ end++ @spec delete_imports_for_site(Site.t()) :: :ok+ def delete_imports_for_site(site) do+ Repo.delete_all(from i in SiteImport, where: i.site_id == ^site.id)++ :ok+ end+enddiff --git a/lib/plausible/imported/browser.ex b/lib/plausible/imported/browser.exindex 56d2a87725..5523bb0616 100644--- a/lib/plausible/imported/browser.ex+++ b/lib/plausible/imported/browser.ex@@ -5,6 +5,7 @@ defmodule Plausible.Imported.Browser do@primary_key falseschema "imported_browsers" dofield :site_id, Ch, type: "UInt64"+ field :import_id, Ch, type: "UInt64"field :date, :datefield :browser, :stringfield :visitors, Ch, type: "UInt64"diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.exnew file mode 100644index 0000000000..8c7844fe30--- /dev/null+++ b/lib/plausible/imported/csv_importer.ex@@ -0,0 +1,25 @@+defmodule Plausible.Imported.CSVImporter do+ @moduledoc """+ CSV importer stub.+ """++ use Plausible.Imported.Importer++ @impl true+ def name(), do: :csv++ @impl true+ def label(), do: "CSV"++ # NOTE: change it once CSV import is implemented+ @impl true+ def email_template(), do: "google_analytics_import.html"++ @impl true+ def parse_args(%{"s3_path" => s3_path}), do: [s3_path: s3_path]++ @impl true+ def import_data(_site_import, _opts) do+ :ok+ end+enddiff --git a/lib/plausible/imported/device.ex b/lib/plausible/imported/device.exindex 896196f296..39a7d9aa44 100644--- a/lib/plausible/imported/device.ex+++ b/lib/plausible/imported/device.ex@@ -5,6 +5,7 @@ defmodule Plausible.Imported.Device do@primary_key falseschema "imported_devices" dofield :site_id, Ch, type: "UInt64"+ field :import_id, Ch, type: "UInt64"field :date, :datefield :device, :stringfield :visitors, Ch, type: "UInt64"diff --git a/lib/plausible/imported/entry_page.ex b/lib/plausible/imported/entry_page.exindex 7d9b7b90de..9141bf41a4 100644--- a/lib/plausible/imported/entry_page.ex+++ b/lib/plausible/imported/entry_page.ex@@ -5,6 +5,7 @@ defmodule Plausible.Imported.EntryPage do@primary_key falseschema "imported_entry_pages" dofield :site_id, Ch, type: "UInt64"+ field :import_id, Ch, type: "UInt64"field :date, :datefield :entry_page, :stringfield :visitors, Ch, type: "UInt64"diff --git a/lib/plausible/imported/exit_page.ex b/lib/plausible/imported/exit_page.exindex cbea72b84b..94bd3ab7cf 100644--- a/lib/plausible/imported/exit_page.ex+++ b/lib/plausible/imported/exit_page.ex@@ -5,6 +5,7 @@ defmodule Plausible.Imported.ExitPage do@primary_key falseschema "imported_exit_pages" dofield :site_id, Ch, type: "UInt64"+ field :import_id, Ch, type: "UInt64"field :date, :datefield :exit_page, :stringfield :visitors, Ch, type: "UInt64"diff --git a/lib/plausible/imported/import_sources.ex b/lib/plausible/imported/import_sources.exindex 83a980ef7a..bcf1eb8d45 100644--- a/lib/plausible/imported/import_sources.ex+++ b/lib/plausible/imported/import_sources.ex@@ -5,11 +5,18 @@ defmodule Plausible.Imported.ImportSources do@sources [Plausible.Imported.UniversalAnalytics,- Plausible.Imported.NoopImporter+ Plausible.Imported.NoopImporter,+ Plausible.Imported.CSVImporter]@sources_map Map.new(@sources, &{&1.name(), &1})+ @source_names Enum.map(@sources, & &1.name())++ @spec names() :: [atom()]+ def names(), do: @source_names++ @spec by_name(atom()) :: module()def by_name(name) doMap.fetch!(@sources_map, name)enddiff --git a/lib/plausible/imported/importer.ex b/lib/plausible/imported/importer.exnew file mode 100644index 0000000000..60cebea1c1--- /dev/null+++ b/lib/plausible/imported/importer.ex@@ -0,0 +1,232 @@+defmodule Plausible.Imported.Importer do+ @moduledoc """+ Behaviour that should be implemented for each import source.++ All imports are executed as background jobs run via `Plausible.Workers.ImportAnalytics`+ Oban worker. Each import source must define a module conforming `Importer` behaviour.++ The callbacks that need to be implemented:++ * `name/0` - Returns import source name as an atom. Example: `:universal_analytics`.+ * `label/0` - Descriptive, display friendly name of the source.+ Example: "Google Analytics".+ * `email_template/0` - Name of the email template to use for notifications in+ `PlausibleWeb.Email` (`import_success` and `import_failure`). The template+ should have content customized for a particular source.+ * `parse_args/1` - Receives Oban job arguments coming from `new_import/3`. Whatever+ options were passed to `new_import/3` will be present in the input map with string+ keys and values serialized to primitives. If, for instance `start_date: ~D[2024-01-03]`+ is passed as an option, `parse_args/1` receives `%{..., "start_date" => "2024-01-03"}`.+ The expectation is parsing the map values producing a keyword list of options to+ pass to `import_data/2`.+ * `import_data/2` - Receives site import struct and options produced by `parse_args/1`.+ This is where all the import processing is done. The way the import is implemented+ is entirely arbitrary except the requirement that the process as a whole must+ by synchronous. The callback is expected to return either `:ok` or `{:ok, %{...}}`+ on successful import or `{:error, ...}` on failure. The map in success tuple is+ used for updating site import struct and is passed to `on_success/2` callback.+ Please note that error tuple should be only returned on errors that can't be+ recovered from. For transient errors, the import should throw an exception or+ simply crash.+ * `before_start/1` - Optional callback run right before scheduling import job. It's+ expected to either return `:ok` for the import to proceed or `{:error, ...}` tuple,+ which will be returned from `new_import/3` call.+ * `on_success/2` - Optional callback run once site import is completed. Receives map+ returned from `import_data/2`. Expected to always return `:ok`.+ * `on_failure/1` - Optional callback run when import job fails permanently.++ All sources must be added to the list in `Plausible.Imported.ImportSources`.++ In order to schedule a new import job using a given source, respective importer's+ `new_import/3` function must be called. It accepts site, user who is doing the import+ and any options necessary to carry out the import.++ There's an expectation that `start_date` and `end_date` are provided either as options+ passed to `new_import/3` or data in map returned from `import_data/2`. If these parameters+ are not provided, the import will eventually crash. These parameters define time range+ of imported data which is in turn used for efficient querying.++ Logic running inside `import_data/2` is expected to populated all `imported_*` tables+ in ClickHouse with `import_id` column set to site import's ID.++ Managing any configuration or authentication prior to running import is outside of+ scope of importer logic and is expected to be implemented separately.++ ## Running import fully synchronously++ In case it's necessary to run the whole import job fully synchronously, the+ `Plausible.Workers.ImportAnalytics` worker sends an `Oban.Notifier` message+ on completion, failure or transient failure of the import.++ A basic usage scenario looks like this:++ ```elixir+ {:ok, job} = Plausible.Imported.NoopImporter.new_import(+ site,+ user,+ start_date: ~D[2005-01-01],+ end_date: Date.utc_today(),+ # this option is necessary to setup the calling process as listener+ listen?: true+ )++ import_id = job.args[:import_id]++ receive do+ {:notification, :analytics_imports_jobs, %{"complete" => ^import_id}} ->+ IO.puts("Job completed")++ {:notification, :analytics_imports_jobs, %{"transient_fail" => ^import_id}} ->+ IO.puts("Job failed transiently")++ {:notification, :analytics_imports_jobs, %{"fail" => ^import_id}} ->+ IO.puts("Job failed permanently")+ after+ 15_000 ->+ IO.puts("Job didn't finish in 15 seconds")+ end+ ```++ In a more realistic scenario, job scheduling will be done inside a GenServer process+ like LiveView, where notifications can be listened for via `handle_info/2`.+ """++ alias Plausible.Imported.SiteImport+ alias Plausible.Repo++ @callback name() :: atom()+ @callback label() :: String.t()+ @callback email_template() :: String.t()+ @callback parse_args(map()) :: Keyword.t()+ @callback import_data(Plausible.Imported.SiteImport.t(), Keyword.t()) :: :ok | {:error, any()}+ @callback before_start(SiteImport.t()) :: :ok | {:ok, map()} | {:error, any()}+ @callback on_success(SiteImport.t(), map()) :: :ok+ @callback on_failure(SiteImport.t()) :: :ok++ defmacro __using__(_opts) do+ quote do+ @behaviour Plausible.Imported.Importer++ @spec new_import(Plausible.Site.t(), Plausible.Auth.User.t(), Keyword.t()) ::+ {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()}+ def new_import(site, user, opts) do+ Plausible.Imported.Importer.new_import(name(), site, user, opts, &before_start/1)+ end++ @doc false+ @spec run_import(SiteImport.t(), map()) :: :ok | {:error, :any}+ def run_import(site_import, args) do+ Plausible.Imported.Importer.run_import(+ site_import,+ args,+ &parse_args/1,+ &import_data/2,+ &on_success/2+ )+ end++ @doc false+ @spec mark_failed(SiteImport.t()) :: SiteImport.t()+ def mark_failed(site_import) do+ site_import =+ site_import+ |> SiteImport.fail_changeset()+ |> Repo.update!()++ :ok = on_failure(site_import)++ site_import+ end++ @impl true+ def before_start(_site_import), do: :ok++ @impl true+ def on_success(_site_import, _extra_data), do: :ok++ @impl true+ def on_failure(_site_import), do: :ok++ defoverridable before_start: 1, on_success: 2, on_failure: 1+ end+ end++ @doc false+ def new_import(source, site, user, opts, before_start_fun) do+ import_params =+ opts+ |> Keyword.take([:start_date, :end_date])+ |> Keyword.put(:source, source)+ |> Map.new()++ Repo.transaction(fn ->+ result =+ site+ |> SiteImport.create_changeset(user, import_params)+ |> Repo.insert()++ with {:ok, site_import} <- result,+ :ok <- before_start_fun.(site_import),+ {:ok, job} <- schedule_job(site_import, opts) do+ job+ else+ {:error, error} ->+ Repo.rollback(error)+ end+ end)+ end++ @doc false+ def run_import(site_import, args, parse_args_fun, import_fun, on_success_fun) do+ site_import =+ site_import+ |> SiteImport.start_changeset()+ |> Repo.update!()+ |> Repo.preload(:site)++ import_opts = parse_args_fun.(args)++ case import_fun.(site_import, import_opts) do+ :ok ->+ {:ok, mark_complete(site_import, %{}, on_success_fun)}++ {:ok, extra_data} ->+ {:ok, mark_complete(site_import, extra_data, on_success_fun)}++ {:error, error} ->+ {:error, error}+ end+ end++ @oban_channel :analytics_imports_jobs++ @doc false+ def notify(site_import, event) do+ Oban.Notifier.notify(Oban, @oban_channel, %{event => site_import.id})+ end++ defp schedule_job(site_import, opts) do+ {listen?, opts} = Keyword.pop(opts, :listen?, false)++ if listen? do+ :ok = Oban.Notifier.listen([@oban_channel])+ end++ opts+ |> Keyword.put(:import_id, site_import.id)+ |> Map.new()+ |> Plausible.Workers.ImportAnalytics.new()+ |> Oban.insert()+ end++ defp mark_complete(site_import, extra_data, on_success_fun) do+ site_import =+ site_import+ |> SiteImport.complete_changeset(extra_data)+ |> Repo.update!()++ :ok = on_success_fun.(site_import, extra_data)++ site_import+ end+enddiff --git a/lib/plausible/imported/location.ex b/lib/plausible/imported/location.exindex de48e55604..1b9e27c4f8 100644--- a/lib/plausible/imported/location.ex+++ b/lib/plausible/imported/location.ex@@ -5,6 +5,7 @@ defmodule Plausible.Imported.Location do@primary_key falseschema "imported_locations" dofield :site_id, Ch, type: "UInt64"+ field :import_id, Ch, type: "UInt64"field :date, :datefield :country, :stringfield :region, :stringdiff --git a/lib/plausible/imported/noop_importer.ex b/lib/plausible/imported/noop_importer.exindex 9f9cf0349e..6dec4b078c 100644--- a/lib/plausible/imported/noop_importer.ex+++ b/lib/plausible/imported/noop_importer.ex@@ -3,20 +3,44 @@ defmodule Plausible.Imported.NoopImporter doStub import implementation."""- @name "Noop"+ use Plausible.Imported.Importer- def name(), do: @name+ @impl true+ def name(), do: :noop- def create_job(site, opts) do- Plausible.Workers.ImportAnalytics.new(%{- "source" => @name,- "site_id" => site.id,- "error" => opts[:error]- })- end+ @impl true+ def label(), do: "Noop"++ # reusing existing template from another source+ @impl true+ def email_template(), do: "google_analytics_import.html"+ @impl truedef parse_args(opts), do: opts- def import(_site, %{"error" => true}), do: {:error, "Something went wrong"}- def import(_site, _opts), do: :ok+ @impl true+ def import_data(_site_import, %{"error" => true}), do: {:error, "Something went wrong"}+ def import_data(_site_import, %{"crash" => true}), do: raise("boom")+ def import_data(_site_import, _opts), do: :ok++ @impl true+ def before_start(site_import) do+ send(self(), {:before_start, site_import.id})++ :ok+ end++ @impl true+ def on_success(site_import, _extra_data) do+ send(self(), {:on_success, site_import.id})++ :ok+ end++ @impl true+ def on_failure(site_import) do+ send(self(), {:on_failure, site_import.id})++ :ok+ endenddiff --git a/lib/plausible/imported/operating_system.ex b/lib/plausible/imported/operating_system.exindex fc5292ddb0..3069ac39b5 100644--- a/lib/plausible/imported/operating_system.ex+++ b/lib/plausible/imported/operating_system.ex@@ -5,6 +5,7 @@ defmodule Plausible.Imported.OperatingSystem do@primary_key falseschema "imported_operating_systems" dofield :site_id, Ch, type: "UInt64"+ field :import_id, Ch, type: "UInt64"field :date, :datefield :operating_system, :stringfield :visitors, Ch, type: "UInt64"diff --git a/lib/plausible/imported/page.ex b/lib/plausible/imported/page.exindex 7a320ba241..390463e064 100644--- a/lib/plausible/imported/page.ex+++ b/lib/plausible/imported/page.ex@@ -5,6 +5,7 @@ defmodule Plausible.Imported.Page do@primary_key falseschema "imported_pages" dofield :site_id, Ch, type: "UInt64"+ field :import_id, Ch, type: "UInt64"field :date, :datefield :hostname, :stringfield :page, :stringdiff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.exdeleted file mode 100644index e304f1e833..0000000000--- a/lib/plausible/imported/site.ex+++ /dev/null@@ -1,13 +0,0 @@-defmodule Plausible.Imported do- @tables ~w(- imported_visitors imported_sources imported_pages imported_entry_pages- imported_exit_pages imported_locations imported_devices imported_browsers- imported_operating_systems- )- @spec tables() :: [String.t()]- def tables, do: @tables-- def forget(site) do- Plausible.Purge.delete_imported_stats!(site)- end-enddiff --git a/lib/plausible/imported/site_import.ex b/lib/plausible/imported/site_import.exnew file mode 100644index 0000000000..d8774fcfb1--- /dev/null+++ b/lib/plausible/imported/site_import.ex@@ -0,0 +1,68 @@+defmodule Plausible.Imported.SiteImport do+ @moduledoc """+ Site import schema.+ """++ use Ecto.Schema++ import Ecto.Changeset++ alias Plausible.Auth.User+ alias Plausible.Imported.ImportSources+ alias Plausible.Site++ @statuses [:pending, :importing, :completed, :failed]++ @type t() :: %__MODULE__{}++ schema "site_imports" do+ field :start_date, :date+ field :end_date, :date+ field :source, Ecto.Enum, values: ImportSources.names()+ field :status, Ecto.Enum, values: @statuses++ belongs_to :site, Site+ belongs_to :imported_by, User++ timestamps()+ end++ for status <- @statuses do+ defmacro unquote(status)(), do: unquote(status)+ end++ # NOTE: this is necessary for backwards compatibility+ # with legacy imports+ @spec label(t() | Site.ImportedData.t()) :: String.t()+ def label(%__MODULE__{source: source}), do: ImportSources.by_name(source).label()+ def label(%Site.ImportedData{source: source}), do: source++ @spec create_changeset(Site.t(), User.t(), map()) :: Ecto.Changeset.t()+ def create_changeset(site, user, params) do+ %__MODULE__{}+ |> cast(params, [:source, :start_date, :end_date])+ |> validate_required([:source])+ |> put_assoc(:site, site)+ |> put_assoc(:imported_by, user)+ |> put_change(:status, pending())+ end++ @spec start_changeset(t()) :: Ecto.Changeset.t()+ def start_changeset(site_import) do+ site_import+ |> change(status: importing())+ end++ @spec complete_changeset(t(), map()) :: Ecto.Changeset.t()+ def complete_changeset(site_import, params \\ %{}) do+ site_import+ |> cast(params, [:start_date, :end_date])+ |> put_change(:status, completed())+ |> validate_required([:start_date, :end_date])+ end++ @spec fail_changeset(t()) :: Ecto.Changeset.t()+ def fail_changeset(site_import) do+ change(site_import, status: failed())+ end+enddiff --git a/lib/plausible/imported/source.ex b/lib/plausible/imported/source.exindex c19ca07c6c..3ea5cc6198 100644--- a/lib/plausible/imported/source.ex+++ b/lib/plausible/imported/source.ex@@ -5,6 +5,7 @@ defmodule Plausible.Imported.Source do@primary_key falseschema "imported_sources" dofield :site_id, Ch, type: "UInt64"+ field :import_id, Ch, type: "UInt64"field :date, :datefield :source, :stringfield :utm_medium, :stringdiff --git a/lib/plausible/imported/universal_analytics.ex b/lib/plausible/imported/universal_analytics.exindex 243704a7eb..246be89f4d 100644--- a/lib/plausible/imported/universal_analytics.ex+++ b/lib/plausible/imported/universal_analytics.ex@@ -3,50 +3,59 @@ defmodule Plausible.Imported.UniversalAnalytics doImport implementation for Universal Analytics."""- use Plausible.ClickhouseRepo+ use Plausible.Imported.Importer- alias Plausible.Site+ alias Plausible.Repo@missing_values ["(none)", "(not set)", "(not provided)", "(other)"]- @type job_opt() ::- {:view_id, non_neg_integer()}- | {:start_date | :end_date | :access_token | :refresh_token | :token_expires_at,- String.t()}+ @impl true+ def name(), do: :universal_analytics- @type import_opt() ::- {:view_id, non_neg_integer()}- | {:date_range, Date.Range.t()}- | {:auth, {String.t(), String.t(), String.t()}}+ @impl true+ def label(), do: "Google Analytics"- # NOTE: we have to use old name for now- @name "Google Analytics"+ @impl true+ def email_template(), do: "google_analytics_import.html"- @spec name() :: String.t()- def name(), do: @name+ @impl true+ def before_start(site_import) do+ site = Repo.preload(site_import, :site).site- @spec create_job(Site.t(), [job_opt()]) :: Ecto.Changeset.t()- def create_job(site, opts) do- view_id = Keyword.fetch!(opts, :view_id)- start_date = Keyword.fetch!(opts, :start_date)- end_date = Keyword.fetch!(opts, :end_date)- access_token = Keyword.fetch!(opts, :access_token)- refresh_token = Keyword.fetch!(opts, :refresh_token)- token_expires_at = Keyword.fetch!(opts, :token_expires_at)-- Plausible.Workers.ImportAnalytics.new(%{- "source" => @name,- "site_id" => site.id,- "view_id" => view_id,- "start_date" => start_date,- "end_date" => end_date,- "access_token" => access_token,- "refresh_token" => refresh_token,- "token_expires_at" => token_expires_at- })+ site+ |> Plausible.Site.start_import(+ site_import.start_date,+ site_import.end_date,+ label()+ )+ |> Repo.update!()++ :ok+ end++ @impl true+ def on_success(site_import, _extra_data) do+ site = Repo.preload(site_import, :site).site++ site+ |> Plausible.Site.import_success()+ |> Repo.update!()++ :ok+ end++ @impl true+ def on_failure(site_import) do+ site = Repo.preload(site_import, :site).site++ site+ |> Plausible.Site.import_failure()+ |> Repo.update!()++ :okend- @spec parse_args(map()) :: [import_opt()]+ @impl truedef parse_args(%{"view_id" => view_id, "start_date" => start_date, "end_date" => end_date} = args) do@@ -73,8 +82,8 @@ defmodule Plausible.Imported.UniversalAnalytics doThis function fetches Google Analytics reports which are then passed in batchesto Clickhouse by the `Plausible.Imported.Buffer` process."""- @spec import(Site.t(), [import_opt()]) :: :ok | {:error, any()}- def import(site, opts) do+ @impl true+ def import_data(site_import, opts) dodate_range = Keyword.fetch!(opts, :date_range)view_id = Keyword.fetch!(opts, :view_id)auth = Keyword.fetch!(opts, :auth)@@ -82,7 +91,7 @@ defmodule Plausible.Imported.UniversalAnalytics do{:ok, buffer} = Plausible.Imported.Buffer.start_link()persist_fn = fn table, rows ->- records = from_report(rows, site.id, table)+ records = from_report(rows, site_import.site_id, site_import.id, table)Plausible.Imported.Buffer.insert_many(buffer, table, records)end@@ -94,14 +103,14 @@ defmodule Plausible.Imported.UniversalAnalytics doendend- def from_report(nil, _site_id, _metric), do: nil+ def from_report(nil, _site_id, _import_id, _metric), do: nil- def from_report(data, site_id, table) do+ def from_report(data, site_id, import_id, table) doEnum.reduce(data, [], fn row, acc ->if Map.get(row.dimensions, "ga:date") in @missing_values doaccelse- [new_from_report(site_id, table, row) | acc]+ [new_from_report(site_id, import_id, table, row) | acc]endend)end@@ -111,9 +120,10 @@ defmodule Plausible.Imported.UniversalAnalytics doround(float)end- defp new_from_report(site_id, "imported_visitors", row) do+ defp new_from_report(site_id, import_id, "imported_visitors", row) do%{site_id: site_id,+ import_id: import_id,date: get_date(row),visitors: row.metrics |> Map.fetch!("ga:users") |> parse_number(),pageviews: row.metrics |> Map.fetch!("ga:pageviews") |> parse_number(),@@ -123,9 +133,10 @@ defmodule Plausible.Imported.UniversalAnalytics do}end- defp new_from_report(site_id, "imported_sources", row) do+ defp new_from_report(site_id, import_id, "imported_sources", row) do%{site_id: site_id,+ import_id: import_id,date: get_date(row),source: row.dimensions |> Map.fetch!("ga:source") |> parse_referrer(),utm_medium: row.dimensions |> Map.fetch!("ga:medium") |> default_if_missing(),@@ -139,9 +150,10 @@ defmodule Plausible.Imported.UniversalAnalytics do}end- defp new_from_report(site_id, "imported_pages", row) do+ defp new_from_report(site_id, import_id, "imported_pages", row) do%{site_id: site_id,+ import_id: import_id,date: get_date(row),hostname: row.dimensions |> Map.fetch!("ga:hostname") |> String.replace_prefix("www.", ""),page: row.dimensions |> Map.fetch!("ga:pagePath") |> URI.parse() |> Map.get(:path),@@ -152,9 +164,10 @@ defmodule Plausible.Imported.UniversalAnalytics do}end- defp new_from_report(site_id, "imported_entry_pages", row) do+ defp new_from_report(site_id, import_id, "imported_entry_pages", row) do%{site_id: site_id,+ import_id: import_id,date: get_date(row),entry_page: row.dimensions |> Map.fetch!("ga:landingPagePath"),visitors: row.metrics |> Map.fetch!("ga:users") |> parse_number(),@@ -164,9 +177,10 @@ defmodule Plausible.Imported.UniversalAnalytics do}end- defp new_from_report(site_id, "imported_exit_pages", row) do+ defp new_from_report(site_id, import_id, "imported_exit_pages", row) do%{site_id: site_id,+ import_id: import_id,date: get_date(row),exit_page: Map.fetch!(row.dimensions, "ga:exitPagePath"),visitors: row.metrics |> Map.fetch!("ga:users") |> parse_number(),@@ -174,13 +188,14 @@ defmodule Plausible.Imported.UniversalAnalytics do}end- defp new_from_report(site_id, "imported_locations", row) do+ defp new_from_report(site_id, import_id, "imported_locations", row) docountry_code = row.dimensions |> Map.fetch!("ga:countryIsoCode") |> default_if_missing("")city_name = row.dimensions |> Map.fetch!("ga:city") |> default_if_missing("")city_data = Location.get_city(city_name, country_code)%{site_id: site_id,+ import_id: import_id,date: get_date(row),country: country_code,region: row.dimensions |> Map.fetch!("ga:regionIsoCode") |> default_if_missing(""),@@ -192,9 +207,10 @@ defmodule Plausible.Imported.UniversalAnalytics do}end- defp new_from_report(site_id, "imported_devices", row) do+ defp new_from_report(site_id, import_id, "imported_devices", row) do%{site_id: site_id,+ import_id: import_id,date: get_date(row),device: row.dimensions |> Map.fetch!("ga:deviceCategory") |> String.capitalize(),visitors: row.metrics |> Map.fetch!("ga:users") |> parse_number(),@@ -214,11 +230,12 @@ defmodule Plausible.Imported.UniversalAnalytics do"(not set)" => ""}- defp new_from_report(site_id, "imported_browsers", row) do+ defp new_from_report(site_id, import_id, "imported_browsers", row) dobrowser = Map.fetch!(row.dimensions, "ga:browser")%{site_id: site_id,+ import_id: import_id,date: get_date(row),browser: Map.get(@browser_google_to_plausible, browser, browser),visitors: row.metrics |> Map.fetch!("ga:users") |> parse_number(),@@ -234,11 +251,12 @@ defmodule Plausible.Imported.UniversalAnalytics do"(not set)" => ""}- defp new_from_report(site_id, "imported_operating_systems", row) do+ defp new_from_report(site_id, import_id, "imported_operating_systems", row) doos = Map.fetch!(row.dimensions, "ga:operatingSystem")%{site_id: site_id,+ import_id: import_id,date: get_date(row),operating_system: Map.get(@os_google_to_plausible, os, os),visitors: row.metrics |> Map.fetch!("ga:users") |> parse_number(),diff --git a/lib/plausible/imported/visitor.ex b/lib/plausible/imported/visitor.exindex 7cad7a1961..2e4d4d26d7 100644--- a/lib/plausible/imported/visitor.ex+++ b/lib/plausible/imported/visitor.ex@@ -5,6 +5,7 @@ defmodule Plausible.Imported.Visitor do@primary_key falseschema "imported_visitors" dofield :site_id, Ch, type: "UInt64"+ field :import_id, Ch, type: "UInt64"field :date, :datefield :visitors, Ch, type: "UInt64"field :pageviews, Ch, type: "UInt64"diff --git a/lib/plausible/purge.ex b/lib/plausible/purge.exindex b774798ba1..9fe11b077e 100644--- a/lib/plausible/purge.ex+++ b/lib/plausible/purge.ex@@ -33,18 +33,42 @@ defmodule Plausible.Purge do- [Synchronicity of `ALTER` Queries](https://clickhouse.com/docs/en/sql-reference/statements/alter/#synchronicity-of-alter-queries)"""- @spec delete_imported_stats!(Plausible.Site.t()) :: :ok+ alias Plausible.Repo++ @spec delete_imported_stats!(Plausible.Site.t() | Plausible.Imported.SiteImport.t()) :: :ok@doc """- Deletes imported stats from Google Analytics, and clears the- `stats_start_date` field.+ Deletes imported stats from and clears the `stats_start_date` field.++ The `stats_start_date` is expected to get repopulated the next time+ `Plausible.Sites.stats_start_date/1` is called.++ If the input argument is a site, all imported stats are deleted. If it's a site import,+ only imported stats for that import are deleted."""- def delete_imported_stats!(site) do+ def delete_imported_stats!(%Plausible.Site{} = site) doEnum.each(Plausible.Imported.tables(), fn table ->sql = "ALTER TABLE #{table} DELETE WHERE site_id = {$0:UInt64}"Ecto.Adapters.SQL.query!(Plausible.ImportDeletionRepo, sql, [site.id])end)- clear_stats_start_date!(site)+ Plausible.Sites.clear_stats_start_date!(site)++ :ok+ end++ def delete_imported_stats!(%Plausible.Imported.SiteImport{} = site_import) do+ site_import = Repo.preload(site_import, :site)++ Enum.each(Plausible.Imported.tables(), fn table ->+ sql = "ALTER TABLE #{table} DELETE WHERE site_id = {$0:UInt64} AND import_id = {$1:UInt64}"++ Ecto.Adapters.SQL.query!(Plausible.ImportDeletionRepo, sql, [+ site_import.site_id,+ site_import.id+ ])+ end)++ Plausible.Sites.clear_stats_start_date!(site_import.site):okend@@ -67,10 +91,4 @@ defmodule Plausible.Purge do)|> Plausible.Repo.update!()end-- defp clear_stats_start_date!(site) do- site- |> Ecto.Changeset.change(stats_start_date: nil)- |> Plausible.Repo.update!()- endenddiff --git a/lib/plausible/site.ex b/lib/plausible/site.exindex df99de2f61..c5124f6e2b 100644--- a/lib/plausible/site.ex+++ b/lib/plausible/site.ex@@ -6,7 +6,6 @@ defmodule Plausible.Site doimport Ecto.Changesetalias Plausible.Auth.Useralias Plausible.Site.GoogleAuth- alias Plausible.Timezones@type t() :: %__MODULE__{}@@ -55,6 +54,14 @@ defmodule Plausible.Site dofield :entry_type, :string, virtual: truefield :pinned_at, :naive_datetime, virtual: true+ # Used for caching imports data for the duration of the whole request+ # to avoid multiple identical fetches. Populated by plugs putting+ # `site` in `assigns`.+ field :import_data_loaded, :boolean, default: false, virtual: true+ field :earliest_import_start_date, :date, virtual: true+ field :earliest_import_end_date, :date, virtual: true+ field :complete_import_ids, {:array, :integer}, default: [], virtual: true+timestamps()end@@ -159,7 +166,6 @@ defmodule Plausible.Site dodef import_success(site) dochange(site,- stats_start_date: site.imported_data.start_date,imported_data: %{status: "ok"})end@@ -168,50 +174,10 @@ defmodule Plausible.Site dochange(site, imported_data: %{status: "error"})end- def set_imported_source(site, imported_source) do- change(site,- imported_data: %Plausible.Site.ImportedData{- end_date: Timex.today(),- source: imported_source- }- )- end-def remove_imported_data(site) dochange(site, imported_data: nil)end- @doc """- Returns the date of the first recorded stat in the timezone configured by the user.- This function does 2 transformations:- UTC %NaiveDateTime{} -> Local %DateTime{} -> Local %Date-- ## Examples-- iex> Plausible.Site.local_start_date(%Plausible.Site{stats_start_date: nil})- nil-- iex> utc_start = ~N[2022-09-28 00:00:00]- iex> tz = "Europe/Helsinki"- iex> site = %Plausible.Site{stats_start_date: utc_start, timezone: tz}- iex> Plausible.Site.local_start_date(site)- ~D[2022-09-28]-- iex> utc_start = ~N[2022-09-28 00:00:00]- iex> tz = "America/Los_Angeles"- iex> site = %Plausible.Site{stats_start_date: utc_start, timezone: tz}- iex> Plausible.Site.local_start_date(site)- ~D[2022-09-27]- """- def local_start_date(%__MODULE__{stats_start_date: nil}) do- nil- end-- def local_start_date(site) do- site.stats_start_date- |> Timezones.to_date_in_timezone(site.timezone)- end-defp clean_domain(changeset) doclean_domain =(get_field(changeset, :domain) || "")diff --git a/lib/plausible/site/imported_data.ex b/lib/plausible/site/imported_data.exindex c1db36caee..bec7e571e6 100644--- a/lib/plausible/site/imported_data.ex+++ b/lib/plausible/site/imported_data.ex@@ -4,6 +4,8 @@ defmodule Plausible.Site.ImportedData do"""use Ecto.Schema+ @type t() :: %__MODULE__{}+embedded_schema dofield :start_date, :datefield :end_date, :datediff --git a/lib/plausible/sites.ex b/lib/plausible/sites.exindex 14fc2f0f50..1c8ac51e18 100644--- a/lib/plausible/sites.ex+++ b/lib/plausible/sites.ex@@ -189,13 +189,50 @@ defmodule Plausible.Sites doendend+ @spec clear_stats_start_date!(Site.t()) :: Site.t()+ def clear_stats_start_date!(site) do+ site+ |> Ecto.Changeset.change(stats_start_date: nil)+ |> Plausible.Repo.update!()+ end++ @doc """+ Returns the date of the first recorded stat in the timezone configured by the user.+ This function does 2 transformations:+ UTC %NaiveDateTime{} -> Local %DateTime{} -> Local %Date++ ## Examples++ iex> Plausible.Site.local_start_date(%Plausible.Site{stats_start_date: nil})+ nil++ iex> utc_start = ~N[2022-09-28 00:00:00]+ iex> tz = "Europe/Helsinki"+ iex> site = %Plausible.Site{stats_start_date: utc_start, timezone: tz}+ iex> Plausible.Site.local_start_date(site)+ ~D[2022-09-28]++ iex> utc_start = ~N[2022-09-28 00:00:00]+ iex> tz = "America/Los_Angeles"+ iex> site = %Plausible.Site{stats_start_date: utc_start, timezone: tz}+ iex> Plausible.Site.local_start_date(site)+ ~D[2022-09-27]+ """+ @spec local_start_date(Site.t()) :: Date.t() | nil+ def local_start_date(site) do+ if stats_start_date = stats_start_date(site) do+ Plausible.Timezones.to_date_in_timezone(stats_start_date, site.timezone)+ end+ end+@spec stats_start_date(Site.t()) :: Date.t() | nil@doc """Returns the date of the first event of the given site, or `nil` if the sitedoes not have stats yet.If this is the first time the function is called for the site, it queries- Clickhouse and saves the date in the sites table.+ imported stats and Clickhouse, choosing the earliest start date and saves+ it in the sites table."""def stats_start_date(site)@@ -204,7 +241,17 @@ defmodule Plausible.Sites doenddef stats_start_date(%Site{} = site) do- if start_date = Plausible.Stats.Clickhouse.pageview_start_date_local(site) do+ site = Plausible.Imported.load_import_data(site)++ start_date =+ [+ site.earliest_import_start_date,+ Plausible.Stats.Clickhouse.pageview_start_date_local(site)+ ]+ |> Enum.reject(&is_nil/1)+ |> Enum.min(Date, fn -> nil end)++ if start_date doupdated_site =site|> Site.set_stats_start_date(start_date)diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.exindex 394ba6fe82..8cfe023848 100644--- a/lib/plausible/stats/imported.ex+++ b/lib/plausible/stats/imported.ex@@ -16,9 +16,12 @@ defmodule Plausible.Stats.Imported doquery,metrics) do+ import_ids = site.complete_import_ids+imported_q =from(v in "imported_visitors",where: v.site_id == ^site.id,+ where: v.import_id in ^import_ids,where: v.date >= ^query.date_range.first and v.date <= ^query.date_range.last,select: %{})@@ -107,11 +110,14 @@ defmodule Plausible.Stats.Imported do{"imported_#{dim}s", String.to_existing_atom(dim)}end+ import_ids = site.complete_import_ids+imported_q =from(i in table,group_by: field(i, ^dim),where: i.site_id == ^site.id,+ where: i.import_id in ^import_ids,where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,where: i.visitors > 0,select: %{}@@ -315,10 +321,13 @@ defmodule Plausible.Stats.Imported doenddef merge_imported(q, site, query, :aggregate, metrics) do+ import_ids = site.complete_import_ids+imported_q =from(i in "imported_visitors",where: i.site_id == ^site.id,+ where: i.import_id in ^import_ids,where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,select: %{})diff --git a/lib/plausible/stats/interval.ex b/lib/plausible/stats/interval.exindex 4ad13a9c34..116f711c99 100644--- a/lib/plausible/stats/interval.ex+++ b/lib/plausible/stats/interval.ex@@ -78,7 +78,7 @@ defmodule Plausible.Stats.Interval do@valid_by_periodend- with %Date{} = stats_start <- site.stats_start_date,+ with %Date{} = stats_start <- Plausible.Sites.stats_start_date(site),true <- abs(Timex.diff(Date.utc_today(), stats_start, :months)) > 12 doMap.replace(table, "all", ["week", "month"])elsediff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex 9432ccfd46..934c293ddd 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -121,7 +121,7 @@ defmodule Plausible.Stats.Query dodefp put_period(query, site, %{"period" => "all"}) donow = today(site.timezone)- start_date = Plausible.Site.local_start_date(site) || now+ start_date = Plausible.Sites.local_start_date(site) || nowstruct!(query,period: "all",@@ -251,9 +251,8 @@ defmodule Plausible.Stats.Query do@spec include_imported?(t(), Plausible.Site.t(), boolean()) :: boolean()def include_imported?(query, site, requested?) docond do- is_nil(site.imported_data) -> false- site.imported_data.status != "ok" -> false- Timex.after?(query.date_range.first, site.imported_data.end_date) -> false+ is_nil(site.earliest_import_end_date) -> false+ Date.after?(query.date_range.first, site.earliest_import_end_date) -> falseEnum.any?(query.filters) -> falsequery.period == "realtime" -> falsetrue -> requested?diff --git a/lib/plausible/timezones.ex b/lib/plausible/timezones.exindex def1a35804..03461e35e2 100644--- a/lib/plausible/timezones.ex+++ b/lib/plausible/timezones.ex@@ -22,7 +22,14 @@ defmodule Plausible.Timezones do@spec to_date_in_timezone(Date.t() | NaiveDateTime.t() | DateTime.t(), String.t()) :: Date.t()def to_date_in_timezone(dt, timezone) do- utc_dt = Timex.Timezone.convert(dt, "UTC")+ utc_dt =+ case dt do+ %Date{} ->+ Timex.to_datetime(dt, "UTC")++ dt ->+ Timex.Timezone.convert(dt, "UTC")+ endcase Timex.Timezone.convert(utc_dt, timezone) do%DateTime{} = tz_dt ->diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex dda5eb3a05..97f9692efb 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -3,6 +3,8 @@ defmodule PlausibleWeb.Api.StatsController douse PlausibleWeb, :controlleruse Plausible.Repouse PlausibleWeb.Plugs.ErrorHandler++ alias Plausible.Imported.SiteImportalias Plausible.Statsalias Plausible.Stats.{Query, Comparisons}alias PlausibleWeb.Api.Helpers, as: H@@ -132,6 +134,8 @@ defmodule PlausibleWeb.Api.StatsController dopresent_index = present_index_for(site, query, labels)full_intervals = build_full_intervals(query, labels)+ site_import = Plausible.Imported.get_earliest_import(site)+json(conn, %{plot: plot_timeseries(timeseries_result, selected_metric),labels: labels,@@ -140,7 +144,7 @@ defmodule PlausibleWeb.Api.StatsController dopresent_index: present_index,interval: query.interval,with_imported: with_imported?(query, comparison_query),- imported_source: site.imported_data && site.imported_data.source,+ imported_source: site_import && SiteImport.label(site_import),full_intervals: full_intervals})else@@ -216,12 +220,14 @@ defmodule PlausibleWeb.Api.StatsController do{top_stats, sample_percent} = fetch_top_stats(site, query, comparison_query)+ site_import = Plausible.Imported.get_earliest_import(site)+json(conn, %{top_stats: top_stats,interval: query.interval,sample_percent: sample_percent,with_imported: with_imported?(query, comparison_query),- imported_source: site.imported_data && site.imported_data.source,+ imported_source: site_import && SiteImport.label(site_import),comparing_from: comparison_query && comparison_query.date_range.first,comparing_to: comparison_query && comparison_query.date_range.last,from: query.date_range.first,diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.exindex aa2775b0b4..e34581302e 100644--- a/lib/plausible_web/controllers/site_controller.ex+++ b/lib/plausible_web/controllers/site_controller.ex@@ -764,11 +764,12 @@ defmodule PlausibleWeb.SiteController do"expires_at" => expires_at}) dosite = conn.assigns.site- source = "Google Analytics"+ current_user = conn.assigns.current_user- job =- Plausible.Imported.UniversalAnalytics.create_job(+ {:ok, _} =+ Plausible.Imported.UniversalAnalytics.new_import(site,+ current_user,view_id: view_id,start_date: start_date,end_date: end_date,@@ -777,49 +778,46 @@ defmodule PlausibleWeb.SiteController dotoken_expires_at: expires_at)- Ecto.Multi.new()- |> Ecto.Multi.update(- :update_site,- Plausible.Site.start_import(site, start_date, end_date, source)- )- |> Oban.insert(:oban_job, job)- |> Repo.transaction()-conn|> put_flash(:success, "Import scheduled. An email will be sent when it completes.")|> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain))end- # NOTE: To be cleaned up once #3700 is released- @analytics_queues ["analytics_imports", "google_analytics_imports"]-def forget_imported(conn, _params) do- site = conn.assigns[:site]+ site = conn.assigns.site- cond do- site.imported_data ->- Oban.cancel_all_jobs(- from j in Oban.Job,- where:- j.queue in @analytics_queues and- fragment("(? ->> 'site_id')::int", j.args) == ^site.id- )+ import_ids =+ site+ |> Plausible.Imported.list_all_imports()+ |> Enum.map(& &1.id)++ import_ids =+ if site.imported_data do+ [0 | import_ids]+ else+ import_ids+ end- Plausible.Imported.forget(site)+ if import_ids != [] do+ Oban.cancel_all_jobs(+ from j in Oban.Job,+ where:+ j.queue == "analytics_imports" and+ fragment("(? ->> 'import_id')::int", j.args) in ^import_ids+ )- site- |> Plausible.Site.remove_imported_data()- |> Repo.update!()+ Plausible.Purge.delete_imported_stats!(site)- conn- |> put_flash(:success, "Imported data has been cleared")- |> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain))+ Plausible.Imported.delete_imports_for_site(site)- true ->- conn- |> put_flash(:error, "No data has been imported")- |> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain))+ site+ |> Plausible.Site.remove_imported_data()+ |> Repo.update!()end++ conn+ |> put_flash(:success, "Imported data has been cleared")+ |> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain))enddef change_domain(conn, _params) dodiff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.exindex 55910dc3bf..422b7d78db 100644--- a/lib/plausible_web/controllers/stats_controller.ex+++ b/lib/plausible_web/controllers/stats_controller.ex@@ -309,6 +309,7 @@ defmodule PlausibleWeb.StatsController docond do!shared_link.site.locked ->shared_link = Plausible.Repo.preload(shared_link, site: :owner)+ stats_start_date = Plausible.Sites.stats_start_date(shared_link.site)conn|> put_resp_header("x-robots-tag", "noindex, nofollow")@@ -318,7 +319,7 @@ defmodule PlausibleWeb.StatsController dohas_goals: Sites.has_goals?(shared_link.site),funnels: list_funnels(shared_link.site),has_props: Plausible.Props.configured?(shared_link.site),- stats_start_date: shared_link.site.stats_start_date,+ stats_start_date: stats_start_date,native_stats_start_date: NaiveDateTime.to_date(shared_link.site.native_stats_start_at),title: title(conn, shared_link.site),offer_email_report: false,diff --git a/lib/plausible_web/email.ex b/lib/plausible_web/email.exindex 427f031f11..76af846279 100644--- a/lib/plausible_web/email.ex+++ b/lib/plausible_web/email.ex@@ -313,29 +313,35 @@ defmodule PlausibleWeb.Email do)end- # NOTE: make email different depending on import source- def import_success(source, user, site) do+ def import_success(site_import, user) do+ import_api = Plausible.Imported.ImportSources.by_name(site_import.source)+ label = import_api.label()+priority_email()|> to(user)|> tag("import-success-email")- |> subject("#{source} data imported for #{site.domain}")- |> render("google_analytics_import.html", %{- site: site,- link: PlausibleWeb.Endpoint.url() <> "/" <> URI.encode_www_form(site.domain),+ |> subject("#{label} data imported for #{site_import.site.domain}")+ |> render(import_api.email_template(), %{+ site_import: site_import,+ label: label,+ link: PlausibleWeb.Endpoint.url() <> "/" <> URI.encode_www_form(site_import.site.domain),user: user,success: true})end- # NOTE: make email different depending on import source- def import_failure(source, user, site) do+ def import_failure(site_import, user) do+ import_api = Plausible.Imported.ImportSources.by_name(site_import.source)+ label = import_api.label()+priority_email()|> to(user)|> tag("import-failure-email")- |> subject("#{source} import failed for #{site.domain}")- |> render("google_analytics_import.html", %{+ |> subject("#{label} import failed for #{site_import.site.domain}")+ |> render(import_api.email_template(), %{+ site_import: site_import,+ label: label,user: user,- site: site,success: false})enddiff --git a/lib/plausible_web/plugs/authorize_site_access.ex b/lib/plausible_web/plugs/authorize_site_access.exindex 16995a5bcd..de35461c50 100644--- a/lib/plausible_web/plugs/authorize_site_access.ex+++ b/lib/plausible_web/plugs/authorize_site_access.ex@@ -47,6 +47,8 @@ defmodule PlausibleWeb.AuthorizeSiteAccess doSentry.Context.set_extra_context(%{site_id: site.id, domain: site.domain})Plausible.OpenTelemetry.add_site_attributes(site)+ site = Plausible.Imported.load_import_data(site)+merge_assigns(conn, site: site, current_user_role: role)elsePlausibleWeb.ControllerHelpers.render_error(conn, 404) |> haltdiff --git a/lib/plausible_web/plugs/authorize_stats_api.ex b/lib/plausible_web/plugs/authorize_stats_api.exindex b45c41de9b..4dc68b5bd3 100644--- a/lib/plausible_web/plugs/authorize_stats_api.ex+++ b/lib/plausible_web/plugs/authorize_stats_api.ex@@ -16,6 +16,7 @@ defmodule PlausibleWeb.AuthorizeStatsApiPlug do:ok <- check_api_key_rate_limit(api_key),{:ok, site} <- verify_access(api_key, conn.params["site_id"]) doPlausible.OpenTelemetry.add_site_attributes(site)+ site = Plausible.Imported.load_import_data(site)assign(conn, :site, site)else{:error, :missing_api_key} ->diff --git a/lib/plausible_web/templates/email/google_analytics_import.html.heex b/lib/plausible_web/templates/email/google_analytics_import.html.heexindex d7d1ce539f..a9879165c8 100644--- a/lib/plausible_web/templates/email/google_analytics_import.html.heex+++ b/lib/plausible_web/templates/email/google_analytics_import.html.heex@@ -1,11 +1,11 @@<%= if @success do %>- Your Google Analytics import has completed successfully. The Plausible dashboard for <%= @site.domain %> now contains historical imported data from <%= date_format(- @site.imported_data.start_date- ) %> to <%= date_format(@site.imported_data.end_date) %>+ Your Google Analytics import has completed successfully. The Plausible dashboard for <%= @site_import.site.domain %> now contains historical imported data from <%= date_format(+ @site_import.start_date+ ) %> to <%= date_format(@site_import.end_date) %><br /><br /><%= link("Click here", to: @link) %> to view your dashboard.<% else %>- Unfortunately, your Google Analytics import for <%= @site.domain %> did not complete successfully. Sorry about that!+ Unfortunately, your Google Analytics import for <%= @site_import.site.domain %> did not complete successfully. Sorry about that!<br /> <br />This may be due to the user metric setting in your Google Analytics account. We found that changing the user metric setting is necessary to get the correct data out of the Google API.<br /> <br />diff --git a/lib/workers/import_analytics.ex b/lib/workers/import_analytics.exindex 0be5885bb9..7675419b3d 100644--- a/lib/workers/import_analytics.ex+++ b/lib/workers/import_analytics.ex@@ -9,33 +9,41 @@ defmodule Plausible.Workers.ImportAnalytics douse Oban.Worker,queue: :analytics_imports,max_attempts: 3,- unique: [fields: [:args], period: 60]+ unique: [fields: [:args], keys: [:import_id], period: 60]alias Plausible.Imported.ImportSources+ alias Plausible.Imported.Importer+ alias Plausible.Imported.SiteImport@impl Oban.Workerdef perform(%Oban.Job{- args: %{"site_id" => site_id, "source" => source} = args+ args: %{"import_id" => import_id} = args}) do- import_api = ImportSources.by_name(source)- import_opts = import_api.parse_args(args)+ site_import =+ SiteImport+ |> Repo.get!(import_id)+ |> Repo.preload(:site)- site = Repo.get!(Plausible.Site, site_id)+ import_api = ImportSources.by_name(site_import.source)- case import_api.import(site, import_opts) do- :ok ->- import_success(source, site)+ case import_api.run_import(site_import, args) do+ {:ok, site_import} ->+ import_complete(site_import):ok{:error, error} ->- Sentry.capture_message("Failed to import from Google Analytics",- extra: %{site: site.domain, error: inspect(error)}+ Sentry.capture_message("Failed to import from #{site_import.source}",+ extra: %{+ import_id: site_import.id,+ site: site_import.site.domain,+ error: inspect(error)+ })- import_failed(source, site)+ import_fail(site_import)- {:error, error}+ {:discard, error}endend@@ -45,33 +53,44 @@ defmodule Plausible.Workers.ImportAnalytics do300end- def import_success(source, site) do- site = Repo.preload(site, memberships: :user)+ def import_complete(site_import) do+ site_import = Repo.preload(site_import, site: [memberships: :user])- site- |> Plausible.Site.import_success()- |> Repo.update!()-- Enum.each(site.memberships, fn membership ->+ Enum.each(site_import.site.memberships, fn membership ->if membership.role in [:owner, :admin] do- PlausibleWeb.Email.import_success(source, membership.user, site)+ PlausibleWeb.Email.import_success(site_import, membership.user)|> Plausible.Mailer.send()endend)++ Plausible.Sites.clear_stats_start_date!(site_import.site)++ Importer.notify(site_import, :complete)++ :okend- def import_failed(source, site) do- site = Repo.preload(site, memberships: :user)+ def import_fail_transient(site_import) do+ Plausible.Purge.delete_imported_stats!(site_import)++ Importer.notify(site_import, :transient_fail)+ end++ def import_fail(site_import) do+ Plausible.Purge.delete_imported_stats!(site_import)++ import_api = ImportSources.by_name(site_import.source)- site- |> Plausible.Site.import_failure()- |> Repo.update!()+ site_import =+ site_import+ |> import_api.mark_failed()+ |> Repo.preload(site: [memberships: :user])- Plausible.Purge.delete_imported_stats!(site)+ Importer.notify(site_import, :fail)- Enum.each(site.memberships, fn membership ->+ Enum.each(site_import.site.memberships, fn membership ->if membership.role in [:owner, :admin] do- PlausibleWeb.Email.import_failure(source, membership.user, site)+ PlausibleWeb.Email.import_failure(site_import, membership.user)|> Plausible.Mailer.send()endend)diff --git a/test/plausible/google/api_test.exs b/test/plausible/google/api_test.exsindex 92897270af..c448e7c431 100644--- a/test/plausible/google/api_test.exs+++ b/test/plausible/google/api_test.exs@@ -40,7 +40,7 @@ defmodule Plausible.Google.ApiTest do{:ok, buffer} = Plausible.Imported.Buffer.start_link()persist_fn = fn table, rows ->- records = UniversalAnalytics.from_report(rows, site.id, table)+ records = UniversalAnalytics.from_report(rows, site.id, _import_id = 123, table)Plausible.Imported.Buffer.insert_many(buffer, table, records)end@@ -82,7 +82,7 @@ defmodule Plausible.Google.ApiTest do{:ok, buffer} = Plausible.Imported.Buffer.start_link()persist_fn = fn table, rows ->- records = UniversalAnalytics.from_report(rows, site.id, table)+ records = UniversalAnalytics.from_report(rows, site.id, _import_id = 123, table)Plausible.Imported.Buffer.insert_many(buffer, table, records)enddiff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exsindex dead7ba12e..03ccfb0c0d 100644--- a/test/plausible/imported/imported_test.exs+++ b/test/plausible/imported/imported_test.exs@@ -4,1109 +4,1219 @@ defmodule Plausible.ImportedTest do@user_id 123- defp import_data(ga_data, site_id, table_name) do+ defp import_data(ga_data, site_id, import_id, table_name) doga_data- |> Plausible.Imported.UniversalAnalytics.from_report(site_id, table_name)+ |> Plausible.Imported.UniversalAnalytics.from_report(site_id, import_id, table_name)|> then(&Plausible.Imported.Buffer.insert_all(table_name, &1))end- describe "Parse and import third party data fetched from Google Analytics" do- setup [:create_user, :log_in, :create_new_site, :add_imported_data]-- test "Visitors data imported from Google Analytics", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, timestamp: ~N[2021-01-31 00:00:00])- ])-- import_data(- [- %{- dimensions: %{"ga:date" => "20210101"},- metrics: %{- "ga:users" => "1",- "ga:pageviews" => "1",- "ga:bounces" => "0",- "ga:sessions" => "1",- "ga:sessionDuration" => "60"- }- },- %{- dimensions: %{"ga:date" => "20210131"},- metrics: %{- "ga:users" => "1",- "ga:pageviews" => "1",- "ga:bounces" => "0",- "ga:sessions" => "1",- "ga:sessionDuration" => "60"+ for import_type <- [:legacy, :new_and_legacy, :new] do+ describe "Parse and import third party data fetched from Google Analytics as #{import_type} import" do+ if import_type == :new do+ setup [:create_user, :log_in, :create_new_site]+ else+ setup [:create_user, :log_in, :create_new_site, :add_imported_data]+ end++ setup %{user: user, site: site} do+ if unquote(import_type) in [:new, :new_and_legacy] do+ import_params =+ if unquote(import_type) == :new_and_legacy do+ site.imported_data+ |> Map.from_struct()+ |> Map.put(:source, :universal_analytics)+ else+ %{+ source: :universal_analytics,+ start_date: ~D[2005-01-01],+ end_date: Timex.today()+ }+ end++ site_import =+ site+ |> Plausible.Imported.SiteImport.create_changeset(+ user,+ import_params+ )+ |> Plausible.Repo.insert!()+ |> Plausible.Imported.SiteImport.complete_changeset()+ |> Plausible.Repo.update!()++ {:ok, %{import_id: site_import.id}}+ else+ {:ok, %{import_id: 0}}+ end+ end++ test "Visitors data imported from Google Analytics", %{+ conn: conn,+ site: site,+ import_id: import_id+ } do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-31 00:00:00])+ ])++ import_data(+ [+ %{+ dimensions: %{"ga:date" => "20210101"},+ metrics: %{+ "ga:users" => "1",+ "ga:pageviews" => "1",+ "ga:bounces" => "0",+ "ga:sessions" => "1",+ "ga:sessionDuration" => "60"+ }+ },+ %{+ dimensions: %{"ga:date" => "20210131"},+ metrics: %{+ "ga:users" => "1",+ "ga:pageviews" => "1",+ "ga:bounces" => "0",+ "ga:sessions" => "1",+ "ga:sessionDuration" => "60"+ }}- }- ],- site.id,- "imported_visitors"- )-- conn =- get(- conn,- "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true"+ ],+ site.id,+ import_id,+ "imported_visitors")- assert %{"plot" => plot, "imported_source" => "Google Analytics"} = json_response(conn, 200)-- assert Enum.count(plot) == 31- assert List.first(plot) == 2- assert List.last(plot) == 2- assert Enum.sum(plot) == 4- end-- test "returns data grouped by week", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, timestamp: ~N[2021-01-31 00:00:00])- ])-- import_data(- [- %{- dimensions: %{"ga:date" => "20210101"},- metrics: %{- "ga:users" => "1",- "ga:pageviews" => "1",- "ga:bounces" => "0",- "ga:sessions" => "1",- "ga:sessionDuration" => "60"- }- },- %{- dimensions: %{"ga:date" => "20210131"},- metrics: %{- "ga:users" => "1",- "ga:pageviews" => "1",- "ga:bounces" => "0",- "ga:sessions" => "1",- "ga:sessionDuration" => "60"+ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true"+ )++ assert %{"plot" => plot, "imported_source" => "Google Analytics"} =+ json_response(conn, 200)++ assert Enum.count(plot) == 31+ assert List.first(plot) == 2+ assert List.last(plot) == 2+ assert Enum.sum(plot) == 4+ end++ test "returns data grouped by week", %{conn: conn, site: site, import_id: import_id} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-31 00:00:00])+ ])++ import_data(+ [+ %{+ dimensions: %{"ga:date" => "20210101"},+ metrics: %{+ "ga:users" => "1",+ "ga:pageviews" => "1",+ "ga:bounces" => "0",+ "ga:sessions" => "1",+ "ga:sessionDuration" => "60"+ }+ },+ %{+ dimensions: %{"ga:date" => "20210131"},+ metrics: %{+ "ga:users" => "1",+ "ga:pageviews" => "1",+ "ga:bounces" => "0",+ "ga:sessions" => "1",+ "ga:sessionDuration" => "60"+ }}- }- ],- site.id,- "imported_visitors"- )-- conn =- get(- conn,- "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true&interval=week"+ ],+ site.id,+ import_id,+ "imported_visitors")- assert %{"plot" => plot, "imported_source" => "Google Analytics"} = json_response(conn, 200)- assert Enum.count(plot) == 5- assert List.first(plot) == 2- assert List.last(plot) == 2- assert Enum.sum(plot) == 4- end-- test "Sources are imported", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview,- referrer_source: "Google",- referrer: "google.com",- timestamp: ~N[2021-01-01 00:00:00]- ),- build(:pageview,- referrer_source: "Google",- referrer: "google.com",- timestamp: ~N[2021-01-01 00:00:00]- ),- build(:pageview,- referrer_source: "DuckDuckGo",- referrer: "duckduckgo.com",- timestamp: ~N[2021-01-01 00:00:00]- )- ])-- import_data(- [- %{- dimensions: %{- "ga:adContent" => "",- "ga:campaign" => "",- "ga:date" => "20210101",- "ga:keyword" => "",- "ga:medium" => "organic",- "ga:source" => "duckduckgo.com"+ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true&interval=week"+ )++ assert %{"plot" => plot, "imported_source" => "Google Analytics"} =+ json_response(conn, 200)++ assert Enum.count(plot) == 5+ assert List.first(plot) == 2+ assert List.last(plot) == 2+ assert Enum.sum(plot) == 4+ end++ test "Sources are imported", %{conn: conn, site: site, import_id: import_id} do+ populate_stats(site, [+ build(:pageview,+ referrer_source: "Google",+ referrer: "google.com",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ referrer_source: "Google",+ referrer: "google.com",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ referrer_source: "DuckDuckGo",+ referrer: "duckduckgo.com",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ import_data(+ [+ %{+ dimensions: %{+ "ga:adContent" => "",+ "ga:campaign" => "",+ "ga:date" => "20210101",+ "ga:keyword" => "",+ "ga:medium" => "organic",+ "ga:source" => "duckduckgo.com"+ },+ metrics: %{+ "ga:bounces" => "0",+ "ga:sessionDuration" => "60",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }},- metrics: %{- "ga:bounces" => "0",- "ga:sessionDuration" => "60",- "ga:sessions" => "1",- "ga:users" => "1"- }- },- %{- dimensions: %{- "ga:adContent" => "",- "ga:campaign" => "",- "ga:date" => "20210131",- "ga:keyword" => "",- "ga:medium" => "organic",- "ga:source" => "google.com"+ %{+ dimensions: %{+ "ga:adContent" => "",+ "ga:campaign" => "",+ "ga:date" => "20210131",+ "ga:keyword" => "",+ "ga:medium" => "organic",+ "ga:source" => "google.com"+ },+ metrics: %{+ "ga:bounces" => "1",+ "ga:sessionDuration" => "60",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }},- metrics: %{- "ga:bounces" => "1",- "ga:sessionDuration" => "60",- "ga:sessions" => "1",- "ga:users" => "1"- }- },- %{- dimensions: %{- "ga:adContent" => "",- "ga:campaign" => "",- "ga:date" => "20210101",- "ga:keyword" => "",- "ga:medium" => "paid",- "ga:source" => "google.com"+ %{+ dimensions: %{+ "ga:adContent" => "",+ "ga:campaign" => "",+ "ga:date" => "20210101",+ "ga:keyword" => "",+ "ga:medium" => "paid",+ "ga:source" => "google.com"+ },+ metrics: %{+ "ga:bounces" => "1",+ "ga:sessionDuration" => "60",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }},- metrics: %{- "ga:bounces" => "1",- "ga:sessionDuration" => "60",- "ga:sessions" => "1",- "ga:users" => "1"- }- },- %{- dimensions: %{- "ga:adContent" => "",- "ga:campaign" => "",- "ga:date" => "20210101",- "ga:keyword" => "",- "ga:medium" => "social",- "ga:source" => "Twitter"- },- metrics: %{- "ga:bounces" => "1",- "ga:sessionDuration" => "60",- "ga:sessions" => "1",- "ga:users" => "1"- }- },- %{- dimensions: %{- "ga:adContent" => "",- "ga:campaign" => "newsletter",- "ga:date" => "20210131",- "ga:keyword" => "",- "ga:medium" => "email",- "ga:source" => "A Nice Newsletter"+ %{+ dimensions: %{+ "ga:adContent" => "",+ "ga:campaign" => "",+ "ga:date" => "20210101",+ "ga:keyword" => "",+ "ga:medium" => "social",+ "ga:source" => "Twitter"+ },+ metrics: %{+ "ga:bounces" => "1",+ "ga:sessionDuration" => "60",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }},- metrics: %{- "ga:bounces" => "1",- "ga:sessionDuration" => "60",- "ga:sessions" => "1",- "ga:users" => "1"- }- },- %{- dimensions: %{- "ga:adContent" => "",- "ga:campaign" => "",- "ga:date" => "20210101",- "ga:keyword" => "",- "ga:medium" => "(none)",- "ga:source" => "(direct)"+ %{+ dimensions: %{+ "ga:adContent" => "",+ "ga:campaign" => "newsletter",+ "ga:date" => "20210131",+ "ga:keyword" => "",+ "ga:medium" => "email",+ "ga:source" => "A Nice Newsletter"+ },+ metrics: %{+ "ga:bounces" => "1",+ "ga:sessionDuration" => "60",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }},- metrics: %{- "ga:bounces" => "1",- "ga:sessionDuration" => "60",- "ga:sessions" => "1",- "ga:users" => "1"+ %{+ dimensions: %{+ "ga:adContent" => "",+ "ga:campaign" => "",+ "ga:date" => "20210101",+ "ga:keyword" => "",+ "ga:medium" => "(none)",+ "ga:source" => "(direct)"+ },+ metrics: %{+ "ga:bounces" => "1",+ "ga:sessionDuration" => "60",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }}- }- ],- site.id,- "imported_sources"- )-- conn =- get(- conn,- "/api/stats/#{site.domain}/sources?period=month&date=2021-01-01&with_imported=true"+ ],+ site.id,+ import_id,+ "imported_sources")- assert conn |> json_response(200) |> Enum.sort() == [- %{"name" => "A Nice Newsletter", "visitors" => 1},- %{"name" => "Direct / None", "visitors" => 1},- %{"name" => "DuckDuckGo", "visitors" => 2},- %{"name" => "Google", "visitors" => 4},- %{"name" => "Twitter", "visitors" => 1}- ]- end-- test "UTM mediums data imported from Google Analytics", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview,- utm_medium: "social",- timestamp: ~N[2021-01-01 00:00:00]- ),- build(:pageview,- utm_medium: "social",- timestamp: ~N[2021-01-01 12:00:00]- )- ])-- import_data(- [- %{- dimensions: %{- "ga:adContent" => "",- "ga:campaign" => "",- "ga:date" => "20210101",- "ga:keyword" => "",- "ga:medium" => "social",- "ga:source" => "Twitter"- },- metrics: %{- "ga:bounces" => "1",- "ga:sessionDuration" => "60",- "ga:sessions" => "1",- "ga:users" => "1"- }- },- %{- dimensions: %{- "ga:adContent" => "",- "ga:campaign" => "",- "ga:date" => "20210101",- "ga:keyword" => "",- "ga:medium" => "(none)",- "ga:source" => "(direct)"+ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/sources?period=month&date=2021-01-01&with_imported=true"+ )++ assert conn |> json_response(200) |> Enum.sort() == [+ %{"name" => "A Nice Newsletter", "visitors" => 1},+ %{"name" => "Direct / None", "visitors" => 1},+ %{"name" => "DuckDuckGo", "visitors" => 2},+ %{"name" => "Google", "visitors" => 4},+ %{"name" => "Twitter", "visitors" => 1}+ ]+ end++ test "UTM mediums data imported from Google Analytics", %{+ conn: conn,+ site: site,+ import_id: import_id+ } do+ populate_stats(site, [+ build(:pageview,+ utm_medium: "social",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ utm_medium: "social",+ timestamp: ~N[2021-01-01 12:00:00]+ )+ ])++ import_data(+ [+ %{+ dimensions: %{+ "ga:adContent" => "",+ "ga:campaign" => "",+ "ga:date" => "20210101",+ "ga:keyword" => "",+ "ga:medium" => "social",+ "ga:source" => "Twitter"+ },+ metrics: %{+ "ga:bounces" => "1",+ "ga:sessionDuration" => "60",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }},- metrics: %{- "ga:bounces" => "1",- "ga:sessionDuration" => "60",- "ga:sessions" => "1",- "ga:users" => "1"+ %{+ dimensions: %{+ "ga:adContent" => "",+ "ga:campaign" => "",+ "ga:date" => "20210101",+ "ga:keyword" => "",+ "ga:medium" => "(none)",+ "ga:source" => "(direct)"+ },+ metrics: %{+ "ga:bounces" => "1",+ "ga:sessionDuration" => "60",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }}- }- ],- site.id,- "imported_sources"- )-- conn =- get(- conn,- "/api/stats/#{site.domain}/utm_mediums?period=day&date=2021-01-01&with_imported=true"+ ],+ site.id,+ import_id,+ "imported_sources")- assert json_response(conn, 200) == [- %{- "bounce_rate" => 100.0,- "name" => "social",- "visit_duration" => 20,- "visitors" => 3- }- ]- end-- test "UTM campaigns data imported from Google Analytics", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview, utm_campaign: "profile", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, utm_campaign: "august", timestamp: ~N[2021-01-01 00:00:00])- ])-- import_data(- [- %{- dimensions: %{- "ga:adContent" => "",- "ga:campaign" => "profile",- "ga:date" => "20210101",- "ga:keyword" => "",- "ga:medium" => "social",- "ga:source" => "Twitter"- },- metrics: %{- "ga:bounces" => "1",- "ga:sessionDuration" => "100",- "ga:sessions" => "1",- "ga:users" => "1"- }- },- %{- dimensions: %{- "ga:adContent" => "",- "ga:campaign" => "august",- "ga:date" => "20210101",- "ga:keyword" => "",- "ga:medium" => "email",- "ga:source" => "Gmail"+ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/utm_mediums?period=day&date=2021-01-01&with_imported=true"+ )++ assert json_response(conn, 200) == [+ %{+ "bounce_rate" => 100.0,+ "name" => "social",+ "visit_duration" => 20,+ "visitors" => 3+ }+ ]+ end++ test "UTM campaigns data imported from Google Analytics", %{+ conn: conn,+ site: site,+ import_id: import_id+ } do+ populate_stats(site, [+ build(:pageview, utm_campaign: "profile", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, utm_campaign: "august", timestamp: ~N[2021-01-01 00:00:00])+ ])++ import_data(+ [+ %{+ dimensions: %{+ "ga:adContent" => "",+ "ga:campaign" => "profile",+ "ga:date" => "20210101",+ "ga:keyword" => "",+ "ga:medium" => "social",+ "ga:source" => "Twitter"+ },+ metrics: %{+ "ga:bounces" => "1",+ "ga:sessionDuration" => "100",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }},- metrics: %{- "ga:bounces" => "0",- "ga:sessionDuration" => "100",- "ga:sessions" => "1",- "ga:users" => "1"- }- },- %{- dimensions: %{- "ga:adContent" => "",- "ga:campaign" => "(not set)",- "ga:date" => "20210101",- "ga:keyword" => "",- "ga:medium" => "email",- "ga:source" => "Gmail"+ %{+ dimensions: %{+ "ga:adContent" => "",+ "ga:campaign" => "august",+ "ga:date" => "20210101",+ "ga:keyword" => "",+ "ga:medium" => "email",+ "ga:source" => "Gmail"+ },+ metrics: %{+ "ga:bounces" => "0",+ "ga:sessionDuration" => "100",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }},- metrics: %{- "ga:bounces" => "0",- "ga:sessionDuration" => "100",- "ga:sessions" => "1",- "ga:users" => "1"+ %{+ dimensions: %{+ "ga:adContent" => "",+ "ga:campaign" => "(not set)",+ "ga:date" => "20210101",+ "ga:keyword" => "",+ "ga:medium" => "email",+ "ga:source" => "Gmail"+ },+ metrics: %{+ "ga:bounces" => "0",+ "ga:sessionDuration" => "100",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }}- }- ],- site.id,- "imported_sources"- )-- conn =- get(- conn,- "/api/stats/#{site.domain}/utm_campaigns?period=day&date=2021-01-01&with_imported=true"+ ],+ site.id,+ import_id,+ "imported_sources")- assert json_response(conn, 200) == [- %{- "name" => "august",- "visitors" => 2,- "bounce_rate" => 50.0,- "visit_duration" => 50.0- },- %{- "name" => "profile",- "visitors" => 2,- "bounce_rate" => 100.0,- "visit_duration" => 50.0- }- ]- end-- test "UTM terms data imported from Google Analytics", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview, utm_term: "oat milk", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, utm_term: "Sweden", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, utm_term: "Sweden", timestamp: ~N[2021-01-01 00:00:00])- ])-- import_data(- [- %{- dimensions: %{- "ga:adContent" => "",- "ga:campaign" => "",- "ga:date" => "20210101",- "ga:keyword" => "oat milk",- "ga:medium" => "paid",- "ga:source" => "Google"- },- metrics: %{- "ga:bounces" => "1",- "ga:sessionDuration" => "100",- "ga:sessions" => "1",- "ga:users" => "1"- }- },- %{- dimensions: %{- "ga:adContent" => "",- "ga:campaign" => "",- "ga:date" => "20210101",- "ga:keyword" => "Sweden",- "ga:medium" => "paid",- "ga:source" => "Google"+ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/utm_campaigns?period=day&date=2021-01-01&with_imported=true"+ )++ assert json_response(conn, 200) == [+ %{+ "name" => "august",+ "visitors" => 2,+ "bounce_rate" => 50.0,+ "visit_duration" => 50.0+ },+ %{+ "name" => "profile",+ "visitors" => 2,+ "bounce_rate" => 100.0,+ "visit_duration" => 50.0+ }+ ]+ end++ test "UTM terms data imported from Google Analytics", %{+ conn: conn,+ site: site,+ import_id: import_id+ } do+ populate_stats(site, [+ build(:pageview, utm_term: "oat milk", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, utm_term: "Sweden", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, utm_term: "Sweden", timestamp: ~N[2021-01-01 00:00:00])+ ])++ import_data(+ [+ %{+ dimensions: %{+ "ga:adContent" => "",+ "ga:campaign" => "",+ "ga:date" => "20210101",+ "ga:keyword" => "oat milk",+ "ga:medium" => "paid",+ "ga:source" => "Google"+ },+ metrics: %{+ "ga:bounces" => "1",+ "ga:sessionDuration" => "100",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }},- metrics: %{- "ga:bounces" => "0",- "ga:sessionDuration" => "100",- "ga:sessions" => "1",- "ga:users" => "1"- }- },- %{- dimensions: %{- "ga:adContent" => "",- "ga:campaign" => "",- "ga:date" => "20210101",- "ga:keyword" => "(not set)",- "ga:medium" => "paid",- "ga:source" => "Google"+ %{+ dimensions: %{+ "ga:adContent" => "",+ "ga:campaign" => "",+ "ga:date" => "20210101",+ "ga:keyword" => "Sweden",+ "ga:medium" => "paid",+ "ga:source" => "Google"+ },+ metrics: %{+ "ga:bounces" => "0",+ "ga:sessionDuration" => "100",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }},- metrics: %{- "ga:bounces" => "0",- "ga:sessionDuration" => "100",- "ga:sessions" => "1",- "ga:users" => "1"+ %{+ dimensions: %{+ "ga:adContent" => "",+ "ga:campaign" => "",+ "ga:date" => "20210101",+ "ga:keyword" => "(not set)",+ "ga:medium" => "paid",+ "ga:source" => "Google"+ },+ metrics: %{+ "ga:bounces" => "0",+ "ga:sessionDuration" => "100",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }}- }- ],- site.id,- "imported_sources"- )-- conn =- get(- conn,- "/api/stats/#{site.domain}/utm_terms?period=day&date=2021-01-01&with_imported=true"+ ],+ site.id,+ import_id,+ "imported_sources")- assert json_response(conn, 200) == [- %{- "name" => "Sweden",- "visitors" => 3,- "bounce_rate" => 67.0,- "visit_duration" => 33.3- },- %{- "name" => "oat milk",- "visitors" => 2,- "bounce_rate" => 100.0,- "visit_duration" => 50.0- }- ]- end-- test "UTM contents data imported from Google Analytics", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview, utm_content: "ad", timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, utm_content: "blog", timestamp: ~N[2021-01-01 00:00:00])- ])-- import_data(- [- %{- dimensions: %{- "ga:adContent" => "ad",- "ga:campaign" => "",- "ga:date" => "20210101",- "ga:keyword" => "",- "ga:medium" => "paid",- "ga:source" => "Google"- },- metrics: %{- "ga:bounces" => "1",- "ga:sessionDuration" => "100",- "ga:sessions" => "1",- "ga:users" => "1"- }- },- %{- dimensions: %{- "ga:adContent" => "blog",- "ga:campaign" => "",- "ga:date" => "20210101",- "ga:keyword" => "",- "ga:medium" => "paid",- "ga:source" => "Google"+ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/utm_terms?period=day&date=2021-01-01&with_imported=true"+ )++ assert json_response(conn, 200) == [+ %{+ "name" => "Sweden",+ "visitors" => 3,+ "bounce_rate" => 67.0,+ "visit_duration" => 33.3+ },+ %{+ "name" => "oat milk",+ "visitors" => 2,+ "bounce_rate" => 100.0,+ "visit_duration" => 50.0+ }+ ]+ end++ test "UTM contents data imported from Google Analytics", %{+ conn: conn,+ site: site,+ import_id: import_id+ } do+ populate_stats(site, [+ build(:pageview, utm_content: "ad", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, utm_content: "blog", timestamp: ~N[2021-01-01 00:00:00])+ ])++ import_data(+ [+ %{+ dimensions: %{+ "ga:adContent" => "ad",+ "ga:campaign" => "",+ "ga:date" => "20210101",+ "ga:keyword" => "",+ "ga:medium" => "paid",+ "ga:source" => "Google"+ },+ metrics: %{+ "ga:bounces" => "1",+ "ga:sessionDuration" => "100",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }},- metrics: %{- "ga:bounces" => "0",- "ga:sessionDuration" => "100",- "ga:sessions" => "1",- "ga:users" => "1"- }- },- %{- dimensions: %{- "ga:adContent" => "(not set)",- "ga:campaign" => "",- "ga:date" => "20210101",- "ga:keyword" => "",- "ga:medium" => "paid",- "ga:source" => "Google"+ %{+ dimensions: %{+ "ga:adContent" => "blog",+ "ga:campaign" => "",+ "ga:date" => "20210101",+ "ga:keyword" => "",+ "ga:medium" => "paid",+ "ga:source" => "Google"+ },+ metrics: %{+ "ga:bounces" => "0",+ "ga:sessionDuration" => "100",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }},- metrics: %{- "ga:bounces" => "0",- "ga:sessionDuration" => "100",- "ga:sessions" => "1",- "ga:users" => "1"+ %{+ dimensions: %{+ "ga:adContent" => "(not set)",+ "ga:campaign" => "",+ "ga:date" => "20210101",+ "ga:keyword" => "",+ "ga:medium" => "paid",+ "ga:source" => "Google"+ },+ metrics: %{+ "ga:bounces" => "0",+ "ga:sessionDuration" => "100",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }}- }- ],- site.id,- "imported_sources"- )-- conn =- get(- conn,- "/api/stats/#{site.domain}/utm_contents?period=day&date=2021-01-01&with_imported=true"+ ],+ site.id,+ import_id,+ "imported_sources")- assert json_response(conn, 200) == [- %{- "name" => "blog",- "visitors" => 2,- "bounce_rate" => 50.0,- "visit_duration" => 50.0- },- %{- "name" => "ad",- "visitors" => 2,- "bounce_rate" => 100.0,- "visit_duration" => 50.0- }- ]- end-- test "Page event data imported from Google Analytics", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview,- pathname: "/",- hostname: "host-a.com",- user_id: @user_id,- timestamp: ~N[2021-01-01 00:00:00]- ),- build(:pageview,- pathname: "/some-other-page",- hostname: "host-a.com",- user_id: @user_id,- timestamp: ~N[2021-01-01 00:15:00]- )- ])-- import_data(- [- %{- dimensions: %{- "ga:date" => "20210101",- "ga:hostname" => "host-a.com",- "ga:pagePath" => "/"- },- metrics: %{- "ga:exits" => "0",- "ga:pageviews" => "1",- "ga:timeOnPage" => "700",- "ga:users" => "1"- }- },- %{- dimensions: %{- "ga:date" => "20210101",- "ga:hostname" => "host-b.com",- "ga:pagePath" => "/some-other-page"+ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/utm_contents?period=day&date=2021-01-01&with_imported=true"+ )++ assert json_response(conn, 200) == [+ %{+ "name" => "blog",+ "visitors" => 2,+ "bounce_rate" => 50.0,+ "visit_duration" => 50.0+ },+ %{+ "name" => "ad",+ "visitors" => 2,+ "bounce_rate" => 100.0,+ "visit_duration" => 50.0+ }+ ]+ end++ test "Page event data imported from Google Analytics", %{+ conn: conn,+ site: site,+ import_id: import_id+ } do+ populate_stats(site, [+ build(:pageview,+ pathname: "/",+ hostname: "host-a.com",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/some-other-page",+ hostname: "host-a.com",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:15:00]+ )+ ])++ import_data(+ [+ %{+ dimensions: %{+ "ga:date" => "20210101",+ "ga:hostname" => "host-a.com",+ "ga:pagePath" => "/"+ },+ metrics: %{+ "ga:exits" => "0",+ "ga:pageviews" => "1",+ "ga:timeOnPage" => "700",+ "ga:users" => "1"+ }},- metrics: %{- "ga:exits" => "1",- "ga:pageviews" => "2",- "ga:timeOnPage" => "60",- "ga:users" => "1"- }- },- %{- dimensions: %{- "ga:date" => "20210101",- "ga:hostname" => "host-b.com",- "ga:pagePath" => "/some-other-page?wat=wot"+ %{+ dimensions: %{+ "ga:date" => "20210101",+ "ga:hostname" => "host-b.com",+ "ga:pagePath" => "/some-other-page"+ },+ metrics: %{+ "ga:exits" => "1",+ "ga:pageviews" => "2",+ "ga:timeOnPage" => "60",+ "ga:users" => "1"+ }},- metrics: %{- "ga:exits" => "0",- "ga:pageviews" => "1",- "ga:timeOnPage" => "60",- "ga:users" => "1"- }- }- ],- site.id,- "imported_pages"- )-- import_data(- [- %{- dimensions: %{"ga:date" => "20210101", "ga:landingPagePath" => "/"},- metrics: %{- "ga:bounces" => "1",- "ga:entrances" => "3",- "ga:sessionDuration" => "10",- "ga:users" => "1"+ %{+ dimensions: %{+ "ga:date" => "20210101",+ "ga:hostname" => "host-b.com",+ "ga:pagePath" => "/some-other-page?wat=wot"+ },+ metrics: %{+ "ga:exits" => "0",+ "ga:pageviews" => "1",+ "ga:timeOnPage" => "60",+ "ga:users" => "1"+ }}- }- ],- site.id,- "imported_entry_pages"- )-- conn =- get(- conn,- "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true&with_imported=true"+ ],+ site.id,+ import_id,+ "imported_pages")- assert json_response(conn, 200) == [- %{- "bounce_rate" => nil,- "time_on_page" => 60,- "visitors" => 3,- "pageviews" => 4,- "name" => "/some-other-page"- },- %{- "bounce_rate" => 25.0,- "time_on_page" => 800.0,- "visitors" => 2,- "pageviews" => 2,- "name" => "/"- }- ]- end-- test "Exit page event data imported from Google Analytics", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview,- pathname: "/page1",- timestamp: ~N[2021-01-01 00:00:00]- ),- build(:pageview,- pathname: "/page1",- timestamp: ~N[2021-01-01 00:00:00]- ),- build(:pageview,- pathname: "/page1",- user_id: @user_id,- timestamp: ~N[2021-01-01 00:00:00]- ),- build(:pageview,- pathname: "/page2",- user_id: @user_id,- timestamp: ~N[2021-01-01 00:15:00]- )- ])-- import_data(- [- %{- dimensions: %{- "ga:date" => "20210101",- "ga:hostname" => "host-a.com",- "ga:pagePath" => "/page2"- },- metrics: %{- "ga:exits" => "0",- "ga:pageviews" => "4",- "ga:timeOnPage" => "10",- "ga:users" => "2"+ import_data(+ [+ %{+ dimensions: %{"ga:date" => "20210101", "ga:landingPagePath" => "/"},+ metrics: %{+ "ga:bounces" => "1",+ "ga:entrances" => "3",+ "ga:sessionDuration" => "10",+ "ga:users" => "1"+ }}- }- ],- site.id,- "imported_pages"- )-- import_data(- [- %{- dimensions: %{"ga:date" => "20210101", "ga:exitPagePath" => "/page2"},- metrics: %{"ga:exits" => "3", "ga:users" => "2"}- }- ],- site.id,- "imported_exit_pages"- )-- conn =- get(- conn,- "/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01&with_imported=true"+ ],+ site.id,+ import_id,+ "imported_entry_pages")- assert json_response(conn, 200) == [- %{- "name" => "/page2",- "visitors" => 3,- "visits" => 4,- "exit_rate" => 80.0- },- %{"name" => "/page1", "visitors" => 2, "visits" => 2, "exit_rate" => 66}- ]- end-- test "imports city data from Google Analytics", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview,- country_code: "EE",- timestamp: ~N[2021-01-01 00:15:00]- ),- build(:pageview,- country_code: "EE",- timestamp: ~N[2021-01-01 00:15:00]- ),- build(:pageview,- country_code: "GB",- timestamp: ~N[2021-01-01 00:15:00]- )- ])-- import_data(- [- %{- dimensions: %{- "ga:countryIsoCode" => "EE",- "ga:city" => "Tartu",- "ga:date" => "20210101",- "ga:regionIsoCode" => "Tartumaa"- },- metrics: %{- "ga:bounces" => "0",- "ga:sessionDuration" => "10",- "ga:sessions" => "1",- "ga:users" => "1"- }- },- %{- dimensions: %{- "ga:countryIsoCode" => "GB",- "ga:city" => "Edinburgh",- "ga:date" => "20210101",- "ga:regionIsoCode" => "Midlothian"- },- metrics: %{- "ga:bounces" => "0",- "ga:sessionDuration" => "10",- "ga:sessions" => "1",- "ga:users" => "1"+ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true&with_imported=true"+ )++ assert json_response(conn, 200) == [+ %{+ "bounce_rate" => nil,+ "time_on_page" => 60,+ "visitors" => 3,+ "pageviews" => 4,+ "name" => "/some-other-page"+ },+ %{+ "bounce_rate" => 25.0,+ "time_on_page" => 800.0,+ "visitors" => 2,+ "pageviews" => 2,+ "name" => "/"+ }+ ]+ end++ test "Exit page event data imported from Google Analytics", %{+ conn: conn,+ site: site,+ import_id: import_id+ } do+ populate_stats(site, [+ build(:pageview,+ pathname: "/page1",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/page1",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/page1",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/page2",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:15:00]+ )+ ])++ import_data(+ [+ %{+ dimensions: %{+ "ga:date" => "20210101",+ "ga:hostname" => "host-a.com",+ "ga:pagePath" => "/page2"+ },+ metrics: %{+ "ga:exits" => "0",+ "ga:pageviews" => "4",+ "ga:timeOnPage" => "10",+ "ga:users" => "2"+ }}- }- ],- site.id,- "imported_locations"- )-- conn =- get(- conn,- "/api/stats/#{site.domain}/cities?period=day&date=2021-01-01&with_imported=true"+ ],+ site.id,+ import_id,+ "imported_pages")- assert json_response(conn, 200) == [- %{"code" => 588_335, "name" => "Tartu", "visitors" => 1, "country_flag" => "πͺπͺ"},- %{- "code" => 2_650_225,- "name" => "Edinburgh",- "visitors" => 1,- "country_flag" => "π¬π§"- }- ]- end+ import_data(+ [+ %{+ dimensions: %{"ga:date" => "20210101", "ga:exitPagePath" => "/page2"},+ metrics: %{"ga:exits" => "3", "ga:users" => "2"}+ }+ ],+ site.id,+ import_id,+ "imported_exit_pages"+ )- test "imports country data from Google Analytics", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview,- country_code: "EE",- timestamp: ~N[2021-01-01 00:15:00]- ),- build(:pageview,- country_code: "EE",- timestamp: ~N[2021-01-01 00:15:00]- ),- build(:pageview,- country_code: "GB",- timestamp: ~N[2021-01-01 00:15:00]- ),- build(:imported_visitors, date: ~D[2021-01-01], visitors: 2)- ])-- import_data(- [- %{- dimensions: %{- "ga:countryIsoCode" => "EE",- "ga:city" => "Tartu",- "ga:date" => "20210101",- "ga:regionIsoCode" => "Tartumaa"+ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01&with_imported=true"+ )++ assert json_response(conn, 200) == [+ %{+ "name" => "/page2",+ "visitors" => 3,+ "visits" => 4,+ "exit_rate" => 80.0+ },+ %{"name" => "/page1", "visitors" => 2, "visits" => 2, "exit_rate" => 66}+ ]+ end++ test "imports city data from Google Analytics", %{+ conn: conn,+ site: site,+ import_id: import_id+ } do+ populate_stats(site, [+ build(:pageview,+ country_code: "EE",+ timestamp: ~N[2021-01-01 00:15:00]+ ),+ build(:pageview,+ country_code: "EE",+ timestamp: ~N[2021-01-01 00:15:00]+ ),+ build(:pageview,+ country_code: "GB",+ timestamp: ~N[2021-01-01 00:15:00]+ )+ ])++ import_data(+ [+ %{+ dimensions: %{+ "ga:countryIsoCode" => "EE",+ "ga:city" => "Tartu",+ "ga:date" => "20210101",+ "ga:regionIsoCode" => "Tartumaa"+ },+ metrics: %{+ "ga:bounces" => "0",+ "ga:sessionDuration" => "10",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }},- metrics: %{- "ga:bounces" => "0",- "ga:sessionDuration" => "10",- "ga:sessions" => "1",- "ga:users" => "1"+ %{+ dimensions: %{+ "ga:countryIsoCode" => "GB",+ "ga:city" => "Edinburgh",+ "ga:date" => "20210101",+ "ga:regionIsoCode" => "Midlothian"+ },+ metrics: %{+ "ga:bounces" => "0",+ "ga:sessionDuration" => "10",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }}- },- %{- dimensions: %{- "ga:countryIsoCode" => "GB",- "ga:city" => "Edinburgh",- "ga:date" => "20210101",- "ga:regionIsoCode" => "Midlothian"+ ],+ site.id,+ import_id,+ "imported_locations"+ )++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/cities?period=day&date=2021-01-01&with_imported=true"+ )++ assert json_response(conn, 200) == [+ %{"code" => 588_335, "name" => "Tartu", "visitors" => 1, "country_flag" => "πͺπͺ"},+ %{+ "code" => 2_650_225,+ "name" => "Edinburgh",+ "visitors" => 1,+ "country_flag" => "π¬π§"+ }+ ]+ end++ test "imports country data from Google Analytics", %{+ conn: conn,+ site: site,+ import_id: import_id+ } do+ populate_stats(site, import_id, [+ build(:pageview,+ country_code: "EE",+ timestamp: ~N[2021-01-01 00:15:00]+ ),+ build(:pageview,+ country_code: "EE",+ timestamp: ~N[2021-01-01 00:15:00]+ ),+ build(:pageview,+ country_code: "GB",+ timestamp: ~N[2021-01-01 00:15:00]+ ),+ build(:imported_visitors, date: ~D[2021-01-01], visitors: 2)+ ])++ import_data(+ [+ %{+ dimensions: %{+ "ga:countryIsoCode" => "EE",+ "ga:city" => "Tartu",+ "ga:date" => "20210101",+ "ga:regionIsoCode" => "Tartumaa"+ },+ metrics: %{+ "ga:bounces" => "0",+ "ga:sessionDuration" => "10",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }},- metrics: %{- "ga:bounces" => "0",- "ga:sessionDuration" => "10",- "ga:sessions" => "1",- "ga:users" => "1"+ %{+ dimensions: %{+ "ga:countryIsoCode" => "GB",+ "ga:city" => "Edinburgh",+ "ga:date" => "20210101",+ "ga:regionIsoCode" => "Midlothian"+ },+ metrics: %{+ "ga:bounces" => "0",+ "ga:sessionDuration" => "10",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }}- }- ],- site.id,- "imported_locations"- )-- conn =- get(- conn,- "/api/stats/#{site.domain}/countries?period=day&date=2021-01-01&with_imported=true"+ ],+ site.id,+ import_id,+ "imported_locations")- assert json_response(conn, 200) == [- %{- "code" => "EE",- "alpha_3" => "EST",- "name" => "Estonia",- "flag" => "πͺπͺ",- "visitors" => 3,- "percentage" => 60- },- %{- "code" => "GB",- "alpha_3" => "GBR",- "name" => "United Kingdom",- "flag" => "π¬π§",- "visitors" => 2,- "percentage" => 40- }- ]- end-- test "Devices data imported from Google Analytics", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview, screen_size: "Desktop", timestamp: ~N[2021-01-01 00:15:00]),- build(:pageview, screen_size: "Desktop", timestamp: ~N[2021-01-01 00:15:00]),- build(:pageview, screen_size: "Laptop", timestamp: ~N[2021-01-01 00:15:00]),- build(:imported_visitors, date: ~D[2021-01-01], visitors: 2)- ])-- import_data(- [- %{- dimensions: %{"ga:date" => "20210101", "ga:deviceCategory" => "mobile"},- metrics: %{- "ga:bounces" => "0",- "ga:sessionDuration" => "10",- "ga:sessions" => "1",- "ga:users" => "1"- }- },- %{- dimensions: %{"ga:date" => "20210101", "ga:deviceCategory" => "Laptop"},- metrics: %{- "ga:bounces" => "0",- "ga:sessionDuration" => "10",- "ga:sessions" => "1",- "ga:users" => "1"+ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/countries?period=day&date=2021-01-01&with_imported=true"+ )++ assert json_response(conn, 200) == [+ %{+ "code" => "EE",+ "alpha_3" => "EST",+ "name" => "Estonia",+ "flag" => "πͺπͺ",+ "visitors" => 3,+ "percentage" => 60+ },+ %{+ "code" => "GB",+ "alpha_3" => "GBR",+ "name" => "United Kingdom",+ "flag" => "π¬π§",+ "visitors" => 2,+ "percentage" => 40+ }+ ]+ end++ test "Devices data imported from Google Analytics", %{+ conn: conn,+ site: site,+ import_id: import_id+ } do+ populate_stats(site, import_id, [+ build(:pageview, screen_size: "Desktop", timestamp: ~N[2021-01-01 00:15:00]),+ build(:pageview, screen_size: "Desktop", timestamp: ~N[2021-01-01 00:15:00]),+ build(:pageview, screen_size: "Laptop", timestamp: ~N[2021-01-01 00:15:00]),+ build(:imported_visitors, date: ~D[2021-01-01], visitors: 2)+ ])++ import_data(+ [+ %{+ dimensions: %{"ga:date" => "20210101", "ga:deviceCategory" => "mobile"},+ metrics: %{+ "ga:bounces" => "0",+ "ga:sessionDuration" => "10",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }+ },+ %{+ dimensions: %{"ga:date" => "20210101", "ga:deviceCategory" => "Laptop"},+ metrics: %{+ "ga:bounces" => "0",+ "ga:sessionDuration" => "10",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }}- }- ],- site.id,- "imported_devices"- )-- conn =- get(- conn,- "/api/stats/#{site.domain}/screen-sizes?period=day&date=2021-01-01&with_imported=true"+ ],+ site.id,+ import_id,+ "imported_devices")- assert json_response(conn, 200) == [- %{"name" => "Desktop", "visitors" => 2, "percentage" => 40},- %{"name" => "Laptop", "visitors" => 2, "percentage" => 40},- %{"name" => "Mobile", "visitors" => 1, "percentage" => 20}- ]- end+ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/screen-sizes?period=day&date=2021-01-01&with_imported=true"+ )- test "Browsers data imported from Google Analytics", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-01 00:15:00]),- build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:15:00]),- build(:imported_visitors, visitors: 2, date: ~D[2021-01-01])- ])-- import_data(- [- %{- dimensions: %{- "ga:browser" => "User-Agent: Mozilla",- "ga:date" => "20210101"+ assert json_response(conn, 200) == [+ %{"name" => "Desktop", "visitors" => 2, "percentage" => 40},+ %{"name" => "Laptop", "visitors" => 2, "percentage" => 40},+ %{"name" => "Mobile", "visitors" => 1, "percentage" => 20}+ ]+ end++ test "Browsers data imported from Google Analytics", %{+ conn: conn,+ site: site,+ import_id: import_id+ } do+ populate_stats(site, import_id, [+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-01 00:15:00]),+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:15:00]),+ build(:imported_visitors, visitors: 2, date: ~D[2021-01-01])+ ])++ import_data(+ [+ %{+ dimensions: %{+ "ga:browser" => "User-Agent: Mozilla",+ "ga:date" => "20210101"+ },+ metrics: %{+ "ga:bounces" => "0",+ "ga:sessionDuration" => "10",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }},- metrics: %{- "ga:bounces" => "0",- "ga:sessionDuration" => "10",- "ga:sessions" => "1",- "ga:users" => "1"+ %{+ dimensions: %{"ga:browser" => "Android Browser", "ga:date" => "20210101"},+ metrics: %{+ "ga:bounces" => "0",+ "ga:sessionDuration" => "10",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }}- },- %{- dimensions: %{"ga:browser" => "Android Browser", "ga:date" => "20210101"},- metrics: %{- "ga:bounces" => "0",- "ga:sessionDuration" => "10",- "ga:sessions" => "1",- "ga:users" => "1"- }- }- ],- site.id,- "imported_browsers"- )-- conn =- get(- conn,- "/api/stats/#{site.domain}/browsers?period=day&date=2021-01-01&with_imported=true"+ ],+ site.id,+ import_id,+ "imported_browsers")- assert stats = json_response(conn, 200)- assert length(stats) == 3- assert %{"name" => "Firefox", "visitors" => 2, "percentage" => 50.0} in stats- assert %{"name" => "Mobile App", "visitors" => 1, "percentage" => 25.0} in stats- assert %{"name" => "Chrome", "visitors" => 1, "percentage" => 25.0} in stats- end-- test "OS data imported from Google Analytics", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview, operating_system: "Mac", timestamp: ~N[2021-01-01 00:15:00]),- build(:pageview, operating_system: "Mac", timestamp: ~N[2021-01-01 00:15:00]),- build(:pageview, operating_system: "GNU/Linux", timestamp: ~N[2021-01-01 00:15:00]),- build(:imported_visitors, date: ~D[2021-01-01], visitors: 2)- ])-- import_data(- [- %{- dimensions: %{"ga:date" => "20210101", "ga:operatingSystem" => "Macintosh"},- metrics: %{- "ga:bounces" => "0",- "ga:sessionDuration" => "10",- "ga:sessions" => "1",- "ga:users" => "1"- }- },- %{- dimensions: %{"ga:date" => "20210101", "ga:operatingSystem" => "Linux"},- metrics: %{- "ga:bounces" => "0",- "ga:sessionDuration" => "10",- "ga:sessions" => "1",- "ga:users" => "1"+ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/browsers?period=day&date=2021-01-01&with_imported=true"+ )++ assert stats = json_response(conn, 200)+ assert length(stats) == 3+ assert %{"name" => "Firefox", "visitors" => 2, "percentage" => 50.0} in stats+ assert %{"name" => "Mobile App", "visitors" => 1, "percentage" => 25.0} in stats+ assert %{"name" => "Chrome", "visitors" => 1, "percentage" => 25.0} in stats+ end++ test "OS data imported from Google Analytics", %{+ conn: conn,+ site: site,+ import_id: import_id+ } do+ populate_stats(site, import_id, [+ build(:pageview, operating_system: "Mac", timestamp: ~N[2021-01-01 00:15:00]),+ build(:pageview, operating_system: "Mac", timestamp: ~N[2021-01-01 00:15:00]),+ build(:pageview, operating_system: "GNU/Linux", timestamp: ~N[2021-01-01 00:15:00]),+ build(:imported_visitors, date: ~D[2021-01-01], visitors: 2)+ ])++ import_data(+ [+ %{+ dimensions: %{"ga:date" => "20210101", "ga:operatingSystem" => "Macintosh"},+ metrics: %{+ "ga:bounces" => "0",+ "ga:sessionDuration" => "10",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }+ },+ %{+ dimensions: %{"ga:date" => "20210101", "ga:operatingSystem" => "Linux"},+ metrics: %{+ "ga:bounces" => "0",+ "ga:sessionDuration" => "10",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }}- }- ],- site.id,- "imported_operating_systems"- )-- conn =- get(- conn,- "/api/stats/#{site.domain}/operating-systems?period=day&date=2021-01-01&with_imported=true"+ ],+ site.id,+ import_id,+ "imported_operating_systems")- assert json_response(conn, 200) == [- %{"name" => "Mac", "visitors" => 3, "percentage" => 60},- %{"name" => "GNU/Linux", "visitors" => 2, "percentage" => 40}- ]- end+ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/operating-systems?period=day&date=2021-01-01&with_imported=true"+ )- test "Can import visit duration with scientific notation", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, timestamp: ~N[2021-01-31 00:00:00])- ])-- import_data(- [- %{- dimensions: %{"ga:date" => "20210101"},- metrics: %{- "ga:bounces" => "0",- "ga:pageviews" => "1",- "ga:sessionDuration" => "1.391607E7",- "ga:sessions" => "1",- "ga:users" => "1"- }- },- %{- dimensions: %{"ga:date" => "20210131"},- metrics: %{- "ga:bounces" => "1",- "ga:pageviews" => "1",- "ga:sessionDuration" => "60",- "ga:sessions" => "1",- "ga:users" => "1"+ assert json_response(conn, 200) == [+ %{"name" => "Mac", "visitors" => 3, "percentage" => 60},+ %{"name" => "GNU/Linux", "visitors" => 2, "percentage" => 40}+ ]+ end++ test "Can import visit duration with scientific notation", %{+ conn: conn,+ site: site,+ import_id: import_id+ } do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-31 00:00:00])+ ])++ import_data(+ [+ %{+ dimensions: %{"ga:date" => "20210101"},+ metrics: %{+ "ga:bounces" => "0",+ "ga:pageviews" => "1",+ "ga:sessionDuration" => "1.391607E7",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }+ },+ %{+ dimensions: %{"ga:date" => "20210131"},+ metrics: %{+ "ga:bounces" => "1",+ "ga:pageviews" => "1",+ "ga:sessionDuration" => "60",+ "ga:sessions" => "1",+ "ga:users" => "1"+ }}- }- ],- site.id,- "imported_visitors"- )-- conn =- get(- conn,- "/api/stats/#{site.domain}/top-stats?period=month&date=2021-01-01&with_imported=true"+ ],+ site.id,+ import_id,+ "imported_visitors")- assert %{"top_stats" => top_stats} = json_response(conn, 200)-- visit_duration = Enum.find(top_stats, fn stat -> stat["name"] == "Visit duration" end)-- assert visit_duration["value"] == 3_479_033- end-- test "skips empty dates from import", %{conn: conn, site: site} do- import_data(- [- %{- dimensions: %{"ga:date" => "20210101"},- metrics: %{- "ga:users" => "1",- "ga:pageviews" => "1",- "ga:bounces" => "0",- "ga:sessions" => "1",- "ga:sessionDuration" => "60"- }- },- %{- dimensions: %{"ga:date" => "(other)"},- metrics: %{- "ga:users" => "1",- "ga:pageviews" => "1",- "ga:bounces" => "0",- "ga:sessions" => "1",- "ga:sessionDuration" => "60"+ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/top-stats?period=month&date=2021-01-01&with_imported=true"+ )++ assert %{"top_stats" => top_stats} = json_response(conn, 200)++ visit_duration = Enum.find(top_stats, fn stat -> stat["name"] == "Visit duration" end)++ assert visit_duration["value"] == 3_479_033+ end++ test "skips empty dates from import", %{conn: conn, site: site, import_id: import_id} do+ import_data(+ [+ %{+ dimensions: %{"ga:date" => "20210101"},+ metrics: %{+ "ga:users" => "1",+ "ga:pageviews" => "1",+ "ga:bounces" => "0",+ "ga:sessions" => "1",+ "ga:sessionDuration" => "60"+ }+ },+ %{+ dimensions: %{"ga:date" => "(other)"},+ metrics: %{+ "ga:users" => "1",+ "ga:pageviews" => "1",+ "ga:bounces" => "0",+ "ga:sessions" => "1",+ "ga:sessionDuration" => "60"+ }}- }- ],- site.id,- "imported_visitors"- )-- conn =- get(- conn,- "/api/stats/#{site.domain}/top-stats?period=month&date=2021-01-01&with_imported=true"+ ],+ site.id,+ import_id,+ "imported_visitors")- assert %{- "top_stats" => [- %{"name" => "Unique visitors", "value" => 1},- %{"name" => "Total visits", "value" => 1},- %{"name" => "Total pageviews", "value" => 1},- %{"name" => "Views per visit", "value" => +0.0},- %{"name" => "Bounce rate", "value" => 0},- %{"name" => "Visit duration", "value" => 60}- ]- } = json_response(conn, 200)+ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/top-stats?period=month&date=2021-01-01&with_imported=true"+ )++ assert %{+ "top_stats" => [+ %{"name" => "Unique visitors", "value" => 1},+ %{"name" => "Total visits", "value" => 1},+ %{"name" => "Total pageviews", "value" => 1},+ %{"name" => "Views per visit", "value" => +0.0},+ %{"name" => "Bounce rate", "value" => 0},+ %{"name" => "Visit duration", "value" => 60}+ ]+ } = json_response(conn, 200)+ endendendenddiff --git a/test/plausible/imported/universal_analytics_test.exs b/test/plausible/imported/universal_analytics_test.exsindex 6d842be7a3..80ad9caab6 100644--- a/test/plausible/imported/universal_analytics_test.exs+++ b/test/plausible/imported/universal_analytics_test.exs@@ -2,41 +2,58 @@ defmodule Plausible.Imported.UniversalAnalyticsTest douse Plausible.DataCase, async: trueuse Plausible.Test.Support.HTTPMocker+ alias Plausible.Imported.SiteImportalias Plausible.Imported.UniversalAnalytics+ require Plausible.Imported.SiteImport+setup [:create_user, :create_new_site]- describe "create_job/2 and parse_args/1" do- test "parses job args properly" do- site = insert(:site)- site_id = site.id+ describe "new_import/3 and parse_args/1" do+ test "parses job args properly", %{user: user, site: site} doexpires_at = NaiveDateTime.to_iso8601(NaiveDateTime.utc_now())- job =- UniversalAnalytics.create_job(site,- view_id: 123,- start_date: "2023-10-01",- end_date: "2024-01-02",- access_token: "access123",- refresh_token: "refresh123",- token_expires_at: expires_at- )+ assert {:ok, job} =+ UniversalAnalytics.new_import(site, user,+ view_id: 123,+ start_date: "2023-10-01",+ end_date: "2024-01-02",+ access_token: "access123",+ refresh_token: "refresh123",+ token_expires_at: expires_at+ )++ assert %Oban.Job{+ args:+ %{+ "import_id" => import_id,+ "view_id" => 123,+ "start_date" => "2023-10-01",+ "end_date" => "2024-01-02",+ "access_token" => "access123",+ "refresh_token" => "refresh123",+ "token_expires_at" => ^expires_at+ } = args+ } = Repo.reload!(job)++ assert [+ %{+ id: ^import_id,+ source: :universal_analytics,+ start_date: ~D[2023-10-01],+ end_date: ~D[2024-01-02],+ status: SiteImport.pending()+ }+ ] = Plausible.Imported.list_all_imports(site)- assert %Ecto.Changeset{- data: %Oban.Job{},- changes: %{- args:- %{- "site_id" => ^site_id,- "view_id" => 123,- "start_date" => "2023-10-01",- "end_date" => "2024-01-02",- "access_token" => "access123",- "refresh_token" => "refresh123",- "token_expires_at" => ^expires_at- } = args+ assert %{+ imported_data: %{+ source: "Google Analytics",+ start_date: ~D[2023-10-01],+ end_date: ~D[2024-01-02],+ status: "importing"}- } = job+ } = Repo.reload!(site)assert opts = [_ | _] = UniversalAnalytics.parse_args(args)@@ -46,23 +63,34 @@ defmodule Plausible.Imported.UniversalAnalyticsTest doendend- describe "import/2" do+ describe "import_data/2" do@tag :slow- test "imports page views from Google Analytics", %{site: site} do+ test "imports page views from Google Analytics", %{user: user, site: site} domock_http_with("google_analytics_import#1.json")view_id = "54297898"- date_range = Date.range(~D[2011-01-01], ~D[2022-07-19])+ start_date = ~D[2011-01-01]+ end_date = ~D[2022-07-19]future = DateTime.utc_now() |> DateTime.add(3600, :second) |> DateTime.to_iso8601()- auth = {"***", "refresh_token", future}+ access_token = "***"+ refresh_token = "refresh_token"- assert :ok ==- UniversalAnalytics.import(site,- date_range: date_range,- view_id: view_id,- auth: auth- )+ {:ok, job} =+ UniversalAnalytics.new_import(+ site,+ user,+ view_id: view_id,+ start_date: start_date,+ end_date: end_date,+ access_token: access_token,+ refresh_token: refresh_token,+ token_expires_at: future+ )++ job = Repo.reload!(job)++ Plausible.Workers.ImportAnalytics.perform(job)assert 1_495_150 == Plausible.Stats.Clickhouse.imported_pageview_count(site)enddiff --git a/test/plausible/purge_test.exs b/test/plausible/purge_test.exsindex e62b292449..edb7d27fec 100644--- a/test/plausible/purge_test.exs+++ b/test/plausible/purge_test.exs@@ -2,9 +2,28 @@ defmodule Plausible.PurgeTest douse Plausible.DataCasesetup do- site = insert(:site, stats_start_date: ~D[2020-01-01])+ user = insert(:user)+ site = insert(:site, members: [user], stats_start_date: ~D[2005-01-01])- populate_stats(site, [+ import_params = %{+ source: :universal_analytics,+ start_date: ~D[2005-01-01],+ end_date: Timex.today()+ }++ [site_import1, site_import2] =+ Enum.map(1..2, fn _ ->+ site+ |> Plausible.Imported.SiteImport.create_changeset(+ user,+ import_params+ )+ |> Plausible.Repo.insert!()+ |> Plausible.Imported.SiteImport.complete_changeset()+ |> Plausible.Repo.update!()+ end)++ populate_stats(site, site_import1.id, [build(:pageview),build(:imported_visitors),build(:imported_sources),@@ -17,13 +36,26 @@ defmodule Plausible.PurgeTest dobuild(:imported_operating_systems)])- {:ok, %{site: site}}+ populate_stats(site, site_import2.id, [+ build(:pageview),+ build(:imported_visitors),+ build(:imported_sources),+ build(:imported_pages),+ build(:imported_entry_pages),+ build(:imported_exit_pages),+ build(:imported_locations),+ build(:imported_devices),+ build(:imported_browsers),+ build(:imported_operating_systems)+ ])++ {:ok, %{site: site, site_import1: site_import1, site_import2: site_import2}}endtest "delete_imported_stats!/1 deletes imported data", %{site: site} doEnum.each(Plausible.Imported.tables(), fn table ->query = from(imported in table, where: imported.site_id == ^site.id)- assert await_clickhouse_count(query, 1)+ assert await_clickhouse_count(query, 2)end)assert :ok == Plausible.Purge.delete_imported_stats!(site)@@ -34,6 +66,29 @@ defmodule Plausible.PurgeTest doend)end+ test "delete_imported_stats!/1 deletes imported data only for a particular site import", %{+ site_import1: site_import1,+ site_import2: site_import2+ } do+ Enum.each(Plausible.Imported.tables(), fn table ->+ query = from(imported in table, where: imported.import_id == ^site_import1.id)+ assert await_clickhouse_count(query, 1)++ query = from(imported in table, where: imported.import_id == ^site_import2.id)+ assert await_clickhouse_count(query, 1)+ end)++ assert :ok == Plausible.Purge.delete_imported_stats!(site_import1)++ Enum.each(Plausible.Imported.tables(), fn table ->+ query = from(imported in table, where: imported.import_id == ^site_import1.id)+ assert await_clickhouse_count(query, 0)++ query = from(imported in table, where: imported.import_id == ^site_import2.id)+ assert await_clickhouse_count(query, 1)+ end)+ end+test "delete_imported_stats!/1 resets stats_start_date", %{site: site} doassert :ok == Plausible.Purge.delete_imported_stats!(site)assert %Plausible.Site{stats_start_date: nil} = Plausible.Repo.reload(site)diff --git a/test/plausible/stats/interval_test.exs b/test/plausible/stats/interval_test.exsindex 893b7b8c64..36e3c68093 100644--- a/test/plausible/stats/interval_test.exs+++ b/test/plausible/stats/interval_test.exs@@ -72,7 +72,7 @@ defmodule Plausible.Stats.IntervalTest dodescribe "valid_for_period/3" dotest "common" do- site = build(:site)+ site = insert(:site)assert valid_for_period?("month", "date", site: site)refute valid_for_period?("30d", "month", site: site)refute valid_for_period?("realtime", "week", site: site)diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exsindex 1182994431..697e34e20e 100644--- a/test/plausible/stats/query_test.exs+++ b/test/plausible/stats/query_test.exs@@ -120,7 +120,7 @@ defmodule Plausible.Stats.QueryTest doendtest "all time shows hourly if site is completely new", %{site: site} do- site = Map.put(site, :stats_start_date, Timex.now())+ site = Map.put(site, :stats_start_date, Timex.now() |> Timex.to_date())q = Query.from(site, %{"period" => "all"})assert q.date_range.first == Timex.today()@@ -130,7 +130,9 @@ defmodule Plausible.Stats.QueryTest doendtest "all time shows daily if site is more than a day old", %{site: site} do- site = Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(days: -1))+ site =+ Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(days: -1) |> Timex.to_date())+q = Query.from(site, %{"period" => "all"})assert q.date_range.first == Timex.today() |> Timex.shift(days: -1)@@ -140,7 +142,9 @@ defmodule Plausible.Stats.QueryTest doendtest "all time shows monthly if site is more than a month old", %{site: site} do- site = Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(months: -1))+ site =+ Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(months: -1) |> Timex.to_date())+q = Query.from(site, %{"period" => "all"})assert q.date_range.first == Timex.today() |> Timex.shift(months: -1)@@ -150,7 +154,9 @@ defmodule Plausible.Stats.QueryTest doendtest "all time uses passed interval different from the default interval", %{site: site} do- site = Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(months: -1))+ site =+ Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(months: -1) |> Timex.to_date())+q = Query.from(site, %{"period" => "all", "interval" => "week"})assert q.date_range.first == Timex.today() |> Timex.shift(months: -1)diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exsindex b08d878c94..538929ac88 100644--- a/test/plausible_web/controllers/site_controller_test.exs+++ b/test/plausible_web/controllers/site_controller_test.exs@@ -8,6 +8,10 @@ defmodule PlausibleWeb.SiteControllerTest doimport Moximport Plausible.Test.Support.HTML+ alias Plausible.Imported.SiteImport++ require Plausible.Imported.SiteImport+@v4_business_plan_id "857105"setup :verify_on_exit!@@ -1260,7 +1264,7 @@ defmodule PlausibleWeb.SiteControllerTest dodescribe "POST /:website/settings/google-import" dosetup [:create_user, :log_in, :create_new_site]- test "adds in-progress imported tag to site", %{conn: conn, site: site} do+ test "creates site import instance", %{conn: conn, site: site} dopost(conn, "/#{site.domain}/settings/google-import", %{"view_id" => "123","start_date" => "2018-03-01",@@ -1270,12 +1274,11 @@ defmodule PlausibleWeb.SiteControllerTest do"expires_at" => "2022-09-22T20:01:37.112777"})- imported_data = Repo.reload(site).imported_data+ [site_import] = Plausible.Imported.list_all_imports(site)- assert imported_data- assert imported_data.source == "Google Analytics"- assert imported_data.end_date == ~D[2022-03-01]- assert imported_data.status == "importing"+ assert site_import.source == :universal_analytics+ assert site_import.end_date == ~D[2022-03-01]+ assert site_import.status == SiteImport.pending()endtest "schedules an import job in Oban", %{conn: conn, site: site} do@@ -1288,11 +1291,12 @@ defmodule PlausibleWeb.SiteControllerTest do"expires_at" => "2022-09-22T20:01:37.112777"})+ assert [%{id: import_id}] = Plausible.Imported.list_all_imports(site)+assert_enqueued(worker: Plausible.Workers.ImportAnalytics,args: %{- "source" => "Google Analytics",- "site_id" => site.id,+ "import_id" => import_id,"view_id" => "123","start_date" => "2018-03-01","end_date" => "2022-03-01",@@ -1313,9 +1317,13 @@ defmodule PlausibleWeb.SiteControllerTest doassert Repo.reload(site).imported_data == nilend- test "removes actual imported data from Clickhouse", %{conn: conn, site: site} do- Plausible.Site.start_import(site, ~D[2022-01-01], Timex.today(), "Google Analytics")- |> Repo.update!()+ test "removes actual imported data from Clickhouse", %{conn: conn, user: user, site: site} do+ Plausible.Imported.NoopImporter.new_import(+ site,+ user,+ start_date: ~D[2022-01-01],+ end_date: Timex.today()+ )populate_stats(site, [build(:imported_visitors, pageviews: 10)@@ -1329,20 +1337,14 @@ defmodule PlausibleWeb.SiteControllerTest doend)end- test "cancels Oban job if it exists", %{conn: conn, site: site} do+ test "cancels Oban job if it exists", %{conn: conn, user: user, site: site} do{:ok, job} =- Plausible.Workers.ImportAnalytics.new(%{- "source" => "Google Analytics",- "site_id" => site.id,- "view_id" => "123",- "start_date" => "2022-01-01",- "end_date" => "2023-01-01",- "access_token" => "token"- })- |> Oban.insert()-- Plausible.Site.start_import(site, ~D[2022-01-01], Timex.today(), "Google Analytics")- |> Repo.update!()+ Plausible.Imported.NoopImporter.new_import(+ site,+ user,+ start_date: ~D[2022-01-01],+ end_date: Timex.today()+ )populate_stats(site, [build(:imported_visitors, pageviews: 10)diff --git a/test/support/test_utils.ex b/test/support/test_utils.exindex 034447acff..ac21fc29c3 100644--- a/test/support/test_utils.ex+++ b/test/support/test_utils.ex@@ -145,6 +145,21 @@ defmodule Plausible.TestUtils do:okend+ def populate_stats(site, import_id, events) do+ Enum.map(events, fn event ->+ event = Map.put(event, :site_id, site.id)++ case event do+ %Plausible.ClickhouseEventV2{} ->+ event++ imported_event ->+ Map.put(imported_event, :import_id, import_id)+ end+ end)+ |> populate_stats+ end+def populate_stats(site, events) doEnum.map(events, fn event ->Map.put(event, :site_id, site.id)diff --git a/test/workers/import_analytics_test.exs b/test/workers/import_analytics_test.exsindex cb9782ccbf..fe7f1ad392 100644--- a/test/workers/import_analytics_test.exs+++ b/test/workers/import_analytics_test.exs@@ -2,64 +2,77 @@ defmodule Plausible.Workers.ImportAnalyticsTest douse Plausible.DataCaseuse Bamboo.Test+ alias Plausible.Imported.SiteImportalias Plausible.Workers.ImportAnalytics+ require Plausible.Imported.SiteImport+@moduletag capture_log: true- describe "Universal Analytics" do+ describe "perform/1" dosetup do%{- imported_data: %Plausible.Site.ImportedData{+ import_opts: [start_date: Timex.today() |> Timex.shift(days: -7),- end_date: Timex.today(),- source: "Noop",- status: "importing"- }+ end_date: Timex.today()+ ]}end- test "updates the imported_data field for the site after successful import", %{- imported_data: imported_data+ test "updates site import after successful import", %{+ import_opts: import_opts} douser = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))- site = insert(:site, members: [user], imported_data: imported_data)+ site = insert(:site, members: [user])++ {:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)++ assert [%{status: SiteImport.pending()}] = Plausible.Imported.list_all_imports(site)++ # before_start callback triggered+ assert_received {:before_start, import_id}++ assert :ok =+ job+ |> Repo.reload!()+ |> ImportAnalytics.perform()- ImportAnalytics.perform(%Oban.Job{- args: %{- "source" => "Noop",- "site_id" => site.id- }- })+ assert [%{id: ^import_id, status: SiteImport.completed()}] =+ Plausible.Imported.list_all_imports(site)- assert Repo.reload!(site).imported_data.status == "ok"+ # on_success callback triggered+ assert_received {:on_success, ^import_id}end- test "updates the stats_start_date field for the site after successful import", %{- imported_data: imported_data+ test "clears stats_start_date field for the site after successful import", %{+ import_opts: import_opts} douser = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))- site = insert(:site, members: [user], imported_data: imported_data)+ site = insert(:site, members: [user], stats_start_date: ~D[2005-01-01])- ImportAnalytics.perform(%Oban.Job{- args: %{- "source" => "Noop",- "site_id" => site.id- }- })+ {:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)- assert Repo.reload!(site).stats_start_date == imported_data.start_date+ assert :ok =+ job+ |> Repo.reload!()+ |> ImportAnalytics.perform()++ site = Repo.reload!(site)+ assert site.stats_start_date == nil+ assert Plausible.Sites.stats_start_date(site) == import_opts[:start_date]+ assert Repo.reload!(site).stats_start_date == import_opts[:start_date]end- test "sends email to owner after successful import", %{imported_data: imported_data} do+ test "sends email to owner after successful import", %{import_opts: import_opts} douser = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))- site = insert(:site, members: [user], imported_data: imported_data)+ site = insert(:site, members: [user])++ {:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)- ImportAnalytics.perform(%Oban.Job{- args: %{- "source" => "Noop",- "site_id" => site.id- }- })+ assert :ok =+ job+ |> Repo.reload!()+ |> ImportAnalytics.perform()assert_email_delivered_with(to: [user],@@ -67,54 +80,57 @@ defmodule Plausible.Workers.ImportAnalyticsTest do)end- test "updates site record after failed import", %{imported_data: imported_data} do+ test "updates site import record after failed import", %{import_opts: import_opts} douser = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))- site = insert(:site, members: [user], imported_data: imported_data)+ site = insert(:site, members: [user])+ import_opts = Keyword.put(import_opts, :error, true)- ImportAnalytics.perform(%Oban.Job{- args: %{- "source" => "Noop",- "site_id" => site.id,- "error" => true- }- })+ {:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)- assert Repo.reload!(site).imported_data.status == "error"+ assert {:discard, "Something went wrong"} =+ job+ |> Repo.reload!()+ |> ImportAnalytics.perform()++ assert [%{status: SiteImport.failed()}] = Plausible.Imported.list_all_imports(site)end- test "clears any orphaned data during import", %{imported_data: imported_data} do+ test "clears any orphaned data during import", %{import_opts: import_opts} douser = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))- site = insert(:site, members: [user], imported_data: imported_data)+ site = insert(:site, members: [user])+ import_opts = Keyword.put(import_opts, :error, true)++ {:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)populate_stats(site, [- build(:imported_visitors, pageviews: 10)+ build(:imported_visitors, import_id: job.args.import_id, pageviews: 10)])- ImportAnalytics.perform(%Oban.Job{- args: %{- "source" => "Noop",- "site_id" => site.id,- "error" => true- }- })+ assert {:discard, _} =+ job+ |> Repo.reload!()+ |> ImportAnalytics.perform()assert eventually(fn ->count = Plausible.Stats.Clickhouse.imported_pageview_count(site){count == 0, count}end)++ # on_failure callback triggered+ assert_received {:on_failure, _import_id}end- test "sends email to owner after failed import", %{imported_data: imported_data} do+ test "sends email to owner after failed import", %{import_opts: import_opts} douser = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))- site = insert(:site, members: [user], imported_data: imported_data)+ site = insert(:site, members: [user])+ import_opts = Keyword.put(import_opts, :error, true)++ {:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)- ImportAnalytics.perform(%Oban.Job{- args: %{- "source" => "Noop",- "site_id" => site.id,- "error" => true- }- })+ assert {:discard, _} =+ job+ |> Repo.reload!()+ |> ImportAnalytics.perform()assert_email_delivered_with(to: [user],@@ -122,4 +138,96 @@ defmodule Plausible.Workers.ImportAnalyticsTest do)endend++ describe "perform/1 notifications" do+ setup do+ on_exit(fn ->+ Ecto.Adapters.SQL.Sandbox.unboxed_run(Plausible.Repo, fn ->+ Repo.delete_all(Plausible.Site)+ Repo.delete_all(Plausible.Auth.User)+ end)++ :ok+ end)++ %{+ import_opts: [+ start_date: Timex.today() |> Timex.shift(days: -7),+ end_date: Timex.today()+ ]+ }+ end++ test "sends oban notification to calling process on completion when instructed", %{+ import_opts: import_opts+ } do+ Ecto.Adapters.SQL.Sandbox.unboxed_run(Plausible.Repo, fn ->+ user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))+ site = insert(:site, members: [user])+ import_opts = Keyword.put(import_opts, :listen?, true)++ {:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)+ import_id = job.args[:import_id]++ job+ |> Repo.reload!()+ |> ImportAnalytics.perform()++ assert_receive {:notification, :analytics_imports_jobs, %{"complete" => ^import_id}}+ end)+ end++ test "sends oban notification to calling process on permanent failure when instructed", %{+ import_opts: import_opts+ } do+ Ecto.Adapters.SQL.Sandbox.unboxed_run(Plausible.Repo, fn ->+ user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))+ site = insert(:site, members: [user])++ import_opts =+ import_opts+ |> Keyword.put(:listen?, true)+ |> Keyword.put(:error, true)++ {:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)+ import_id = job.args[:import_id]++ job+ |> Repo.reload!()+ |> ImportAnalytics.perform()++ assert_receive {:notification, :analytics_imports_jobs, %{"fail" => ^import_id}}+ end)+ end++ test "sends oban notification to calling process on transient failure when instructed", %{+ import_opts: import_opts+ } do+ Ecto.Adapters.SQL.Sandbox.unboxed_run(Plausible.Repo, fn ->+ user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))+ site = insert(:site, members: [user])++ import_opts =+ import_opts+ |> Keyword.put(:listen?, true)+ |> Keyword.put(:crash, true)++ {:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)+ import_id = job.args[:import_id]++ site_import = Repo.get(Plausible.Imported.SiteImport, import_id)++ # emulate oban error handler+ try do+ job+ |> Repo.reload!()+ |> ImportAnalytics.perform()+ rescue+ _ -> ImportAnalytics.import_fail_transient(site_import)+ end++ assert_receive {:notification, :analytics_imports_jobs, %{"transient_fail" => ^import_id}}+ end)+ end+ endend