Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Keep track of native stats start timestamp when retrieving data (#2715)
Adam8f86036e570c
diff --git a/config/runtime.exs b/config/runtime.exsindex 6870714027..bbbb8d3ee7 100644--- a/config/runtime.exs+++ b/config/runtime.exs@@ -401,8 +401,7 @@ base_queues = [site_setup_emails: 1,clean_email_verification_codes: 1,clean_invitations: 1,- google_analytics_imports: 1,- site_stats_removal: 1+ google_analytics_imports: 1]cloud_queues = [diff --git a/lib/plausible/purge.ex b/lib/plausible/purge.exindex 13865a7004..a4be392136 100644--- a/lib/plausible/purge.ex+++ b/lib/plausible/purge.ex@@ -27,19 +27,23 @@ defmodule Plausible.Purge do@spec delete_native_stats!(Plausible.Site.t()) :: :ok@doc """- Deletes native stats for a site, and clears the `stats_start_date` field.+ Move stats pointers so that no historical stats are available."""def delete_native_stats!(site) do- events_sql = "ALTER TABLE events DELETE WHERE domain = ?"- sessions_sql = "ALTER TABLE sessions DELETE WHERE domain = ?"- Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, events_sql, [site.domain])- Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, sessions_sql, [site.domain])-- clear_stats_start_date!(site)+ reset!(site):okend+ def reset!(site) do+ site+ |> Ecto.Changeset.change(+ native_stats_start_at: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second),+ stats_start_date: nil+ )+ |> Plausible.Repo.update!()+ end+defp clear_stats_start_date!(site) dosite|> Ecto.Changeset.change(stats_start_date: nil)diff --git a/lib/plausible/site.ex b/lib/plausible/site.exindex 8c2859dd78..97397ffbfe 100644--- a/lib/plausible/site.ex+++ b/lib/plausible/site.ex@@ -16,6 +16,7 @@ defmodule Plausible.Site dofield :public, :booleanfield :locked, :booleanfield :stats_start_date, :date+ field :native_stats_start_at, :naive_datetimefield :ingest_rate_limit_scale_seconds, :integer, default: 60field :ingest_rate_limit_threshold, :integerdiff --git a/lib/plausible/site/removal.ex b/lib/plausible/site/removal.exindex c8bd91ad72..af1fd6387f 100644--- a/lib/plausible/site/removal.ex+++ b/lib/plausible/site/removal.ex@@ -1,41 +1,14 @@defmodule Plausible.Site.Removal do@moduledoc """- A service responsible for site and its stats deletion.- The site deletion alone is done in postgres and is executed first,- the latter deletions (events, sessions and imported tables in clickhouse)- are performed asynchrnounsly via `Plausible.Workers.StatsRemoval`.-- This is to avoid race condition in which the site is deleted, but stats- writes are pending (either in the buffers or are about to be buffered, due- to Sites.Cache keeping the now obsolete record until refresh is triggered).+ A site deletion service stub."""- @stats_deletion_delay_seconds 60 * 20-- alias Plausible.Workers.StatsRemovalalias Plausible.Repo- alias Ecto.Multiimport Ecto.Query- @spec stats_deletion_delay_seconds() :: pos_integer()- def stats_deletion_delay_seconds() do- @stats_deletion_delay_seconds- end-@spec run(String.t()) :: {:ok, map()}def run(domain) do- site_by_domain_q = from s in Plausible.Site, where: s.domain == ^domain-- Multi.new()- |> Multi.run(:site_id, fn _, _ ->- {:ok, Repo.one(from s in site_by_domain_q, select: s.id)}- end)- |> Multi.delete_all(:delete_all, site_by_domain_q)- |> Oban.insert(:delayed_metrics_removal, fn %{site_id: site_id} ->- StatsRemoval.new(%{domain: domain, site_id: site_id},- schedule_in: stats_deletion_delay_seconds()- )- end)- |> Repo.transaction()+ result = Repo.delete_all(from(s in Plausible.Site, where: s.domain == ^domain))+ {:ok, %{delete_all: result}}endenddiff --git a/lib/plausible/sites.ex b/lib/plausible/sites.exindex 67fabbd72b..e791986089 100644--- a/lib/plausible/sites.ex+++ b/lib/plausible/sites.ex@@ -24,13 +24,6 @@ defmodule Plausible.Sites doendend)|> Ecto.Multi.insert(:site, site_changeset)- |> Ecto.Multi.run(:existing_events, fn _, _ ->- site_changeset- |> Ecto.Changeset.validate_change(:domain, fn :domain, domain ->- check_for_existing_events(domain, params)- end)- |> Ecto.Changeset.apply_action(:insert)- end)|> Ecto.Multi.run(:site_membership, fn repo, %{site: site} ->membership_changeset =Site.Membership.changeset(%Site.Membership{}, %{@@ -183,19 +176,4 @@ defmodule Plausible.Sites dowhere: sm.role == :owner)end-- defp check_for_existing_events(domain, params) do- if has_events?(domain) do- Sentry.capture_message("Refused to create a site with existing events",- extra: %{params: params}- )-- [- domain:- "This domain cannot be registered. Perhaps one of your colleagues registered it? Or did you recently delete it from your account? The deletion may take up to 48 hours before you can add the same site again. If that's not the case, please contact [email protected]"- ]- else- []- end- endenddiff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.exindex 9e42c536a1..7bba222dc9 100644--- a/lib/plausible/stats/base.ex+++ b/lib/plausible/stats/base.ex@@ -31,7 +31,7 @@ defmodule Plausible.Stats.Base doenddef query_events(site, query) do- {first_datetime, last_datetime} = utc_boundaries(query, site.timezone)+ {first_datetime, last_datetime} = utc_boundaries(query, site)q =from(@@ -145,7 +145,7 @@ defmodule Plausible.Stats.Base do}def query_sessions(site, query) do- {first_datetime, last_datetime} = utc_boundaries(query, site.timezone)+ {first_datetime, last_datetime} = utc_boundaries(query, site)sessions_q =from(@@ -395,31 +395,48 @@ defmodule Plausible.Stats.Base dodefp db_prop_val(_, @not_set), do: ""defp db_prop_val(_, val), do: val- def utc_boundaries(%Query{period: "realtime"}, _timezone) do- last_datetime = NaiveDateTime.utc_now() |> Timex.shift(seconds: 5)+ defp beginning_of_time(candidate, native_stats_start_at) do+ if Timex.after?(native_stats_start_at, candidate) do+ native_stats_start_at+ else+ candidate+ end+ end++ def utc_boundaries(%Query{period: "realtime"}, site) do+ last_datetime =+ NaiveDateTime.utc_now()+ |> Timex.shift(seconds: 5)+ |> beginning_of_time(site.native_stats_start_at)+first_datetime = NaiveDateTime.utc_now() |> Timex.shift(minutes: -5){first_datetime, last_datetime}end- def utc_boundaries(%Query{period: "30m"}, _timezone) do- last_datetime = NaiveDateTime.utc_now() |> Timex.shift(seconds: 5)+ def utc_boundaries(%Query{period: "30m"}, site) do+ last_datetime =+ NaiveDateTime.utc_now()+ |> Timex.shift(seconds: 5)+ |> beginning_of_time(site.native_stats_start_at)+first_datetime = NaiveDateTime.utc_now() |> Timex.shift(minutes: -30){first_datetime, last_datetime}end- def utc_boundaries(%Query{date_range: date_range}, timezone) do+ def utc_boundaries(%Query{date_range: date_range}, site) do{:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00])first_datetime =- Timex.to_datetime(first, timezone)+ Timex.to_datetime(first, site.timezone)|> Timex.Timezone.convert("UTC")+ |> beginning_of_time(site.native_stats_start_at){:ok, last} = NaiveDateTime.new(date_range.last |> Timex.shift(days: 1), ~T[00:00:00])last_datetime =- Timex.to_datetime(last, timezone)+ Timex.to_datetime(last, site.timezone)|> Timex.Timezone.convert("UTC"){first_datetime, last_datetime}diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.exindex 32077ebb43..5ca3551c99 100644--- a/lib/plausible/stats/clickhouse.ex+++ b/lib/plausible/stats/clickhouse.ex@@ -11,7 +11,8 @@ defmodule Plausible.Stats.Clickhouse doClickhouseRepo.one(from e in "events",select: fragment("min(?)", e.timestamp),- where: e.domain == ^site.domain+ where: e.domain == ^site.domain,+ where: e.timestamp >= ^site.native_stats_start_at)case datetime do@@ -155,19 +156,15 @@ defmodule Plausible.Stats.Clickhouse do)end- def has_pageviews?([]), do: false-- def has_pageviews?(domains) when is_list(domains) do- ClickhouseRepo.exists?(- from e in "events",- select: e.timestamp,- where: fragment("? IN tuple(?)", e.domain, ^domains)- )- end-def has_pageviews?(site) doClickhouseRepo.exists?(- from e in "events", where: e.domain == ^site.domain and e.name == "pageview"+ from(e in "events",+ where:+ e.domain == ^site.domain and+ e.name == "pageview" and+ e.timestamp >=+ ^site.native_stats_start_at+ ))end@@ -187,7 +184,7 @@ defmodule Plausible.Stats.Clickhouse doenddefp base_session_query(site, query) do- {first_datetime, last_datetime} = utc_boundaries(query, site.timezone)+ {first_datetime, last_datetime} = utc_boundaries(query, site)q =from(s in "sessions",@@ -306,7 +303,7 @@ defmodule Plausible.Stats.Clickhouse doenddefp base_query_bare(site, query) do- {first_datetime, last_datetime} = utc_boundaries(query, site.timezone)+ {first_datetime, last_datetime} = utc_boundaries(query, site)q =from(e in "events",@@ -437,31 +434,36 @@ defmodule Plausible.Stats.Clickhouse dobase_query_bare(site, query) |> include_goal_conversions(query)end- defp utc_boundaries(%Query{period: "30m"}, _timezone) do+ defp utc_boundaries(%Query{period: "30m"}, site) dolast_datetime = NaiveDateTime.utc_now()- first_datetime = last_datetime |> Timex.shift(minutes: -30)+ first_datetime =+ last_datetime |> Timex.shift(minutes: -30) |> beginning_of_time(site.native_stats_start_at)+{first_datetime, last_datetime}end- defp utc_boundaries(%Query{period: "realtime"}, _timezone) do+ defp utc_boundaries(%Query{period: "realtime"}, site) dolast_datetime = NaiveDateTime.utc_now()- first_datetime = last_datetime |> Timex.shift(minutes: -5)+ first_datetime =+ last_datetime |> Timex.shift(minutes: -5) |> beginning_of_time(site.native_stats_start_at)+{first_datetime, last_datetime}end- defp utc_boundaries(%Query{date_range: date_range}, timezone) do+ defp utc_boundaries(%Query{date_range: date_range}, site) do{:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00])first_datetime =- Timex.to_datetime(first, timezone)+ Timex.to_datetime(first, site.timezone)|> Timex.Timezone.convert("UTC")+ |> beginning_of_time(site.native_stats_start_at){:ok, last} = NaiveDateTime.new(date_range.last |> Timex.shift(days: 1), ~T[00:00:00])last_datetime =- Timex.to_datetime(last, timezone)+ Timex.to_datetime(last, site.timezone)|> Timex.Timezone.convert("UTC"){first_datetime, last_datetime}@@ -589,4 +591,12 @@ defmodule Plausible.Stats.Clickhouse dodb_queryendend++ defp beginning_of_time(candidate, site_creation_date) do+ if Timex.after?(site_creation_date, candidate) do+ site_creation_date+ else+ candidate+ end+ endenddiff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.exindex 3788f734f2..789a99c38b 100644--- a/lib/plausible/stats/imported.ex+++ b/lib/plausible/stats/imported.ex@@ -1,8 +1,8 @@defmodule Plausible.Stats.Imported douse Plausible.ClickhouseRepoalias Plausible.Stats.Query+import Ecto.Query- import Plausible.Stats.Baseimport Plausible.Stats.Fragments@no_ref "Direct / None"@@ -23,7 +23,7 @@ defmodule Plausible.Stats.Imported doselect: %{})|> select_imported_metrics(metrics)- |> apply_interval(query, site.timezone)+ |> apply_interval(query, site)from(s in Ecto.Query.subquery(native_q),full_join: i in subquery(imported_q),@@ -33,21 +33,19 @@ defmodule Plausible.Stats.Imported do|> select_joined_metrics(metrics)end- defp apply_interval(imported_q, %Plausible.Stats.Query{interval: "month"}, _timezone) do+ defp apply_interval(imported_q, %Plausible.Stats.Query{interval: "month"}, _site) doimported_q|> group_by([i], fragment("toStartOfMonth(?)", i.date))|> select_merge([i], %{date: fragment("toStartOfMonth(?)", i.date)})end- defp apply_interval(imported_q, %Plausible.Stats.Query{interval: "week"} = query, timezone) do- {first_datetime, _} = utc_boundaries(query, timezone)-+ defp apply_interval(imported_q, %Plausible.Stats.Query{interval: "week"} = query, _site) doimported_q- |> group_by([i], weekstart_not_before(i.date, ^first_datetime))- |> select_merge([i], %{date: weekstart_not_before(i.date, ^first_datetime)})+ |> group_by([i], weekstart_not_before(i.date, ^query.date_range.first))+ |> select_merge([i], %{date: weekstart_not_before(i.date, ^query.date_range.first)})end- defp apply_interval(imported_q, _query, _timezone) do+ defp apply_interval(imported_q, _query, _site) doimported_q|> group_by([i], i.date)|> select_merge([i], %{date: i.date})diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.exindex 37ca49aa59..6b330d6bee 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/timeseries.ex@@ -121,7 +121,7 @@ defmodule Plausible.Stats.Timeseries doenddefp select_bucket(q, site, %Query{interval: "week"} = query) do- {first_datetime, _} = utc_boundaries(query, site.timezone)+ {first_datetime, _} = utc_boundaries(query, site)from(e in q,diff --git a/lib/plausible_web/controllers/api/internal_controller.ex b/lib/plausible_web/controllers/api/internal_controller.exindex 31b16ecf6d..40ea59b92b 100644--- a/lib/plausible_web/controllers/api/internal_controller.ex+++ b/lib/plausible_web/controllers/api/internal_controller.ex@@ -4,7 +4,9 @@ defmodule PlausibleWeb.Api.InternalController doalias Plausible.Stats.Clickhouse, as: Statsdef domain_status(conn, %{"domain" => domain}) do- if Stats.has_pageviews?(%Plausible.Site{domain: domain}) do+ site = Plausible.Sites.get_by_domain(domain)++ if Stats.has_pageviews?(site) dojson(conn, "READY")elsejson(conn, "WAITING")diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.exindex 145b6fc3c7..51f293402b 100644--- a/lib/plausible_web/controllers/site_controller.ex+++ b/lib/plausible_web/controllers/site_controller.ex@@ -349,7 +349,7 @@ defmodule PlausibleWeb.SiteController dodef reset_stats(conn, _params) dosite = conn.assigns[:site]- Plausible.Purge.delete_native_stats!(site)+ Plausible.Purge.reset!(site)conn|> put_flash(:success, "#{site.domain} stats will be reset in a few minutes")diff --git a/lib/workers/stats_removal.ex b/lib/workers/stats_removal.exdeleted file mode 100644index 6ca86ba0d7..0000000000--- a/lib/workers/stats_removal.ex+++ /dev/null@@ -1,59 +0,0 @@-defmodule Plausible.Workers.StatsRemoval do- @moduledoc """- Asynchronous worker firing deletion mutations to clickhouse.- For now only ALTER TABLE deletions are supported. Experimental- DELETE FROM support is going to be introduced once production db- is upgraded.-- At most 3 attempts are made, with 15m backoff value.-- Imported stats tables keep site reference through a numeric id, whilist- events and sessions store domain as-is - hence two different deletes,- one of which cannot be performed anymore once the site identifier is permanently- gone from postgres.- """- use Plausible.Repo-- use Oban.Worker,- queue: :site_stats_removal,- max_attempts: 3,- unique: [period: :infinity, fields: [:args]]-- @impl Oban.Worker- def perform(%{args: args}) do- domain = Map.fetch!(args, "domain")- site_id = Map.get(args, "site_id")-- imported_result = delete_imported!(site_id)- native_result = delete_native!(domain)-- {:ok, Map.merge(imported_result, native_result)}- end-- @impl Oban.Worker- def backoff(_job) do- 15 * 60- end-- defp delete_imported!(nil) do- %{}- end-- defp delete_imported!(id) when is_integer(id) do- Enum.map(Plausible.Imported.tables(), fn table ->- sql = "ALTER TABLE #{table} DELETE WHERE site_id = ?"- {table, Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, sql, [id])}- end)- |> Enum.into(%{})- end-- defp delete_native!(domain) do- events_sql = "ALTER TABLE events DELETE WHERE domain = ?"- sessions_sql = "ALTER TABLE sessions DELETE WHERE domain = ?"-- %{- "events" => Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, events_sql, [domain]),- "sessions" => Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, sessions_sql, [domain])- }- end-enddiff --git a/priv/repo/seeds.exs b/priv/repo/seeds.exsindex d9cb1904c1..91398f2fe0 100644--- a/priv/repo/seeds.exs+++ b/priv/repo/seeds.exs@@ -12,16 +12,29 @@user = Plausible.Factory.insert(:user, email: "[email protected]", password: "plausible")-site = Plausible.Factory.insert(:site, domain: "dummy.site")+beginning_of_time = NaiveDateTime.add(NaiveDateTime.utc_now(), -721, :day)++site =+ Plausible.Factory.insert(:site, domain: "dummy.site", native_stats_start_at: beginning_of_time)_membership = Plausible.Factory.insert(:site_membership, user: user, site: site, role: :owner)-put_random_time = fn date ->- random_time = Time.new!(:rand.uniform(23), :rand.uniform(59), 0)+put_random_time = fn+ date, 0 ->+ current_hour = Time.utc_now().hour+ current_minute = Time.utc_now().minute+ random_time = Time.new!(:rand.uniform(current_hour), :rand.uniform(current_minute - 1), 0)++ date+ |> NaiveDateTime.new!(random_time)+ |> NaiveDateTime.truncate(:second)++ date, _ ->+ random_time = Time.new!(:rand.uniform(23), :rand.uniform(59), 0)- date- |> NaiveDateTime.new!(random_time)- |> NaiveDateTime.truncate(:second)+ date+ |> NaiveDateTime.new!(random_time)+ |> NaiveDateTime.truncate(:second)endgeolocations = [@@ -74,7 +87,7 @@ Enum.flat_map(-720..0, fn day_index ->[domain: site.domain,hostname: site.domain,- timestamp: put_random_time.(date),+ timestamp: put_random_time.(date, day_index),referrer_source: Enum.random(["", "Facebook", "Twitter", "DuckDuckGo", "Google"]),browser: Enum.random(["Edge", "Chrome", "Safari", "Firefox", "Vivaldi"]),browser_version: to_string(Enum.random(0..50)),diff --git a/test/plausible/purge_test.exs b/test/plausible/purge_test.exsindex a05fc51e9b..e62b292449 100644--- a/test/plausible/purge_test.exs+++ b/test/plausible/purge_test.exs@@ -39,17 +39,13 @@ defmodule Plausible.PurgeTest doassert %Plausible.Site{stats_start_date: nil} = Plausible.Repo.reload(site)end- test "delete_native_stats!/1 deletes native stats", %{site: site} do- events_query = from(s in Plausible.ClickhouseEvent, where: s.domain == ^site.domain)- assert await_clickhouse_count(events_query, 1)-- sessions_query = from(s in Plausible.ClickhouseSession, where: s.domain == ^site.domain)- assert await_clickhouse_count(sessions_query, 1)-+ test "delete_native_stats!/1 moves the native_stats_start_at pointer", %{site: site} doassert :ok == Plausible.Purge.delete_native_stats!(site)- assert await_clickhouse_count(events_query, 0)- assert await_clickhouse_count(sessions_query, 0)+ assert %Plausible.Site{native_stats_start_at: native_stats_start_at} =+ Plausible.Repo.reload(site)++ assert NaiveDateTime.compare(native_stats_start_at, site.native_stats_start_at) == :gtendtest "delete_native_stats!/1 resets stats_start_date", %{site: site} dodiff --git a/test/plausible/site/site_removal_test.exs b/test/plausible/site/site_removal_test.exsindex 544acbf17c..931cb29dff 100644--- a/test/plausible/site/site_removal_test.exs+++ b/test/plausible/site/site_removal_test.exs@@ -4,113 +4,16 @@ defmodule Plausible.Site.SiteRemovalTest doalias Plausible.Site.Removalalias Plausible.Sites- alias Plausible.Workers.StatsRemoval- describe "execution and scheduling" do- test "site from postgres is immediately deleted" do- site = insert(:site)- assert {:ok, context} = Removal.run(site.domain)- assert context.delete_all == {1, nil}- assert context.site_id == site.id- refute Sites.get_by_domain(site.domain)- end-- test "deletion is idempotent" do- assert {:ok, context} = Removal.run("some.example.com")- assert context.delete_all == {0, nil}- end-- test "stats deletion job is scheduled when no site exists in postgres" do- assert {:ok, _} = Removal.run("a.domain.example.com")-- assert_enqueued(- worker: StatsRemoval,- args: %{"domain" => "a.domain.example.com", "site_id" => nil}- )- end-- test "stats deletion job is scheduled when site exists in postgres" do- site = insert(:site)- assert {:ok, _} = Removal.run(site.domain)-- assert_enqueued(- worker: StatsRemoval,- args: %{"domain" => site.domain, "site_id" => site.id}- )- end-- test "stats deletion is always scheduled ~20m in the future" do- assert {:ok, _} = Removal.run("foo.example.com")-- in_20m = DateTime.add(DateTime.utc_now(), 1200, :second)-- assert_enqueued(- worker: StatsRemoval,- scheduled_at: {in_20m, delta: 5}- )- end-- test "stats deletion is always scheduled late enough for sites cache to expire" do- delay_ms = Removal.stats_deletion_delay_seconds() * 1000- assert delay_ms > Plausible.Site.Cache.Warmer.interval()- end- end-- describe "the background worker" do- test "the job runs deletes at clickhouse" do- assert {:ok, %{"events" => r, "sessions" => r}} =- perform_job(StatsRemoval, %{"domain" => "foo.example.com"})-- assert %Clickhousex.Result{command: :updated} = r-- assert {:ok, %{"events" => r, "sessions" => r, "imported_browsers" => r}} =- perform_job(StatsRemoval, %{"domain" => "foo.example.com", "site_id" => 777})-- assert %Clickhousex.Result{command: :updated} = r- end+ test "site from postgres is immediately deleted" do+ site = insert(:site)+ assert {:ok, context} = Removal.run(site.domain)+ assert context.delete_all == {1, nil}+ refute Sites.get_by_domain(site.domain)end- describe "integration" do- setup do- site = insert(:site, stats_start_date: ~D[2020-01-01])-- populate_stats(site, [- build(:pageview),- build(:imported_visitors),- build(:imported_sources),- build(:imported_pages),- build(:imported_entry_pages),- build(:imported_exit_pages),- build(:imported_locations),- build(:imported_devices),- build(:imported_browsers),- build(:imported_operating_systems)- ])-- {:ok, %{site: site}}- end-- test "the job actually removes stats from clickhouse", %{site: site} do- Enum.each(Plausible.Imported.tables(), fn table ->- query = from(imported in table, where: imported.site_id == ^site.id)- assert await_clickhouse_count(query, 1)- end)-- events_query = from(s in Plausible.ClickhouseEvent, where: s.domain == ^site.domain)- assert await_clickhouse_count(events_query, 1)-- sessions_query = from(s in Plausible.ClickhouseSession, where: s.domain == ^site.domain)- assert await_clickhouse_count(sessions_query, 1)-- perform_job(StatsRemoval, %{"domain" => site.domain, "site_id" => site.id})-- assert await_clickhouse_count(events_query, 0)- assert await_clickhouse_count(sessions_query, 0)-- Enum.each(Plausible.Imported.tables(), fn table ->- query = from(imported in table, where: imported.site_id == ^site.id)- assert await_clickhouse_count(query, 0)- end)- end+ test "deletion is idempotent" do+ assert {:ok, context} = Removal.run("some.example.com")+ assert context.delete_all == {0, nil}endenddiff --git a/test/plausible_web/controllers/api/external_sites_controller_test.exs b/test/plausible_web/controllers/api/external_sites_controller_test.exsindex 8fb28022d7..25e5b15ea0 100644--- a/test/plausible_web/controllers/api/external_sites_controller_test.exs+++ b/test/plausible_web/controllers/api/external_sites_controller_test.exs@@ -79,29 +79,6 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do}end- test "does not allow creating a site when external events are present", %{- conn: conn- } do- domain = "events-exist.example.com"-- populate_stats(%{domain: domain}, [- build(:pageview)- ])-- :inserted = eventually(fn -> {Plausible.Sites.has_events?(domain), :inserted} end)-- conn =- post(conn, "/api/v1/sites", %{- "domain" => domain,- "timezone" => "Europe/Tallinn"- })-- assert json_response(conn, 400) == %{- "error" =>- "domain: This domain cannot be registered. Perhaps one of your colleagues registered it? Or did you recently delete it from your account? The deletion may take up to 48 hours before you can add the same site again. If that's not the case, please contact [email protected]"- }- end-test "cannot access with a bad API key scope", %{conn: conn, user: user} doapi_key = insert(:api_key, user: user, scopes: ["stats:read:*"])diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exsindex 0230510eba..766f82c99b 100644--- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs@@ -41,7 +41,12 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest dotest "displays hourly stats in configured timezone", %{conn: conn, user: user} do# UTC+1- site = insert(:site, domain: "tz-test.com", members: [user], timezone: "CET")+ site =+ insert(:site,+ domain: "tz-test.com",+ members: [user],+ timezone: "CET"+ )populate_stats(site, [build(:pageview, timestamp: ~N[2021-01-01 00:00:00])diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exsindex 11f3d0f53d..647b9df834 100644--- a/test/plausible_web/controllers/site_controller_test.exs+++ b/test/plausible_web/controllers/site_controller_test.exs@@ -108,29 +108,6 @@ defmodule PlausibleWeb.SiteControllerTest doassert Repo.get_by(Plausible.Site, domain: "example.com")end- test "refuses to create the site when events exist (pending deletion)", %{conn: conn} do- domain = "events-exist.example.com"-- populate_stats(%{domain: domain}, [- build(:pageview)- ])-- :inserted = eventually(fn -> {Plausible.Sites.has_events?(domain), :inserted} end)-- conn =- post(conn, "/sites", %{- "site" => %{- "domain" => domain,- "timezone" => "Europe/London"- }- })-- assert html = html_response(conn, 200)- assert html =~ "This domain cannot be registered"- assert html =~ "please contact support"- refute Repo.get_by(Plausible.Site, domain: domain)- end-test "starts trial if user does not have trial yet", %{conn: conn, user: user} doPlausible.Auth.User.remove_trial_expiry(user) |> Repo.update!()diff --git a/test/support/factory.ex b/test/support/factory.exindex 9797fb2fb3..86f5d5719a 100644--- a/test/support/factory.ex+++ b/test/support/factory.ex@@ -25,6 +25,7 @@ defmodule Plausible.Factory dodomain = sequence(:domain, &"example-#{&1}.com")%Plausible.Site{+ native_stats_start_at: ~N[2000-01-01 00:00:00],domain: domain,timezone: "UTC"}diff --git a/test/support/test_utils.ex b/test/support/test_utils.exindex 92b9cc18c9..bb506edc50 100644--- a/test/support/test_utils.ex+++ b/test/support/test_utils.ex@@ -35,7 +35,12 @@ defmodule Plausible.TestUtils doenddef create_site(%{user: user}) do- site = Factory.insert(:site, domain: "test-site.com", members: [user])+ site =+ Factory.insert(:site,+ domain: "test-site.com",+ members: [user]+ )+{:ok, site: site}enddiff --git a/test/workers/send_email_report_test.exs b/test/workers/send_email_report_test.exsindex 9b70110c92..0b22fd5577 100644--- a/test/workers/send_email_report_test.exs+++ b/test/workers/send_email_report_test.exs@@ -24,7 +24,11 @@ defmodule Plausible.Workers.SendEmailReportTest doendtest "calculates timezone correctly" do- site = insert(:site, timezone: "US/Eastern")+ site =+ insert(:site,+ timezone: "US/Eastern"+ )+insert(:weekly_report, site: site, recipients: ["[email protected]"])now = Timex.now(site.timezone)@@ -61,9 +65,9 @@ defmodule Plausible.Workers.SendEmailReportTest doendtest "includes the correct stats" do- site = insert(:site, domain: "test-site.com")- insert(:weekly_report, site: site, recipients: ["[email protected]"])now = NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second)+ site = insert(:site, domain: "test-site.com", inserted_at: Timex.shift(now, days: -8))+ insert(:weekly_report, site: site, recipients: ["[email protected]"])populate_stats(site, [build(:pageview,diff --git a/test/workers/send_site_setup_emails_test.exs b/test/workers/send_site_setup_emails_test.exsindex 15f203f9d1..3634f06609 100644--- a/test/workers/send_site_setup_emails_test.exs+++ b/test/workers/send_site_setup_emails_test.exs@@ -40,6 +40,7 @@ defmodule Plausible.Workers.SendSiteSetupEmailsTest dodescribe "when user has managed to set up their site" dotest "sends the setup completed email as soon as possible" douser = insert(:user)+insert(:site, members: [user], domain: "test-site.com")perform_job(SendSiteSetupEmails, %{})