Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Conditionally support switching between v1 and v2 clickhouse schemas (#2780)
hq1d2f2c69387bd
diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.ymlindex 540b8a47c0..003689b0c0 100644--- a/.github/workflows/elixir.yml+++ b/.github/workflows/elixir.yml@@ -70,8 +70,12 @@ jobs:run: npm run deploy --prefix ./tracker- name: Check Credo Warningsrun: mix credo diff --from-git-merge-base origin/master- - name: Run tests+ - name: Run tests (against v1 schema)run: mix test --include slow --max-failures 1 --warnings-as-errors+ - name: Run tests (against v2 schema)+ run: mix test --include slow --max-failures 1 --warnings-as-errors+ env:+ V2_MIGRATION_DONE: 1- name: Check Dialyzerrun: mix dialyzerenv:diff --git a/config/runtime.exs b/config/runtime.exsindex 744b66e1cc..c937bd1c7d 100644--- a/config/runtime.exs+++ b/config/runtime.exs@@ -99,6 +99,8 @@ ch_db_url =|> get_var_from_path_or_env("CLICKHOUSE_MAX_BUFFER_SIZE", "10000")|> Integer.parse()+v2_migration_done = get_var_from_path_or_env(config_dir, "V2_MIGRATION_DONE")+### Mandatory params Endbuild_metadata_raw = get_var_from_path_or_env(config_dir, "BUILD_METADATA", "{}")@@ -131,6 +133,8 @@ runtime_metadata = [config :plausible, :runtime_metadata, runtime_metadata+config :plausible, :v2_migration_done, v2_migration_done+sentry_dsn = get_var_from_path_or_env(config_dir, "SENTRY_DSN")honeycomb_api_key = get_var_from_path_or_env(config_dir, "HONEYCOMB_API_KEY")honeycomb_dataset = get_var_from_path_or_env(config_dir, "HONEYCOMB_DATASET")diff --git a/lib/plausible.ex b/lib/plausible.exindex ebdaaaaedb..0dbc04bf0d 100644--- a/lib/plausible.ex+++ b/lib/plausible.ex@@ -6,4 +6,9 @@ defmodule Plausible doContexts are also responsible for managing your data, regardlessif it comes from the database, an external API or others."""++ @spec v2?() :: boolean()+ def v2?() do+ Plausible.DataMigration.NumericIDs.ready?()+ endenddiff --git a/lib/plausible/billing/billing.ex b/lib/plausible/billing/billing.exindex 2c09d8bce0..9967c8a87f 100644--- a/lib/plausible/billing/billing.ex+++ b/lib/plausible/billing/billing.ex@@ -113,17 +113,36 @@ defmodule Plausible.Billing dodef last_two_billing_months_usage(user, today \\ Timex.today()) do{first, second} = last_two_billing_cycles(user, today)- domains = Plausible.Sites.owned_sites_domains(user)- usage_for_sites = fn domains, date_range ->- {pageviews, custom_events} = Plausible.Stats.Clickhouse.usage_breakdown(domains, date_range)- pageviews + custom_events- end+ if Plausible.v2?() do+ site_ids = Plausible.Sites.owned_site_ids(user)- {- usage_for_sites.(domains, first),- usage_for_sites.(domains, second)- }+ usage_for_sites = fn site_ids, date_range ->+ {pageviews, custom_events} =+ Plausible.Stats.Clickhouse.usage_breakdown(site_ids, date_range)++ pageviews + custom_events+ end++ {+ usage_for_sites.(site_ids, first),+ usage_for_sites.(site_ids, second)+ }+ else+ domains = Plausible.Sites.owned_sites_domains(user)++ usage_for_sites = fn domains, date_range ->+ {pageviews, custom_events} =+ Plausible.Stats.Clickhouse.usage_breakdown(domains, date_range)++ pageviews + custom_events+ end++ {+ usage_for_sites.(domains, first),+ usage_for_sites.(domains, second)+ }+ endenddef last_two_billing_cycles(user, today \\ Timex.today()) do@@ -147,8 +166,13 @@ defmodule Plausible.Billing doenddef usage_breakdown(user) do- domains = Plausible.Sites.owned_sites_domains(user)- Plausible.Stats.Clickhouse.usage_breakdown(domains)+ if Plausible.v2?() do+ site_ids = Plausible.Sites.owned_site_ids(user)+ Plausible.Stats.Clickhouse.usage_breakdown(site_ids)+ else+ domains = Plausible.Sites.owned_sites_domains(user)+ Plausible.Stats.Clickhouse.usage_breakdown(domains)+ endend@doc """diff --git a/lib/plausible/data_migration.ex b/lib/plausible/data_migration.exindex 611826cef2..be38859286 100644--- a/lib/plausible/data_migration.ex+++ b/lib/plausible/data_migration.ex@@ -13,7 +13,7 @@ defmodule Plausible.DataMigration do@repo repodef run_sql_confirm(name, assigns \\ []) do- query = unwrap(name, assigns)+ query = unwrap_with_io(name, assigns)confirm("Execute?", fn -> do_run(name, query) end)end@@ -30,7 +30,9 @@ defmodule Plausible.DataMigration doenddefp unwrap(name, assigns) do- "priv/data_migrations"+ :plausible+ |> :code.priv_dir()+ |> Path.join("data_migrations")|> Path.join(@dir)|> Path.join("sql")|> Path.join(name <> ".sql.eex")diff --git a/lib/plausible/data_migration/numeric_ids.ex b/lib/plausible/data_migration/numeric_ids.exindex 8782df7a2c..4774f465b6 100644--- a/lib/plausible/data_migration/numeric_ids.ex+++ b/lib/plausible/data_migration/numeric_ids.ex@@ -7,6 +7,10 @@ defmodule Plausible.DataMigration.NumericIDs do@table_settings "SETTINGS index_granularity = 8192, storage_policy = 'tiered'"+ def ready?() do+ Application.get_env(:plausible, :v2_migration_done) || false+ end+# credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexitydef run(opts \\ []) dointeractive? = Keyword.get(opts, :interactive?, true)@@ -65,7 +69,7 @@ defmodule Plausible.DataMigration.NumericIDs do- table_settings: #{table_settings}- db url: #{db_url}- cluster?: #{cluster?}- - partitions to do: #{inspect(partitions, pretty: true, width: 80)}+ - partitions to do: #{inspect(partitions, pretty: true, limit: :infinity, width: 80)}- start from: #{start_from}- stop at: #{stop_at}""")diff --git a/lib/plausible/event/write_buffer.ex b/lib/plausible/event/write_buffer.exindex ccb3fcb25f..f6af74b17e 100644--- a/lib/plausible/event/write_buffer.ex+++ b/lib/plausible/event/write_buffer.ex@@ -64,7 +64,12 @@ defmodule Plausible.Event.WriteBuffer doevents ->Logger.info("Flushing #{length(events)} events")events = Enum.map(events, &(Map.from_struct(&1) |> Map.delete(:__meta__)))- IngestRepo.insert_all(Plausible.ClickhouseEvent, events)++ if Plausible.v2?() do+ IngestRepo.insert_all(Plausible.ClickhouseEventV2, events)+ else+ IngestRepo.insert_all(Plausible.ClickhouseEvent, events)+ endendenddiff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.exindex 72b3253b00..00a475892b 100644--- a/lib/plausible/ingestion/event.ex+++ b/lib/plausible/ingestion/event.ex@@ -7,9 +7,11 @@ defmodule Plausible.Ingestion.Event do"""alias Plausible.Ingestion.Requestalias Plausible.ClickhouseEvent+ alias Plausible.ClickhouseEventV2alias Plausible.Site.GateKeeperdefstruct domain: nil,+ site_id: nil,clickhouse_event_attrs: %{},clickhouse_event: nil,dropped?: false,@@ -26,8 +28,9 @@ defmodule Plausible.Ingestion.Event do@type t() :: %__MODULE__{domain: String.t() | nil,+ site_id: pos_integer() | nil,clickhouse_event_attrs: map(),- clickhouse_event: %ClickhouseEvent{} | nil,+ clickhouse_event: %ClickhouseEvent{} | %ClickhouseEventV2{} | nil,dropped?: boolean(),drop_reason: drop_reason(),request: Request.t(),@@ -43,10 +46,10 @@ defmodule Plausible.Ingestion.Event doelseEnum.reduce(domains, [], fn domain, acc ->case GateKeeper.check(domain) do- :allow ->+ {:allow, site_id} ->processed =domain- |> new(request)+ |> new(site_id, request)|> process_unless_dropped(pipeline())[processed | acc]@@ -117,6 +120,10 @@ defmodule Plausible.Ingestion.Event dostruct!(__MODULE__, domain: domain, request: request)end+ defp new(domain, site_id, request) do+ struct!(__MODULE__, domain: domain, site_id: site_id, request: request)+ end+defp drop(%__MODULE__{} = event, reason, attrs \\ []) dofields =attrs@@ -156,6 +163,7 @@ defmodule Plausible.Ingestion.Event dodefp put_basic_info(%__MODULE__{} = event) doupdate_attrs(event, %{domain: event.domain,+ site_id: event.site_id,timestamp: event.request.timestamp,name: event.request.event_name,hostname: event.request.hostname,@@ -217,9 +225,15 @@ defmodule Plausible.Ingestion.Event dodefp validate_clickhouse_event(%__MODULE__{} = event) doclickhouse_event =- event- |> Map.fetch!(:clickhouse_event_attrs)- |> ClickhouseEvent.new()+ if Plausible.v2?() do+ event+ |> Map.fetch!(:clickhouse_event_attrs)+ |> ClickhouseEventV2.new()+ else+ event+ |> Map.fetch!(:clickhouse_event_attrs)+ |> ClickhouseEvent.new()+ endcase Ecto.Changeset.apply_action(clickhouse_event, nil) do{:ok, valid_clickhouse_event} ->diff --git a/lib/plausible/session/cache_store.ex b/lib/plausible/session/cache_store.exindex b48e1bc229..3563673ed3 100644--- a/lib/plausible/session/cache_store.ex+++ b/lib/plausible/session/cache_store.ex@@ -22,7 +22,14 @@ defmodule Plausible.Session.CacheStore dodefp find_session(_domain, nil), do: nildefp find_session(event, user_id) do- case Cachex.get(:sessions, {event.domain, user_id}) do+ from_cache =+ if Plausible.v2?() do+ Cachex.get(:sessions, {event.site_id, user_id})+ else+ Cachex.get(:sessions, {event.domain, user_id})+ end++ case from_cache do{:ok, nil} ->nil@@ -38,7 +45,13 @@ defmodule Plausible.Session.CacheStore doenddefp persist_session(session) do- key = {session.domain, session.user_id}+ key =+ if Plausible.v2?() do+ {session.site_id, session.user_id}+ else+ {session.domain, session.user_id}+ end+Cachex.put(:sessions, key, session, ttl: :timer.minutes(30))sessionend@@ -68,38 +81,74 @@ defmodule Plausible.Session.CacheStore doenddefp new_session_from_event(event) do- %Plausible.ClickhouseSession{- sign: 1,- session_id: Plausible.ClickhouseSession.random_uint64(),- hostname: event.hostname,- domain: event.domain,- user_id: event.user_id,- entry_page: event.pathname,- exit_page: event.pathname,- is_bounce: true,- duration: 0,- pageviews: if(event.name == "pageview", do: 1, else: 0),- events: 1,- referrer: event.referrer,- referrer_source: event.referrer_source,- utm_medium: event.utm_medium,- utm_source: event.utm_source,- utm_campaign: event.utm_campaign,- utm_content: event.utm_content,- utm_term: event.utm_term,- country_code: event.country_code,- subdivision1_code: event.subdivision1_code,- subdivision2_code: event.subdivision2_code,- city_geoname_id: event.city_geoname_id,- screen_size: event.screen_size,- operating_system: event.operating_system,- operating_system_version: event.operating_system_version,- browser: event.browser,- browser_version: event.browser_version,- timestamp: event.timestamp,- start: event.timestamp,- "entry_meta.key": Map.get(event, :"meta.key"),- "entry_meta.value": Map.get(event, :"meta.value")- }+ if Plausible.v2?() do+ %Plausible.ClickhouseSessionV2{+ sign: 1,+ session_id: Plausible.ClickhouseSession.random_uint64(),+ hostname: event.hostname,+ site_id: event.site_id,+ user_id: event.user_id,+ entry_page: event.pathname,+ exit_page: event.pathname,+ is_bounce: true,+ duration: 0,+ pageviews: if(event.name == "pageview", do: 1, else: 0),+ events: 1,+ referrer: event.referrer,+ referrer_source: event.referrer_source,+ utm_medium: event.utm_medium,+ utm_source: event.utm_source,+ utm_campaign: event.utm_campaign,+ utm_content: event.utm_content,+ utm_term: event.utm_term,+ country_code: event.country_code,+ subdivision1_code: event.subdivision1_code,+ subdivision2_code: event.subdivision2_code,+ city_geoname_id: event.city_geoname_id,+ screen_size: event.screen_size,+ operating_system: event.operating_system,+ operating_system_version: event.operating_system_version,+ browser: event.browser,+ browser_version: event.browser_version,+ timestamp: event.timestamp,+ start: event.timestamp,+ "entry_meta.key": Map.get(event, :"meta.key"),+ "entry_meta.value": Map.get(event, :"meta.value")+ }+ else+ %Plausible.ClickhouseSession{+ sign: 1,+ session_id: Plausible.ClickhouseSession.random_uint64(),+ hostname: event.hostname,+ domain: event.domain,+ user_id: event.user_id,+ entry_page: event.pathname,+ exit_page: event.pathname,+ is_bounce: true,+ duration: 0,+ pageviews: if(event.name == "pageview", do: 1, else: 0),+ events: 1,+ referrer: event.referrer,+ referrer_source: event.referrer_source,+ utm_medium: event.utm_medium,+ utm_source: event.utm_source,+ utm_campaign: event.utm_campaign,+ utm_content: event.utm_content,+ utm_term: event.utm_term,+ country_code: event.country_code,+ subdivision1_code: event.subdivision1_code,+ subdivision2_code: event.subdivision2_code,+ city_geoname_id: event.city_geoname_id,+ screen_size: event.screen_size,+ operating_system: event.operating_system,+ operating_system_version: event.operating_system_version,+ browser: event.browser,+ browser_version: event.browser_version,+ timestamp: event.timestamp,+ start: event.timestamp,+ "entry_meta.key": Map.get(event, :"meta.key"),+ "entry_meta.value": Map.get(event, :"meta.value")+ }+ endendenddiff --git a/lib/plausible/session/write_buffer.ex b/lib/plausible/session/write_buffer.exindex de23a6802c..598dc83933 100644--- a/lib/plausible/session/write_buffer.ex+++ b/lib/plausible/session/write_buffer.ex@@ -69,7 +69,11 @@ defmodule Plausible.Session.WriteBuffer do|> Enum.map(&(Map.from_struct(&1) |> Map.delete(:__meta__)))|> Enum.reverse()- IngestRepo.insert_all(Plausible.ClickhouseSession, sessions)+ if Plausible.v2?() do+ IngestRepo.insert_all(Plausible.ClickhouseSessionV2, sessions)+ else+ IngestRepo.insert_all(Plausible.ClickhouseSession, sessions)+ endendenddiff --git a/lib/plausible/site/admin.ex b/lib/plausible/site/admin.exindex c2f29053e0..032f196624 100644--- a/lib/plausible/site/admin.ex+++ b/lib/plausible/site/admin.ex@@ -84,26 +84,32 @@ defmodule Plausible.SiteAdmin doenddef transfer_data([from_site], params) do- to_site = Repo.get_by(Plausible.Site, domain: params["domain"])+ if Plausible.v2?() do+ {:error, "Transfers temporarily unspported with v2"}+ else+ to_site = Repo.get_by(Plausible.Site, domain: params["domain"])- if to_site do- event_q = event_transfer_query(from_site.domain, to_site.domain)- {:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, event_q, [], timeout: 30_000)+ if to_site do+ event_q = event_transfer_query(from_site.domain, to_site.domain)+ {:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, event_q, [], timeout: 30_000)- session_q = session_transfer_query(from_site.domain, to_site.domain)- {:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, session_q, [], timeout: 30_000)+ session_q = session_transfer_query(from_site.domain, to_site.domain)- start_date = Plausible.Stats.Clickhouse.pageview_start_date_local(from_site)+ {:ok, _} =+ Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, session_q, [], timeout: 30_000)- {:ok, _} =- to_site- |> Plausible.Site.set_stats_start_date(start_date)- |> Plausible.Site.set_native_stats_start_at(from_site.native_stats_start_at)- |> Repo.update()+ start_date = Plausible.Stats.Clickhouse.pageview_start_date_local(from_site)- :ok- else- {:error, "Cannot transfer to non-existing domain"}+ {:ok, _} =+ to_site+ |> Plausible.Site.set_stats_start_date(start_date)+ |> Plausible.Site.set_native_stats_start_at(from_site.native_stats_start_at)+ |> Repo.update()++ :ok+ else+ {:error, "Cannot transfer to non-existing domain"}+ endendenddiff --git a/lib/plausible/site/gate_keeper.ex b/lib/plausible/site/gate_keeper.exindex d0e4fe6efb..b7dcc65d1f 100644--- a/lib/plausible/site/gate_keeper.ex+++ b/lib/plausible/site/gate_keeper.ex@@ -1,9 +1,9 @@defmodule Plausible.Site.GateKeeper do@type policy() :: :allow | :not_found | :block | :throttle+ @type site_id() :: pos_integer()@policy_for_non_existing_sites :not_found- @policy_on_rate_limiting_backend_error :allow- @type t() :: :allow | {:deny, policy()}+ @type t() :: {:allow, site_id()} | {:deny, policy()}@moduledoc """Thin wrapper around Hammer for gate keeping domain-specific events@@ -23,7 +23,7 @@ defmodule Plausible.Site.GateKeeper doThe module defines two policies outside the regular bucket inspection:* when the the site is not found in cache: #{@policy_for_non_existing_sites}* when the underlying rate limiting mechanism returns- an internal error: #{@policy_on_rate_limiting_backend_error}+ an internal error: :allow"""alias Plausible.Sitealias Plausible.Site.Cache@@ -33,7 +33,7 @@ defmodule Plausible.Site.GateKeeper do@spec check(String.t(), Keyword.t()) :: t()def check(domain, opts \\ []) when is_binary(domain) docase policy(domain, opts) do- :allow -> :allow+ {:allow, site_id} -> {:allow, site_id}other -> {:deny, other}endend@@ -56,15 +56,15 @@ defmodule Plausible.Site.GateKeeper doresultend- defp check_rate_limit(%Site{ingest_rate_limit_threshold: nil}, _opts) do- :allow+ defp check_rate_limit(%Site{id: site_id, ingest_rate_limit_threshold: nil}, _opts) do+ {:allow, site_id}enddefp check_rate_limit(%Site{ingest_rate_limit_threshold: 0}, _opts) do:blockend- defp check_rate_limit(%Site{ingest_rate_limit_threshold: threshold} = site, opts)+ defp check_rate_limit(%Site{id: site_id, ingest_rate_limit_threshold: threshold} = site, opts)when is_integer(threshold) dokey = Keyword.get(opts, :key, key(site.domain))scale_ms = site.ingest_rate_limit_scale_seconds * 1_000@@ -74,14 +74,14 @@ defmodule Plausible.Site.GateKeeper do:throttle{:allow, _} ->- :allow+ {:allow, site_id}{:error, reason} ->Logger.error(- "Error checking rate limit for '#{key}': #{inspect(reason)}. Falling back to: #{@policy_on_rate_limiting_backend_error}"+ "Error checking rate limit for '#{key}': #{inspect(reason)}. Falling back to: :allow")- @policy_on_rate_limiting_backend_error+ {:allow, site_id}endendenddiff --git a/lib/plausible/sites.ex b/lib/plausible/sites.exindex e791986089..d936f7b747 100644--- a/lib/plausible/sites.ex+++ b/lib/plausible/sites.ex@@ -1,6 +1,5 @@defmodule Plausible.Sites douse Plausible.Repo- alias Plausible.ClickhouseRepoalias Plausible.Sitealias Plausible.Site.SharedLinkimport Ecto.Query@@ -77,11 +76,6 @@ defmodule Plausible.Sites do!!stats_start_date(site)end- def has_events?(domain) do- q = from e in "events", where: e.domain == ^domain, select: true, limit: 1- ClickhouseRepo.one(q) == true- end-def create_shared_link(site, name, password \\ nil) dochanges =SharedLink.changeset(@@ -158,6 +152,13 @@ defmodule Plausible.Sites do|> Repo.all()end+ def owned_site_ids(user) do+ user+ |> owned_sites_query()+ |> select([site], site.id)+ |> Repo.all()+ end+defp owned_sites_query(user) dofrom(s in Site,join: sm in Site.Membership,diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.exindex 225fc30294..7920ac8e7d 100644--- a/lib/plausible/stats/base.ex+++ b/lib/plausible/stats/base.ex@@ -35,11 +35,19 @@ defmodule Plausible.Stats.Base do{first_datetime, last_datetime} = utc_boundaries(query, site)q =- from(- e in "events",- where: e.domain == ^site.domain,- where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime- )+ if Plausible.v2?() do+ from(+ e in "events_v2",+ where: e.site_id == ^site.id,+ where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime+ )+ else+ from(+ e in "events",+ where: e.domain == ^site.domain,+ where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime+ )+ end|> add_sample_hint(query)q =@@ -190,11 +198,19 @@ defmodule Plausible.Stats.Base do{first_datetime, last_datetime} = utc_boundaries(query, site)sessions_q =- from(- s in "sessions",- where: s.domain == ^site.domain,- where: s.start >= ^first_datetime and s.start < ^last_datetime- )+ if Plausible.v2?() do+ from(+ s in "sessions_v2",+ where: s.site_id == ^site.id,+ where: s.start >= ^first_datetime and s.start < ^last_datetime+ )+ else+ from(+ s in "sessions",+ where: s.domain == ^site.domain,+ where: s.start >= ^first_datetime and s.start < ^last_datetime+ )+ end|> add_sample_hint(query)|> filter_by_entry_props(query)diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex 66f0ea07b8..f06b44e9cd 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -312,7 +312,7 @@ defmodule Plausible.Stats.Breakdown doenddefp do_group_by(- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events", _}}} = q,+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,"event:props:" <> prop) doq =@@ -336,7 +336,7 @@ defmodule Plausible.Stats.Breakdown doenddefp do_group_by(- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events", _}}} = q,+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,"event:name") dofrom(@@ -348,7 +348,7 @@ defmodule Plausible.Stats.Breakdown doenddefp do_group_by(- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events", _}}} = q,+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,"event:page") dofrom(@@ -360,7 +360,7 @@ defmodule Plausible.Stats.Breakdown doenddefp do_group_by(- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events", _}}} = q,+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,"event:page_match") docase Map.get(q, :__private_match_sources__) dodiff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.exindex a23b48d65f..5ce0ed4360 100644--- a/lib/plausible/stats/clickhouse.ex+++ b/lib/plausible/stats/clickhouse.ex@@ -8,12 +8,23 @@ defmodule Plausible.Stats.Clickhouse do@spec pageview_start_date_local(Plausible.Site.t()) :: Date.t() | nildef pageview_start_date_local(site) dodatetime =- ClickhouseRepo.one(- from e in "events",- select: fragment("min(?)", e.timestamp),- where: e.domain == ^site.domain,- where: e.timestamp >= ^site.native_stats_start_at- )+ if Plausible.v2?() do+ ClickhouseRepo.one(+ from(e in "events_v2",+ select: fragment("min(?)", e.timestamp),+ where: e.site_id == ^site.id,+ where: e.timestamp >= ^site.native_stats_start_at+ )+ )+ else+ ClickhouseRepo.one(+ from(e in "events",+ select: fragment("min(?)", e.timestamp),+ where: e.domain == ^site.domain,+ where: e.timestamp >= ^site.native_stats_start_at+ )+ )+ endcase datetime do# no stats for this domain yet@@ -29,28 +40,29 @@ defmodule Plausible.Stats.Clickhouse dodef imported_pageview_count(site) doPlausible.ClickhouseRepo.one(- from i in "imported_visitors",+ from(i in "imported_visitors",where: i.site_id == ^site.id,select: sum(i.pageviews)+ ))end- def usage_breakdown(domains) do+ def usage_breakdown(domains_or_site_ids) dorange =Date.range(Timex.shift(Timex.today(), days: -30),Timex.today())- usage_breakdown(domains, range)+ usage_breakdown(domains_or_site_ids, range)end- def usage_breakdown(domains, date_range) do+ def usage_breakdown([d | _] = domains, date_range) when is_binary(d) doEnum.chunk_every(domains, 300)|> Enum.reduce({0, 0}, fn domains, {pageviews_total, custom_events_total} ->{chunk_pageviews, chunk_custom_events} =ClickhouseRepo.one(- from e in "events",+ from(e in "events",where: e.domain in ^domains,where: fragment("toDate(?)", e.timestamp) >= ^date_range.first,where: fragment("toDate(?)", e.timestamp) <= ^date_range.last,@@ -58,12 +70,35 @@ defmodule Plausible.Stats.Clickhouse dofragment("countIf(? = 'pageview')", e.name),fragment("countIf(? != 'pageview')", e.name)}+ )){pageviews_total + chunk_pageviews, custom_events_total + chunk_custom_events}end)end+ def usage_breakdown([sid | _] = site_ids, date_range) when is_integer(sid) do+ Enum.chunk_every(site_ids, 300)+ |> Enum.reduce({0, 0}, fn site_ids, {pageviews_total, custom_events_total} ->+ {chunk_pageviews, chunk_custom_events} =+ ClickhouseRepo.one(+ from(e in "events_v2",+ where: e.site_id in ^site_ids,+ where: fragment("toDate(?)", e.timestamp) >= ^date_range.first,+ where: fragment("toDate(?)", e.timestamp) <= ^date_range.last,+ select: {+ fragment("countIf(? = 'pageview')", e.name),+ fragment("countIf(? != 'pageview')", e.name)+ }+ )+ )++ {pageviews_total + chunk_pageviews, custom_events_total + chunk_custom_events}+ end)+ end++ def usage_breakdown([], _date_range), do: {0, 0}+def top_sources(site, query, limit, page, show_noref \\ false, include_details) dooffset = (page - 1) * limit@@ -151,47 +186,87 @@ defmodule Plausible.Stats.Clickhouse dodef current_visitors(site, query) doPlausible.ClickhouseRepo.one(- from e in base_query(site, query),+ from(e in base_query(site, query),select: uniq(e.user_id)+ ))enddef has_pageviews?(site) do- ClickhouseRepo.exists?(- from(e in "events",- where:- e.domain == ^site.domain and- e.name == "pageview" and- e.timestamp >=- ^site.native_stats_start_at+ if Plausible.v2?() do+ ClickhouseRepo.exists?(+ from(e in "events_v2",+ where:+ e.site_id == ^site.id and+ e.name == "pageview" and+ e.timestamp >=+ ^site.native_stats_start_at+ ))- )+ else+ ClickhouseRepo.exists?(+ from(e in "events",+ where:+ e.domain == ^site.domain and+ e.name == "pageview" and+ e.timestamp >=+ ^site.native_stats_start_at+ )+ )+ endenddef last_24h_visitors([]), do: %{}def last_24h_visitors(sites) do- domains = Enum.map(sites, & &1.domain)-- ClickhouseRepo.all(- from e in "events",- group_by: e.domain,- where: e.domain in ^domains,- where: e.timestamp > fragment("now() - INTERVAL 24 HOUR"),- select: {e.domain, fragment("uniq(user_id)")}- )- |> Enum.into(%{})+ if Plausible.v2?() do+ site_id_to_domain_mapping = for site <- sites, do: {site.id, site.domain}, into: %{}++ ClickhouseRepo.all(+ from(e in "events_v2",+ group_by: e.site_id,+ where: e.site_id in ^Map.keys(site_id_to_domain_mapping),+ where: e.timestamp > fragment("now() - INTERVAL 24 HOUR"),+ select: {e.site_id, fragment("uniq(user_id)")}+ )+ )+ |> Enum.map(fn {site_id, user_id} ->+ {site_id_to_domain_mapping[site_id], user_id}+ end)+ |> Enum.into(%{})+ else+ domains = Enum.map(sites, & &1.domain)++ ClickhouseRepo.all(+ from(e in "events",+ group_by: e.domain,+ where: e.domain in ^domains,+ where: e.timestamp > fragment("now() - INTERVAL 24 HOUR"),+ select: {e.domain, fragment("uniq(user_id)")}+ )+ )+ |> Enum.into(%{})+ endend+ # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexitydefp base_session_query(site, query) do{first_datetime, last_datetime} = utc_boundaries(query, site)q =- from(s in "sessions",- hints: ["SAMPLE 10000000"],- where: s.domain == ^site.domain,- where: s.timestamp >= ^first_datetime and s.start < ^last_datetime- )+ if Plausible.v2?() do+ from(s in "sessions_v2",+ hints: ["SAMPLE 10000000"],+ where: s.site_id == ^site.id,+ where: s.timestamp >= ^first_datetime and s.start < ^last_datetime+ )+ else+ from(s in "sessions",+ hints: ["SAMPLE 10000000"],+ where: s.domain == ^site.domain,+ where: s.timestamp >= ^first_datetime and s.start < ^last_datetime+ )+ endq =if query.filters["source"] do@@ -302,15 +377,24 @@ defmodule Plausible.Stats.Clickhouse doendend+ # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexitydefp base_query_bare(site, query) do{first_datetime, last_datetime} = utc_boundaries(query, site)q =- from(e in "events",- hints: ["SAMPLE 10000000"],- where: e.domain == ^site.domain,- where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime- )+ if Plausible.v2?() do+ from(e in "events_v2",+ hints: ["SAMPLE 10000000"],+ where: e.site_id == ^site.id,+ where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime+ )+ else+ from(e in "events",+ hints: ["SAMPLE 10000000"],+ where: e.domain == ^site.domain,+ where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime+ )+ endq =if query.filters["screen"] dodiff --git a/lib/plausible/stats/current_visitors.ex b/lib/plausible/stats/current_visitors.exindex c254df889e..ef36b47956 100644--- a/lib/plausible/stats/current_visitors.ex+++ b/lib/plausible/stats/current_visitors.ex@@ -8,11 +8,20 @@ defmodule Plausible.Stats.CurrentVisitors do|> Timex.shift(minutes: -5)|> NaiveDateTime.truncate(:second)- ClickhouseRepo.one(- from e in "events",- where: e.domain == ^site.domain,- where: e.timestamp >= ^first_datetime,- select: uniq(e.user_id)- )+ if Plausible.v2?() do+ ClickhouseRepo.one(+ from e in "events_v2",+ where: e.site_id == ^site.id,+ where: e.timestamp >= ^first_datetime,+ select: uniq(e.user_id)+ )+ else+ ClickhouseRepo.one(+ from e in "events",+ where: e.domain == ^site.domain,+ where: e.timestamp >= ^first_datetime,+ select: uniq(e.user_id)+ )+ endendenddiff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 9a1172d930..386df3b8a9 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -1161,9 +1161,14 @@ defmodule PlausibleWeb.Api.StatsController dosite = conn.assigns[:site]with :ok <- validate_params(params) do- query = Query.from(site, params) |> Filters.add_prefix()-- json(conn, Stats.filter_suggestions(site, query, params["filter_name"], params["q"]))+ query =+ Query.from(site, params)+ |> Filters.add_prefix()++ json(+ conn,+ Stats.filter_suggestions(site, query, params["filter_name"], params["q"])+ )else{:error, message} when is_binary(message) -> bad_request(conn, message)enddiff --git a/mix.lock b/mix.lockindex ce9f06c677..4f1024c679 100644--- a/mix.lock+++ b/mix.lock@@ -10,7 +10,7 @@"cachex": {:hex, :cachex, "3.4.0", "868b2959ea4aeb328c6b60ff66c8d5123c083466ad3c33d3d8b5f142e13101fb", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "370123b1ab4fba4d2965fb18f87fd758325709787c8c5fce35b3fe80645ccbe5"},"castore": {:hex, :castore, "0.1.22", "4127549e411bedd012ca3a308dede574f43819fe9394254ca55ab4895abfa1a2", [:mix], [], "hexpm", "c17576df47eb5aa1ee40cc4134316a99f5cad3e215d5c77b8dd3cfef12a22cac"},"certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"},- "ch": {:git, "https://github.com/ruslandoga/ch.git", "5894f81e246c049b9db5186497189cce226947aa", []},+ "ch": {:git, "https://github.com/plausible/ch.git", "f411daa07c6310d3308f81397905df65330aeb64", []},"chatterbox": {:hex, :ts_chatterbox, "0.13.0", "6f059d97bcaa758b8ea6fffe2b3b81362bd06b639d3ea2bb088335511d691ebf", [:rebar3], [{:hpack, "~> 0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "b93d19104d86af0b3f2566c4cba2a57d2e06d103728246ba1ac6c3c0ff010aa7"},"chto": {:git, "https://github.com/plausible/chto.git", "316421ede947541bac32a853d70b0ca8ba003220", []},"combination": {:hex, :combination, "0.0.3", "746aedca63d833293ec6e835aa1f34974868829b1486b1e1cb0685f0b2ae1f41", [:mix], [], "hexpm", "72b099f463df42ef7dc6371d250c7070b57b6c5902853f69deb894f79eda18ca"},diff --git a/priv/data_migrations/NumericIDs/sql/list-partitions.sql.eex b/priv/data_migrations/NumericIDs/sql/list-partitions.sql.eexindex c945e73764..9522395a30 100644--- a/priv/data_migrations/NumericIDs/sql/list-partitions.sql.eex+++ b/priv/data_migrations/NumericIDs/sql/list-partitions.sql.eex@@ -1,4 +1,4 @@-SELECT partition+SELECT distinct(partition)FROM system.partsWHERE table = 'events'<%= if @start_from do %>AND partition >= '<%= @start_from %>'<% end %>diff --git a/test/plausible/auth/auth_test.exs b/test/plausible/auth/auth_test.exsindex ab9058bf85..98c79a146f 100644--- a/test/plausible/auth/auth_test.exs+++ b/test/plausible/auth/auth_test.exs@@ -18,7 +18,11 @@ defmodule Plausible.AuthTest dotest "is true if user does have events" douser = insert(:user)- insert(:site, members: [user], domain: "test-site.com")+ site = insert(:site, members: [user])++ populate_stats(site, [+ build(:pageview)+ ])assert Auth.has_active_sites?(user)enddiff --git a/test/plausible/billing/billing_test.exs b/test/plausible/billing/billing_test.exsindex 9ca1111f1d..c44567768b 100644--- a/test/plausible/billing/billing_test.exs+++ b/test/plausible/billing/billing_test.exs@@ -133,10 +133,10 @@ defmodule Plausible.BillingTest dosite = insert(:site, members: [user])create_pageviews([- %{domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]},- %{domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]},- %{domain: site.domain, timestamp: ~N[2020-11-01 00:00:00]},- %{domain: site.domain, timestamp: ~N[2020-10-31 00:00:00]}+ %{site: site, timestamp: ~N[2021-01-01 00:00:00]},+ %{site: site, timestamp: ~N[2020-12-31 00:00:00]},+ %{site: site, timestamp: ~N[2020-11-01 00:00:00]},+ %{site: site, timestamp: ~N[2020-10-31 00:00:00]}])assert Billing.last_two_billing_months_usage(user, today) == {1, 1}@@ -163,10 +163,10 @@ defmodule Plausible.BillingTest do)create_pageviews([- %{domain: owner_site.domain, timestamp: ~N[2020-12-31 00:00:00]},- %{domain: admin_site.domain, timestamp: ~N[2020-12-31 00:00:00]},- %{domain: owner_site.domain, timestamp: ~N[2020-11-01 00:00:00]},- %{domain: admin_site.domain, timestamp: ~N[2020-11-01 00:00:00]}+ %{site: owner_site, timestamp: ~N[2020-12-31 00:00:00]},+ %{site: admin_site, timestamp: ~N[2020-12-31 00:00:00]},+ %{site: owner_site, timestamp: ~N[2020-11-01 00:00:00]},+ %{site: admin_site, timestamp: ~N[2020-11-01 00:00:00]}])assert Billing.last_two_billing_months_usage(user, today) == {1, 1}diff --git a/test/plausible/session/cache_store_test.exs b/test/plausible/session/cache_store_test.exsindex 6f14c52064..265d17df48 100644--- a/test/plausible/session/cache_store_test.exs+++ b/test/plausible/session/cache_store_test.exs@@ -37,7 +37,13 @@ defmodule Plausible.Session.CacheStoreTest doassert_receive({WriteBuffer, :insert, [sessions]})assert [session] = sessionsassert session.hostname == event.hostname- assert session.domain == event.domain++ if Plausible.v2?() do+ assert session.site_id == event.site_id+ else+ assert session.domain == event.domain+ end+assert session.user_id == event.user_idassert session.entry_page == event.pathnameassert session.exit_page == event.pathname@@ -68,12 +74,9 @@ defmodule Plausible.Session.CacheStoreTest dotimestamp = Timex.now()event1 = build(:event, name: "pageview", timestamp: timestamp |> Timex.shift(seconds: -10))- event2 =- build(:event,- domain: event1.domain,- user_id: event1.user_id,- name: "pageview",- timestamp: timestamp,+ event2 = %{+ event1+ | timestamp: timestamp,country_code: "US",subdivision1_code: "SUB1",subdivision2_code: "SUB2",@@ -83,7 +86,7 @@ defmodule Plausible.Session.CacheStoreTest dooperating_system_version: "11",browser: "Firefox",browser_version: "10"- )+ }CacheStore.on_event(event1, nil, buffer)CacheStore.on_event(event2, nil, buffer)@@ -107,13 +110,7 @@ defmodule Plausible.Session.CacheStoreTest dotimestamp = Timex.now()event1 = build(:event, name: "pageview", timestamp: timestamp |> Timex.shift(seconds: 10))- event2 =- build(:event,- domain: event1.domain,- user_id: event1.user_id,- name: "pageview",- timestamp: timestamp- )+ event2 = %{event1 | timestamp: timestamp}CacheStore.on_event(event1, nil, buffer)CacheStore.on_event(event2, nil, buffer)diff --git a/test/plausible/site/admin_test.exs b/test/plausible/site/admin_test.exsindex 9a34906890..99d5429c67 100644--- a/test/plausible/site/admin_test.exs+++ b/test/plausible/site/admin_test.exs@@ -1,76 +1,98 @@defmodule Plausible.SiteAdminTest douse Plausible.DataCase, async: true- alias Plausible.{SiteAdmin, ClickhouseRepo, ClickhouseEvent, ClickhouseSession}+ alias Plausible.{+ SiteAdmin,+ ClickhouseRepo,+ ClickhouseEvent,+ ClickhouseSession+ }describe "transfer_data" dotest "event and session structs remain the same after transfer" do- from_site = insert(:site)- to_site = insert(:site)+ if Plausible.v2?() do+ :ok+ else+ from_site = insert(:site)+ to_site = insert(:site)- populate_stats(from_site, [build(:pageview)])+ populate_stats(from_site, [build(:pageview)])- event_before = get_event_by_domain(from_site.domain)- session_before = get_session_by_domain(from_site.domain)+ event_before = get_event_by_domain(from_site.domain)+ session_before = get_session_by_domain(from_site.domain)- SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})+ SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})- event_after = get_event_by_domain(to_site.domain)- session_after = get_session_by_domain(to_site.domain)+ event_after = get_event_by_domain(to_site.domain)+ session_after = get_session_by_domain(to_site.domain)- assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}- assert session_before == %ClickhouseSession{session_after | transferred_from: ""}- assert event_after.transferred_from == from_site.domain- assert session_after.transferred_from == from_site.domain+ assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}+ assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}+ assert session_before == %ClickhouseSession{session_after | transferred_from: ""}+ assert event_after.transferred_from == from_site.domain+ assert session_after.transferred_from == from_site.domain+ endendtest "transfers all events and sessions" do- from_site = insert(:site)- to_site = insert(:site)-- populate_stats(from_site, [- build(:pageview, user_id: 123),- build(:event, name: "Signup", user_id: 123),- build(:pageview, user_id: 456),- build(:event, name: "Signup", user_id: 789)- ])-- SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})-- transferred_events =- ClickhouseRepo.all(- from e in Plausible.ClickhouseEvent, where: e.domain == ^to_site.domain- )-- transferred_sessions =- ClickhouseRepo.all(- from e in Plausible.ClickhouseSession, where: e.domain == ^to_site.domain- )-- assert length(transferred_events) == 4- assert length(transferred_sessions) == 3+ if Plausible.v2?() do+ :ok+ else+ from_site = insert(:site)+ to_site = insert(:site)++ populate_stats(from_site, [+ build(:pageview, user_id: 123),+ build(:event, name: "Signup", user_id: 123),+ build(:pageview, user_id: 456),+ build(:event, name: "Signup", user_id: 789)+ ])++ SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})++ transferred_events =+ ClickhouseRepo.all(+ from e in Plausible.ClickhouseEvent, where: e.domain == ^to_site.domain+ )++ transferred_sessions =+ ClickhouseRepo.all(+ from e in Plausible.ClickhouseSession, where: e.domain == ^to_site.domain+ )++ assert length(transferred_events) == 4+ assert length(transferred_sessions) == 3+ endendtest "updates stats_start_date on site record" do- from_site = insert(:site)- to_site = insert(:site)+ if Plausible.v2?() do+ :ok+ else+ from_site = insert(:site)+ to_site = insert(:site)- populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])])+ populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])])- SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})+ SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})- assert Repo.reload(to_site).stats_start_date == ~D[2022-01-01]+ assert Repo.reload(to_site).stats_start_date == ~D[2022-01-01]+ endendtest "updates native_stats_start_date on based on the from_site record" do- from_site = insert(:site, native_stats_start_at: ~N[2022-01-01 01:00:00])- to_site = insert(:site)+ if Plausible.v2?() do+ :ok+ else+ from_site = insert(:site, native_stats_start_at: ~N[2022-01-01 01:00:00])+ to_site = insert(:site)- populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])])+ populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])])- SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})+ SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})- assert Repo.reload(to_site).native_stats_start_at == ~N[2022-01-01 01:00:00]+ assert Repo.reload(to_site).native_stats_start_at == ~N[2022-01-01 01:00:00]+ endendtest "session_transfer_query" dodiff --git a/test/plausible/site/gate_keeper_test.exs b/test/plausible/site/gate_keeper_test.exsindex e67b229bf5..44a1233a37 100644--- a/test/plausible/site/gate_keeper_test.exs+++ b/test/plausible/site/gate_keeper_test.exs@@ -19,20 +19,21 @@ defmodule Plausible.Site.GateKeeperTest dotest "site from cache with no ingest_rate_limit_threshold is allowed", %{test: test, opts: opts} dodomain = "site1.example.com"- add_site_and_refresh_cache(test, domain: domain)- assert :allow = GateKeeper.check(domain, opts)+ %{id: site_id} = add_site_and_refresh_cache(test, domain: domain)+ assert {:allow, ^site_id} = GateKeeper.check(domain, opts)endtest "rate limiting works with threshold", %{test: test, opts: opts} dodomain = "site1.example.com"- add_site_and_refresh_cache(test,- domain: domain,- ingest_rate_limit_threshold: 1,- ingest_rate_limit_scale_seconds: 60- )+ %{id: site_id} =+ add_site_and_refresh_cache(test,+ domain: domain,+ ingest_rate_limit_threshold: 1,+ ingest_rate_limit_scale_seconds: 60+ )- assert :allow = GateKeeper.check(domain, opts)+ assert {:allow, ^site_id} = GateKeeper.check(domain, opts)assert {:deny, :throttle} = GateKeeper.check(domain, opts)assert {:deny, :throttle} = GateKeeper.check(domain, opts)end@@ -40,17 +41,18 @@ defmodule Plausible.Site.GateKeeperTest dotest "rate limiting works with scale window", %{test: test, opts: opts} dodomain = "site1.example.com"- add_site_and_refresh_cache(test,- domain: domain,- ingest_rate_limit_threshold: 1,- ingest_rate_limit_scale_seconds: 1- )+ %{id: site_id} =+ add_site_and_refresh_cache(test,+ domain: domain,+ ingest_rate_limit_threshold: 1,+ ingest_rate_limit_scale_seconds: 1+ )- assert :allow = GateKeeper.check(domain, opts)+ assert {:allow, ^site_id} = GateKeeper.check(domain, opts)Process.sleep(1)assert {:deny, :throttle} = GateKeeper.check(domain, opts)Process.sleep(1_000)- assert :allow = GateKeeper.check(domain, opts)+ assert {:allow, ^site_id} = GateKeeper.check(domain, opts)endtest "rate limiting prioritises cache lookups", %{test: test, opts: opts} do@@ -67,8 +69,9 @@ defmodule Plausible.Site.GateKeeperTest do# We need some dummy site, otherwise the cache won't refresh in case the DB# is completely emptyinsert(:site)+ deleted_site_id = site.id- assert :allow = GateKeeper.check(domain, opts)+ assert {:allow, ^deleted_site_id} = GateKeeper.check(domain, opts):ok = Cache.refresh_all(opts[:cache_opts])assert {:deny, :not_found} = GateKeeper.check(domain, opts)end@@ -86,18 +89,19 @@ defmodule Plausible.Site.GateKeeperTest doingest_rate_limit_scale_seconds: 600)- assert :allow = GateKeeper.check(domain, opts)+ site_id = site.id+ assert {:allow, ^site_id} = GateKeeper.check(domain, opts)assert {:deny, :throttle} = GateKeeper.check(domain, opts){:ok, :broken} = break_hammer(site)log =capture_log(fn ->- assert :allow = GateKeeper.check(domain, opts)+ assert {:allow, ^site_id} = GateKeeper.check(domain, opts)end)assert log =~ "Error checking rate limit for 'ingest:site:causingerrors.example.com'"- assert log =~ "Falling back to: allow"+ assert log =~ "Falling back to: :allow"end# We need a way to force Hammer to error-out on Hammer.check_rate/3.diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exsindex 0c104954ba..547bc5911d 100644--- a/test/plausible_web/controllers/api/external_controller_test.exs+++ b/test/plausible_web/controllers/api/external_controller_test.exs@@ -8,13 +8,13 @@ defmodule PlausibleWeb.Api.ExternalControllerTest dodescribe "POST /api/event" dosetup do- site1 = insert(:site)- {:ok, domain: site1.domain}+ site = insert(:site)+ {:ok, site: site}end- test "records the event", %{conn: conn, domain: domain} do+ test "records the event", %{conn: conn, site: site} doparams = %{- domain: domain,+ domain: site.domain,name: "pageview",url: "http://gigride.live/",referrer: "http://m.facebook.com/"@@ -25,17 +25,23 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("user-agent", @user_agent)|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert response(conn, 202) == "ok"assert pageview.hostname == "gigride.live"- assert pageview.domain == domain++ if Plausible.v2?() do+ assert pageview.site_id == site.id+ else+ assert pageview.domain == site.domain+ end+assert pageview.pathname == "/"end- test "works with Content-Type: text/plain", %{conn: conn, domain: domain} do+ test "works with Content-Type: text/plain", %{conn: conn, site: site} doparams = %{- domain: domain,+ domain: site.domain,name: "pageview",url: "http://gigride.live/"}@@ -45,11 +51,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("content-type", "text/plain")|> post("/api/event", Jason.encode!(params))- pageview = get_event(domain)+ pageview = get_event(site)assert response(conn, 202) == "ok"assert pageview.hostname == "gigride.live"- assert pageview.domain == domain++ if Plausible.v2?() do+ assert pageview.site_id == site.id+ else+ assert pageview.domain == site.domain+ end+assert pageview.pathname == "/"end@@ -64,15 +76,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest dotest "can send to multiple dashboards by listing multiple domains - same timestamp", %{conn: conn,- domain: domain1+ site: site1} do- domain2 = insert(:site).domain+ site2 = insert(:site)params = %{name: "pageview",url: "http://gigride.live/",referrer: "http://m.facebook.com/",- domain: "#{domain1},#{domain2}"+ domain: "#{site1.domain},#{site2.domain}"}conn =@@ -81,15 +93,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> post("/api/event", params)assert response(conn, 202) == "ok"- assert e1 = get_event(domain1)- assert e2 = get_event(domain2)+ assert e1 = get_event(site1)+ assert e2 = get_event(site2)assert NaiveDateTime.compare(e1.timestamp, e2.timestamp) == :eqend- test "timestamps differ when two events sent in a row", %{conn: conn, domain: domain} do+ test "timestamps differ when two events sent in a row", %{conn: conn, site: site} doparams = %{- domain: domain,+ domain: site.domain,name: "pageview",url: "http://gigride.live/",referrer: "http://m.facebook.com/"@@ -110,90 +122,94 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("user-agent", @user_agent)|> post("/api/event", params)- [e1, e2] = get_events(domain)+ [e1, e2] = get_events(site)assert NaiveDateTime.compare(e1.timestamp, e2.timestamp) == :gtend- test "www. is stripped from domain", %{conn: conn, domain: domain} do+ test "www. is stripped from domain", %{conn: conn, site: site} doparams = %{name: "custom event",url: "http://gigride.live/",- domain: domain+ domain: site.domain}conn|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)- assert pageview.domain == domain+ if Plausible.v2?() do+ assert pageview.site_id == site.id+ else+ assert pageview.domain == site.domain+ endend- test "www. is stripped from hostname", %{conn: conn, domain: domain} do+ test "www. is stripped from hostname", %{conn: conn, site: site} doparams = %{name: "pageview",url: "http://www.example.com/",- domain: domain+ domain: site.domain}conn|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert pageview.hostname == "example.com"end- test "empty path defaults to /", %{conn: conn, domain: domain} do+ test "empty path defaults to /", %{conn: conn, site: site} doparams = %{name: "pageview",url: "http://www.example.com",- domain: domain+ domain: site.domain}conn|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert pageview.pathname == "/"end- test "trailing whitespace is removed", %{conn: conn, domain: domain} do+ test "trailing whitespace is removed", %{conn: conn, site: site} doparams = %{name: "pageview",url: "http://www.example.com/path ",- domain: domain+ domain: site.domain}conn|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert pageview.pathname == "/path"end- test "bots and crawlers are ignored", %{conn: conn, domain: domain} do+ test "bots and crawlers are ignored", %{conn: conn, site: site} doparams = %{name: "pageview",url: "http://www.example.com/",- domain: domain+ domain: site.domain}conn|> put_req_header("user-agent", "generic crawler")|> post("/api/event", params)- assert get_event("external-controller-test-5.com") == nil+ assert get_event(site) == nilend- test "Headless Chrome is ignored", %{conn: conn, domain: domain} do+ test "Headless Chrome is ignored", %{conn: conn, site: site} doparams = %{name: "pageview",url: "http://www.example.com/",- domain: domain+ domain: site.domain}conn@@ -203,14 +219,14 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do)|> post("/api/event", params)- assert get_event(domain) == nil+ assert get_event(site) == nilend- test "parses user_agent", %{conn: conn, domain: domain} do+ test "parses user_agent", %{conn: conn, site: site} doparams = %{name: "pageview",url: "http://gigride.live/",- domain: domain+ domain: site.domain}conn =@@ -218,7 +234,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("user-agent", @user_agent)|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert response(conn, 202) == "ok"assert pageview.operating_system == "Mac"@@ -227,12 +243,12 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doassert pageview.browser_version == "70.0"end- test "parses referrer", %{conn: conn, domain: domain} do+ test "parses referrer", %{conn: conn, site: site} doparams = %{name: "pageview",url: "http://gigride.live/",referrer: "https://facebook.com",- domain: domain+ domain: site.domain}conn =@@ -240,18 +256,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("user-agent", @user_agent)|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert response(conn, 202) == "ok"assert pageview.referrer_source == "Facebook"end- test "strips trailing slash from referrer", %{conn: conn, domain: domain} do+ test "strips trailing slash from referrer", %{conn: conn, site: site} doparams = %{name: "pageview",url: "http://gigride.live/",referrer: "https://facebook.com/page/",- domain: domain+ domain: site.domain}conn =@@ -259,16 +275,16 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("user-agent", @user_agent)|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert response(conn, 202) == "ok"assert pageview.referrer == "facebook.com/page"assert pageview.referrer_source == "Facebook"end- test "ignores event when referrer is a spammer", %{conn: conn, domain: domain} do+ test "ignores event when referrer is a spammer", %{conn: conn, site: site} doparams = %{- domain: domain,+ domain: site.domain,name: "pageview",url: "http://gigride.live/",referrer: "https://www.1-best-seo.com"@@ -280,7 +296,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> post("/api/event", params)assert response(conn, 202) == "ok"- assert !get_event(domain)+ assert !get_event(site)endtest "blocks traffic from a domain when it's blocked", %{@@ -300,15 +316,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> post("/api/event", params)assert response(conn, 202) == "ok"- refute get_event(site.domain)+ refute get_event(site)end- test "ignores when referrer is internal", %{conn: conn, domain: domain} do+ test "ignores when referrer is internal", %{conn: conn, site: site} doparams = %{name: "pageview",url: "http://gigride.live/",referrer: "https://gigride.live",- domain: domain+ domain: site.domain}conn =@@ -316,18 +332,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("user-agent", @user_agent)|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert response(conn, 202) == "ok"assert pageview.referrer_source == ""end- test "ignores localhost referrer", %{conn: conn, domain: domain} do+ test "ignores localhost referrer", %{conn: conn, site: site} doparams = %{name: "pageview",url: "http://gigride.live/",referrer: "http://localhost:4000/",- domain: domain+ domain: site.domain}conn =@@ -335,18 +351,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("user-agent", @user_agent)|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert response(conn, 202) == "ok"assert pageview.referrer_source == ""end- test "parses subdomain referrer", %{conn: conn, domain: domain} do+ test "parses subdomain referrer", %{conn: conn, site: site} doparams = %{name: "pageview",url: "http://gigride.live/",referrer: "https://blog.gigride.live",- domain: domain+ domain: site.domain}conn =@@ -354,68 +370,68 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("user-agent", @user_agent)|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert response(conn, 202) == "ok"assert pageview.referrer_source == "blog.gigride.live"end- test "referrer is cleaned", %{conn: conn, domain: domain} do+ test "referrer is cleaned", %{conn: conn, site: site} doparams = %{name: "pageview",url: "http://www.example.com/",referrer: "https://www.indiehackers.com/page?query=param#hash",- domain: domain+ domain: site.domain}conn|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert pageview.referrer == "indiehackers.com/page"end- test "utm_source overrides referrer source", %{conn: conn, domain: domain} do+ test "utm_source overrides referrer source", %{conn: conn, site: site} doparams = %{name: "pageview",url: "http://www.example.com/?utm_source=betalist",referrer: "https://betalist.com/my-produxct",- domain: domain+ domain: site.domain}conn|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert pageview.referrer_source == "betalist"end- test "utm tags are stored", %{conn: conn, domain: domain} do+ test "utm tags are stored", %{conn: conn, site: site} doparams = %{name: "pageview",url:"http://www.example.com/?utm_medium=ads&utm_source=instagram&utm_campaign=video_story",- domain: domain+ domain: site.domain}conn|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert pageview.utm_medium == "ads"assert pageview.utm_source == "instagram"assert pageview.utm_campaign == "video_story"end- test "if it's an :unknown referrer, just the domain is used", %{conn: conn, domain: domain} do+ test "if it's an :unknown referrer, just the domain is used", %{conn: conn, site: site} doparams = %{name: "pageview",url: "http://gigride.live/",referrer: "https://www.indiehackers.com/landing-page-feedback",- domain: domain+ domain: site.domain}conn =@@ -423,18 +439,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("user-agent", @user_agent)|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert response(conn, 202) == "ok"assert pageview.referrer_source == "indiehackers.com"end- test "if the referrer is not http or https, it is ignored", %{conn: conn, domain: domain} do+ test "if the referrer is not http or https, it is ignored", %{conn: conn, site: site} doparams = %{name: "pageview",url: "http://gigride.live/",referrer: "android-app://com.google.android.gm",- domain: domain+ domain: site.domain}conn =@@ -442,17 +458,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("user-agent", @user_agent)|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert response(conn, 202) == "ok"assert pageview.referrer_source == ""end- test "screen size is calculated from user agent", %{conn: conn, domain: domain} do+ test "screen size is calculated from user agent", %{conn: conn, site: site} doparams = %{name: "pageview",url: "http://gigride.live/",- domain: domain+ domain: site.domain}conn =@@ -460,17 +476,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("user-agent", @user_agent_mobile)|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert response(conn, 202) == "ok"assert pageview.screen_size == "Mobile"end- test "screen size is nil if user agent is unknown", %{conn: conn, domain: domain} do+ test "screen size is nil if user agent is unknown", %{conn: conn, site: site} doparams = %{name: "pageview",url: "http://gigride.live/",- domain: domain+ domain: site.domain}conn =@@ -478,17 +494,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("user-agent", "unknown UA")|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert response(conn, 202) == "ok"assert pageview.screen_size == ""end- test "screen size is calculated from user_agent when is tablet", %{conn: conn, domain: domain} do+ test "screen size is calculated from user_agent when is tablet", %{conn: conn, site: site} doparams = %{name: "pageview",url: "http://gigride.live/",- domain: domain+ domain: site.domain}conn =@@ -496,7 +512,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("user-agent", @user_agent_tablet)|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert response(conn, 202) == "ok"assert pageview.screen_size == "Tablet"@@ -504,12 +520,12 @@ defmodule PlausibleWeb.Api.ExternalControllerTest dotest "screen size is calculated from user_agent when is desktop", %{conn: conn,- domain: domain+ site: site} doparams = %{name: "pageview",url: "http://gigride.live/",- domain: domain+ domain: site.domain}conn =@@ -517,17 +533,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("user-agent", @user_agent)|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert response(conn, 202) == "ok"assert pageview.screen_size == "Desktop"end- test "can trigger a custom event", %{conn: conn, domain: domain} do+ test "can trigger a custom event", %{conn: conn, site: site} doparams = %{name: "custom event",url: "http://gigride.live/",- domain: domain+ domain: site.domain}conn =@@ -535,17 +551,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("user-agent", @user_agent)|> post("/api/event", params)- event = get_event(domain)+ event = get_event(site)assert response(conn, 202) == "ok"assert event.name == "custom event"end- test "casts custom props to string", %{conn: conn, domain: domain} do+ test "casts custom props to string", %{conn: conn, site: site} doparams = %{name: "Signup",url: "http://gigride.live/",- domain: domain,+ domain: site.domain,props: %{bool_test: true,number_test: 12@@ -555,51 +571,51 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doconn|> post("/api/event", params)- event = get_event(domain)+ event = get_event(site)assert Map.get(event, :"meta.key") == ["bool_test", "number_test"]assert Map.get(event, :"meta.value") == ["true", "12"]end- test "ignores malformed custom props", %{conn: conn, domain: domain} do+ test "ignores malformed custom props", %{conn: conn, site: site} doparams = %{name: "Signup",url: "http://gigride.live/",- domain: domain,+ domain: site.domain,props: "\"show-more:button\""}conn|> post("/api/event", params)- event = get_event(domain)+ event = get_event(site)assert Map.get(event, :"meta.key") == []assert Map.get(event, :"meta.value") == []end- test "can send props stringified", %{conn: conn, domain: domain} do+ test "can send props stringified", %{conn: conn, site: site} doparams = %{name: "Signup",url: "http://gigride.live/",- domain: domain,+ domain: site.domain,props: Jason.encode!(%{number_test: 12})}conn|> post("/api/event", params)- event = get_event(domain)+ event = get_event(site)assert Map.get(event, :"meta.key") == ["number_test"]assert Map.get(event, :"meta.value") == ["12"]end- test "ignores custom prop with array value", %{conn: conn, domain: domain} do+ test "ignores custom prop with array value", %{conn: conn, site: site} doparams = %{name: "Signup",url: "http://gigride.live/",- domain: domain,+ domain: site.domain,props: Jason.encode!(%{wat: ["some-thing"], other: "key"})}@@ -607,17 +623,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doassert conn.status == 202- event = get_event(domain)+ event = get_event(site)assert Map.get(event, :"meta.key") == ["other"]assert Map.get(event, :"meta.value") == ["key"]end- test "ignores custom prop with map value", %{conn: conn, domain: domain} do+ test "ignores custom prop with map value", %{conn: conn, site: site} doparams = %{name: "Signup",url: "http://gigride.live/",- domain: domain,+ domain: site.domain,props: Jason.encode!(%{foo: %{bar: "baz"}, other_key: 1})}@@ -625,17 +641,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doassert conn.status == 202- event = get_event(domain)+ event = get_event(site)assert Map.get(event, :"meta.key") == ["other_key"]assert Map.get(event, :"meta.value") == ["1"]end- test "ignores custom prop with empty string value", %{conn: conn, domain: domain} do+ test "ignores custom prop with empty string value", %{conn: conn, site: site} doparams = %{name: "Signup",url: "http://gigride.live/",- domain: domain,+ domain: site.domain,props: Jason.encode!(%{foo: "", other_key: true})}@@ -643,17 +659,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doassert conn.status == 202- event = get_event(domain)+ event = get_event(site)assert Map.get(event, :"meta.key") == ["other_key"]assert Map.get(event, :"meta.value") == ["true"]end- test "ignores custom prop with nil value", %{conn: conn, domain: domain} do+ test "ignores custom prop with nil value", %{conn: conn, site: site} doparams = %{name: "Signup",url: "http://gigride.live/",- domain: domain,+ domain: site.domain,props: Jason.encode!(%{foo: nil, other_key: true})}@@ -661,18 +677,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doassert conn.status == 202- event = get_event(domain)+ event = get_event(site)assert Map.get(event, :"meta.key") == ["other_key"]assert Map.get(event, :"meta.value") == ["true"]end- test "ignores a malformed referrer URL", %{conn: conn, domain: domain} do+ test "ignores a malformed referrer URL", %{conn: conn, site: site} doparams = %{name: "pageview",url: "http://gigride.live/",referrer: "https:://twitter.com",- domain: domain+ domain: site.domain}conn =@@ -680,17 +696,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("user-agent", @user_agent)|> post("/api/event", params)- event = get_event(domain)+ event = get_event(site)assert response(conn, 202) == "ok"assert event.referrer == ""end# Fake geo is loaded from test/priv/GeoLite2-City-Test.mmdb- test "looks up location data from the ip address", %{conn: conn, domain: domain} do+ test "looks up location data from the ip address", %{conn: conn, site: site} doparams = %{name: "pageview",- domain: domain,+ domain: site.domain,url: "http://gigride.live/"}@@ -698,7 +714,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("x-forwarded-for", "2.125.160.216")|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert pageview.country_code == "GB"assert pageview.subdivision1_code == "GB-ENG"@@ -706,10 +722,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doassert pageview.city_geoname_id == 2_655_045end- test "ignores unknown country code ZZ", %{conn: conn, domain: domain} do+ test "ignores unknown country code ZZ", %{conn: conn, site: site} doparams = %{name: "pageview",- domain: domain,+ domain: site.domain,url: "http://gigride.live/"}@@ -717,7 +733,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("x-forwarded-for", "0.0.0.0")|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert pageview.country_code == <<0, 0>>assert pageview.subdivision1_code == ""@@ -725,10 +741,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doassert pageview.city_geoname_id == 0end- test "ignores disputed territory code XX", %{conn: conn, domain: domain} do+ test "ignores disputed territory code XX", %{conn: conn, site: site} doparams = %{name: "pageview",- domain: domain,+ domain: site.domain,url: "http://gigride.live/"}@@ -736,7 +752,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("x-forwarded-for", "0.0.0.1")|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert pageview.country_code == <<0, 0>>assert pageview.subdivision1_code == ""@@ -744,10 +760,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doassert pageview.city_geoname_id == 0end- test "ignores TOR exit node country code T1", %{conn: conn, domain: domain} do+ test "ignores TOR exit node country code T1", %{conn: conn, site: site} doparams = %{name: "pageview",- domain: domain,+ domain: site.domain,url: "http://gigride.live/"}@@ -755,7 +771,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("x-forwarded-for", "0.0.0.2")|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert pageview.country_code == <<0, 0>>assert pageview.subdivision1_code == ""@@ -763,10 +779,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doassert pageview.city_geoname_id == 0end- test "scrubs port from x-forwarded-for", %{conn: conn, domain: domain} do+ test "scrubs port from x-forwarded-for", %{conn: conn, site: site} doparams = %{name: "pageview",- domain: domain,+ domain: site.domain,url: "http://gigride.live/"}@@ -774,15 +790,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("x-forwarded-for", "216.160.83.56:123")|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert pageview.country_code == "US"end- test "works with ipv6 without port in x-forwarded-for", %{conn: conn, domain: domain} do+ test "works with ipv6 without port in x-forwarded-for", %{conn: conn, site: site} doparams = %{name: "pageview",- domain: domain,+ domain: site.domain,url: "http://gigride.live/"}@@ -790,15 +806,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("x-forwarded-for", "2001:218:1:1:1:1:1:1")|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert pageview.country_code == "JP"end- test "works with ipv6 with a port number in x-forwarded-for", %{conn: conn, domain: domain} do+ test "works with ipv6 with a port number in x-forwarded-for", %{conn: conn, site: site} doparams = %{name: "pageview",- domain: domain,+ domain: site.domain,url: "http://gigride.live/"}@@ -806,18 +822,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("x-forwarded-for", "[2001:218:1:1:1:1:1:1]:123")|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert pageview.country_code == "JP"endtest "uses cloudflare's special header for client IP address if present", %{conn: conn,- domain: domain+ site: site} doparams = %{name: "pageview",- domain: domain,+ domain: site.domain,url: "http://gigride.live/"}@@ -826,18 +842,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("cf-connecting-ip", "216.160.83.56")|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert pageview.country_code == "US"endtest "uses BunnyCDN's custom header for client IP address if present", %{conn: conn,- domain: domain+ site: site} doparams = %{name: "pageview",- domain: domain,+ domain: site.domain,url: "http://gigride.live/"}@@ -846,18 +862,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("b-forwarded-for", "216.160.83.56,9.9.9.9")|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert pageview.country_code == "US"endtest "Uses the Forwarded header when cf-connecting-ip and x-forwarded-for are missing", %{conn: conn,- domain: domain+ site: site} doparams = %{name: "pageview",- domain: domain,+ domain: site.domain,url: "http://gigride.live/"}@@ -865,15 +881,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("forwarded", "by=0.0.0.0;for=216.160.83.56;host=somehost.com;proto=https")|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert pageview.country_code == "US"end- test "Forwarded header can parse ipv6", %{conn: conn, domain: domain} do+ test "Forwarded header can parse ipv6", %{conn: conn, site: site} doparams = %{name: "pageview",- domain: domain,+ domain: site.domain,url: "http://gigride.live/"}@@ -884,117 +900,117 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do)|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert pageview.country_code == "JP"end- test "URL is decoded", %{conn: conn, domain: domain} do+ test "URL is decoded", %{conn: conn, site: site} doparams = %{name: "pageview",url:"http://www.example.com/opportunity/category/%D8%AC%D9%88%D8%A7%D8%A6%D8%B2-%D9%88%D9%85%D8%B3%D8%A7%D8%A8%D9%82%D8%A7%D8%AA",- domain: domain+ domain: site.domain}conn|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert pageview.pathname == "/opportunity/category/جوائز-ومسابقات"end- test "accepts shorthand map keys", %{conn: conn, domain: domain} do+ test "accepts shorthand map keys", %{conn: conn, site: site} doparams = %{n: "pageview",u: "http://www.example.com/opportunity",- d: domain,+ d: site.domain,r: "https://facebook.com/page"}conn|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert pageview.pathname == "/opportunity"assert pageview.referrer_source == "Facebook"assert pageview.referrer == "facebook.com/page"end- test "records hash when in hash mode", %{conn: conn, domain: domain} do+ test "records hash when in hash mode", %{conn: conn, site: site} doparams = %{n: "pageview",u: "http://www.example.com/#page-a",- d: domain,+ d: site.domain,h: 1}conn|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert pageview.pathname == "/#page-a"end- test "does not record hash when hash mode is 0", %{conn: conn, domain: domain} do+ test "does not record hash when hash mode is 0", %{conn: conn, site: site} doparams = %{n: "pageview",u: "http://www.example.com/#page-a",- d: domain,+ d: site.domain,h: 0}conn|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert pageview.pathname == "/"end- test "decodes URL pathname, fragment and search", %{conn: conn, domain: domain} do+ test "decodes URL pathname, fragment and search", %{conn: conn, site: site} doparams = %{n: "pageview",u:"https://test.com/%EF%BA%9D%EF%BB%AD%EF%BA%8E%EF%BA%8B%EF%BA%AF-%EF%BB%AE%EF%BB%A4%EF%BA%B3%EF%BA%8E%EF%BA%92%EF%BB%97%EF%BA%8E%EF%BA%97?utm_source=%25balle%25",- d: domain,+ d: site.domain,h: 1}conn|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert pageview.hostname == "test.com"assert pageview.pathname == "/ﺝﻭﺎﺋﺯ-ﻮﻤﺳﺎﺒﻗﺎﺗ"assert pageview.utm_source == "%balle%"end- test "can use double quotes in query params", %{conn: conn, domain: domain} do+ test "can use double quotes in query params", %{conn: conn, site: site} doq = URI.encode_query(%{"utm_source" => "Something \"quoted\""})params = %{n: "pageview",u: "https://test.com/?" <> q,- d: domain,+ d: site.domain,h: 1}conn|> post("/api/event", params)- pageview = get_event(domain)+ pageview = get_event(site)assert pageview.utm_source == "Something \"quoted\""end- test "responds 400 when required fields are missing", %{conn: conn, domain: domain} do+ test "responds 400 when required fields are missing", %{conn: conn, site: site} doparams = %{- domain: domain,+ domain: site.domain,name: "pageview"}@@ -1032,17 +1048,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest dodescribe "user_id generation" dosetup do- site1 = insert(:site)- {:ok, domain: site1.domain}+ site = insert(:site)+ {:ok, site: site}endtest "with same IP address and user agent, the same user ID is generated", %{conn: conn,- domain: domain+ site: site} doparams = %{url: "https://user-id-test-domain.com/",- domain: domain,+ domain: site.domain,name: "pageview"}@@ -1056,15 +1072,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("x-forwarded-for", "127.0.0.1")|> post("/api/event", params)- [one, two] = get_events(domain)+ [one, two] = get_events(site)assert one.user_id == two.user_idend- test "different IP address results in different user ID", %{conn: conn, domain: domain} do+ test "different IP address results in different user ID", %{conn: conn, site: site} doparams = %{url: "https://user-id-test-domain.com/",- domain: domain,+ domain: site.domain,name: "pageview"}@@ -1078,15 +1094,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("x-forwarded-for", "82.32.12.1")|> post("/api/event", params)- [one, two] = get_events(domain)+ [one, two] = get_events(site)assert one.user_id != two.user_idend- test "different user agent results in different user ID", %{conn: conn, domain: domain} do+ test "different user agent results in different user ID", %{conn: conn, site: site} doparams = %{url: "https://user-id-test-domain.com/",- domain: domain,+ domain: site.domain,name: "pageview"}@@ -1100,17 +1116,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("x-forwarded-for", "127.0.0.1")|> post("/api/event", params)- [one, two] = get_events(domain)+ [one, two] = get_events(site)assert one.user_id != two.user_idend- test "different domain value results in different user ID", %{conn: conn, domain: domain1} do- domain2 = insert(:site).domain+ test "different domain value results in different user ID", %{conn: conn, site: site1} do+ site2 = insert(:site)params = %{url: "https://user-id-test-domain.com/",- domain: domain1,+ domain: site1.domain,name: "pageview"}@@ -1122,18 +1138,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doconn|> put_req_header("user-agent", @user_agent)|> put_req_header("x-forwarded-for", "127.0.0.1")- |> post("/api/event", Map.put(params, :domain, domain2))+ |> post("/api/event", Map.put(params, :domain, site2.domain))- one = get_event(domain1)- two = get_event(domain2)+ one = get_event(site1)+ two = get_event(site2)assert one.user_id != two.user_idend- test "different hostname results in different user ID", %{conn: conn, domain: domain} do+ test "different hostname results in different user ID", %{conn: conn, site: site} doparams = %{url: "https://user-id-test-domain.com/",- domain: domain,+ domain: site.domain,name: "pageview"}@@ -1147,18 +1163,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("x-forwarded-for", "127.0.0.1")|> post("/api/event", Map.put(params, :url, "https://other-domain.com/"))- [one, two] = get_events(domain)+ [one, two] = get_events(site)assert one.user_id != two.user_idendtest "different hostname results in the same user ID when the root domain in the same", %{conn: conn,- domain: domain+ site: site} doparams = %{url: "https://user-id-test-domain.com/",- domain: domain,+ domain: site.domain,name: "pageview"}@@ -1172,7 +1188,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("x-forwarded-for", "127.0.0.1")|> post("/api/event", Map.put(params, :url, "https://app.user-id-test-domain.com/"))- [one, two] = get_events(domain)+ [one, two] = get_events(site)assert one.user_id == two.user_idend@@ -1180,13 +1196,13 @@ defmodule PlausibleWeb.Api.ExternalControllerTest dodescribe "remaining" dosetup do- site1 = insert(:site)- {:ok, domain: site1.domain}+ site = insert(:site)+ {:ok, site: site}end- test "defaults hostname to (none) when missing", %{conn: conn, domain: domain} do+ test "defaults hostname to (none) when missing", %{conn: conn, site: site} doparams = %{- domain: domain,+ domain: site.domain,name: "pageview",url: "file:///android_asset/www/index.html"}@@ -1196,15 +1212,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("content-type", "text/plain")|> post("/api/event", Jason.encode!(params))- pageview = get_event(domain)+ pageview = get_event(site)assert response(conn, 202) == "ok"assert pageview.hostname == "(none)"end- test "accepts chrome extension URLs", %{conn: conn, domain: domain} do+ test "accepts chrome extension URLs", %{conn: conn, site: site} doparams = %{- domain: domain,+ domain: site.domain,name: "pageview",url: "chrome-extension://liipgellkffekalgefpjolodblggkmjg/popup.html"}@@ -1214,7 +1230,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do|> put_req_header("content-type", "text/plain")|> post("/api/event", Jason.encode!(params))- pageview = get_event(domain)+ pageview = get_event(site)assert response(conn, 202) == "ok"assert pageview.hostname == "liipgellkffekalgefpjolodblggkmjg"@@ -1232,23 +1248,43 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doendend- defp get_event(domain) do+ defp get_event(site) doPlausible.Event.WriteBuffer.flush()- ClickhouseRepo.one(- from e in Plausible.ClickhouseEvent,- where: e.domain == ^domain,- order_by: [desc: e.timestamp]- )+ if Plausible.v2?() do+ ClickhouseRepo.one(+ from(e in Plausible.ClickhouseEventV2,+ where: e.site_id == ^site.id,+ order_by: [desc: e.timestamp]+ )+ )+ else+ ClickhouseRepo.one(+ from(e in Plausible.ClickhouseEvent,+ where: e.domain == ^site.domain,+ order_by: [desc: e.timestamp]+ )+ )+ endend- defp get_events(domain) do+ defp get_events(site) doPlausible.Event.WriteBuffer.flush()- ClickhouseRepo.all(- from e in Plausible.ClickhouseEvent,- where: e.domain == ^domain,- order_by: [desc: e.timestamp]- )+ if Plausible.v2?() do+ ClickhouseRepo.all(+ from(e in Plausible.ClickhouseEventV2,+ where: e.site_id == ^site.id,+ order_by: [desc: e.timestamp]+ )+ )+ else+ ClickhouseRepo.all(+ from(e in Plausible.ClickhouseEvent,+ where: e.domain == ^site.domain,+ order_by: [desc: e.timestamp]+ )+ )+ endendenddiff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exsindex 15dbc469e9..4fa249163a 100644--- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs@@ -97,10 +97,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doendtest "aggregates a single metric", %{conn: conn, site: site} do- populate_stats([- build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ populate_stats(site, [+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -120,12 +120,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doconn: conn,site: site} do- populate_stats([- build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, user_id: 456, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, user_id: 456, domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ populate_stats(site, [+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview, user_id: 456, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: 456, timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -145,10 +145,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doconn: conn,site: site} do- populate_stats([- build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ populate_stats(site, [+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -171,19 +171,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest dodescribe "comparisons" dotest "compare period=day with previous period", %{conn: conn, site: site} do- populate_stats([- build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]),+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -204,19 +202,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doendtest "compare period=6mo with previous period", %{conn: conn, site: site} do- populate_stats([- build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]),+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-02-01 00:25:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-03-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-03-01 00:00:00])])conn =@@ -239,19 +235,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest dodescribe "filters" dotest "can filter by source", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,referrer_source: "Google",user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -272,20 +266,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doendtest "can filter by no source/referrer", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),build(:pageview,referrer_source: "Google",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00])])@@ -308,19 +299,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doendtest "can filter by referrer", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,referrer: "https://facebook.com",user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -341,19 +330,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doendtest "can filter by utm_medium", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,utm_medium: "social",user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -374,19 +361,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doendtest "can filter by utm_source", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,utm_source: "Twitter",user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -407,19 +392,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doendtest "can filter by utm_campaign", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,utm_campaign: "profile",user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -440,19 +423,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doendtest "can filter by device type", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,screen_size: "Desktop",user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -473,19 +454,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doendtest "can filter by browser", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,browser: "Chrome",user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -506,20 +485,18 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doendtest "can filter by browser version", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,browser: "Chrome",browser_version: "56",user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -540,19 +517,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doendtest "can filter by operating system", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,operating_system: "Mac",user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -573,19 +548,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doendtest "can filter by operating system version", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,operating_system_version: "10.5",user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -606,19 +579,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doendtest "can filter by country", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,country_code: "EE",user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -642,22 +613,19 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doconn: conn,site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,user_id: @user_id,pathname: "/blogpost",- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,pathname: "/blogpost",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00])])@@ -680,25 +648,22 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doendtest "filtering by event:name", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:event,name: "Signup",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,name: "Signup",- domain: site.domain,user_id: @user_id,timestamp: ~N[2021-01-01 00:25:00]),build(:event,name: "Signup",- domain: site.domain,user_id: @user_id,timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -717,23 +682,20 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doendtest "combining filters", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,pathname: "/blogpost",country_code: "EE",user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,pathname: "/blogpost",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00])])@@ -830,15 +792,54 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest dosite: site} docreate_sessions([- %{domain: site.domain, session_id: 1000, country_code: "EE", sign: 1, events: 1},- %{domain: site.domain, session_id: 1000, country_code: "EE", sign: -1, events: 1},- %{domain: site.domain, session_id: 1000, country_code: "EE", sign: 1, events: 2}+ %{+ domain: site.domain,+ site_id: site.id,+ session_id: 1000,+ country_code: "EE",+ sign: 1,+ events: 1+ },+ %{+ domain: site.domain,+ site_id: site.id,+ session_id: 1000,+ country_code: "EE",+ sign: -1,+ events: 1+ },+ %{+ domain: site.domain,+ site_id: site.id,+ session_id: 1000,+ country_code: "EE",+ sign: 1,+ events: 2+ }])create_events([- %{domain: site.domain, session_id: 1000, country_code: "EE", name: "pageview"},- %{domain: site.domain, session_id: 1000, country_code: "EE", name: "pageview"},- %{domain: site.domain, session_id: 1000, country_code: "EE", name: "pageview"}+ %{+ domain: site.domain,+ site_id: site.id,+ session_id: 1000,+ country_code: "EE",+ name: "pageview"+ },+ %{+ domain: site.domain,+ site_id: site.id,+ session_id: 1000,+ country_code: "EE",+ name: "pageview"+ },+ %{+ domain: site.domain,+ site_id: site.id,+ session_id: 1000,+ country_code: "EE",+ name: "pageview"+ }])conn =diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exsindex 9f930dfcfe..ccd5a67172 100644--- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs@@ -131,20 +131,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "breakdown by visit:source", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,referrer_source: "Google",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,referrer_source: "Google",- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),build(:pageview,referrer_source: "",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00])])@@ -166,10 +163,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "breakdown by visit:country", %{conn: conn, site: site} do- populate_stats([- build(:pageview, country_code: "EE", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, country_code: "EE", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, country_code: "US", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ populate_stats(site, [+ build(:pageview, country_code: "EE", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, country_code: "EE", timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview, country_code: "US", timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -189,20 +186,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "breakdown by visit:referrer", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,referrer: "https://ref.com",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,referrer: "https://ref.com",- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),build(:pageview,referrer: "",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00])])@@ -224,20 +218,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "breakdown by visit:utm_medium", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,utm_medium: "Search",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,utm_medium: "Search",- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),build(:pageview,utm_medium: "",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00])])@@ -259,20 +250,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "breakdown by visit:utm_source", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,utm_source: "Google",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,utm_source: "Google",- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),build(:pageview,utm_source: "",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00])])@@ -390,20 +378,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "breakdown by visit:device", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,screen_size: "Desktop",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,screen_size: "Desktop",- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),build(:pageview,screen_size: "Mobile",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00])])@@ -425,20 +410,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "breakdown by visit:os", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,operating_system: "Mac",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,operating_system: "Mac",- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),build(:pageview,operating_system: "Windows",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00])])@@ -460,20 +442,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "breakdown by visit:os_version", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,operating_system_version: "10.5",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,operating_system_version: "10.5",- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),build(:pageview,operating_system_version: "10.6",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00])])@@ -495,10 +474,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "breakdown by visit:browser", %{conn: conn, site: site} do- populate_stats([- build(:pageview, browser: "Firefox", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, browser: "Firefox", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, browser: "Safari", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ populate_stats(site, [+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview, browser: "Safari", timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -518,20 +497,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "breakdown by visit:browser_version", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,browser_version: "56",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,browser_version: "56",- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),build(:pageview,browser_version: "57",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00])])@@ -553,12 +529,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "breakdown by event:page", %{conn: conn, site: site} do- populate_stats([- build(:pageview, pathname: "/", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, pathname: "/", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),+ populate_stats(site, [+ build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),build(:pageview,pathname: "/plausible.io",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00])])@@ -583,12 +558,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doconn: conn,site: site} do- populate_stats([- build(:pageview, pathname: "/", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, pathname: "/", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),+ populate_stats(site, [+ build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),build(:pageview,pathname: "/plausible.io",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00])])@@ -609,19 +583,16 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest dodescribe "custom events" dotest "can breakdown by event:name", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:event,name: "Signup",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,name: "Signup",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00])])@@ -643,40 +614,34 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "can breakdown by event:name with visitors and events metrics", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,- domain: site.domain,pathname: "/non-existing",user_id: @user_id,timestamp: ~N[2021-01-01 00:00:00]),build(:event,name: "404",- domain: site.domain,pathname: "/non-existing",user_id: @user_id,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,- domain: site.domain,pathname: "/non-existing",timestamp: ~N[2021-01-01 00:00:01]),build(:pageview,- domain: site.domain,pathname: "/non-existing",user_id: @user_id,timestamp: ~N[2021-01-01 00:00:02]),build(:event,name: "404",- domain: site.domain,pathname: "/non-existing",user_id: @user_id,timestamp: ~N[2021-01-01 00:00:02]),build(:pageview,- domain: site.domain,pathname: "/non-existing",user_id: @user_id,timestamp: ~N[2021-01-01 00:00:03]@@ -701,39 +666,34 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "can breakdown by event:name while filtering for something", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:event,name: "Signup",pathname: "/pageA",browser: "Chrome",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,name: "Signup",pathname: "/pageA",browser: "Chrome",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,name: "Signup",pathname: "/pageA",browser: "Safari",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,name: "Signup",pathname: "/pageB",browser: "Chrome",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,pathname: "/pageA",browser: "Chrome",- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00])])@@ -759,21 +719,18 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doconn: conn,site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,referrer_source: "Google",user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,name: "Signup",user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,- domain: site.domain,referrer_source: "Twitter",timestamp: ~N[2021-01-01 00:25:00])@@ -796,26 +753,22 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "can breakdown by event:name when filtering by event:page", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,pathname: "/pageA",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,pathname: "/pageA",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,pathname: "/pageA",name: "Signup",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,pathname: "/pageB",- domain: site.domain,referrer_source: "Twitter",timestamp: ~N[2021-01-01 00:25:00])@@ -839,28 +792,24 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "can breakdown by event:page when filtering by event:name", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:event,name: "Signup",pathname: "/pageA",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,name: "Signup",pathname: "/pageA",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,name: "Signup",pathname: "/pageB",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,pathname: "/pageB",- domain: site.domain,referrer_source: "Twitter",timestamp: ~N[2021-01-01 00:25:00])@@ -911,33 +860,29 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "breakdown by custom event property", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:event,name: "Purchase","meta.key": ["package"],"meta.value": ["business"],- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,name: "Purchase","meta.key": ["package"],"meta.value": ["personal"],- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,name: "Purchase","meta.key": ["package"],"meta.value": ["business"],- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),build(:event,name: "Some other event","meta.key": ["package"],"meta.value": ["business"],- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00])])@@ -960,45 +905,39 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "breakdown by custom event property, with (none)", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:event,name: "Purchase","meta.key": ["cost"],"meta.value": ["16"],- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,name: "Purchase","meta.key": ["cost"],"meta.value": ["16"],- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,name: "Purchase","meta.key": ["cost"],"meta.value": ["16"],- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,name: "Purchase",- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),build(:event,name: "Purchase","meta.key": ["cost"],"meta.value": ["14"],- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),build(:event,name: "Purchase","meta.key": ["cost"],"meta.value": ["14"],- domain: site.domain,timestamp: ~N[2021-01-01 00:26:00])])@@ -1022,33 +961,29 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "breakdown by custom event property, limited", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:event,name: "Purchase","meta.key": ["cost"],"meta.value": ["16"],- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,name: "Purchase","meta.key": ["cost"],"meta.value": ["18"],- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),build(:event,name: "Purchase","meta.key": ["cost"],"meta.value": ["14"],- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),build(:event,name: "Purchase","meta.key": ["cost"],"meta.value": ["14"],- domain: site.domain,timestamp: ~N[2021-01-01 00:26:00])])@@ -1072,40 +1007,35 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "breakdown by custom event property, paginated", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:event,name: "Purchase","meta.key": ["cost"],"meta.value": ["16"],- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,name: "Purchase","meta.key": ["cost"],"meta.value": ["16"],- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,name: "Purchase","meta.key": ["cost"],"meta.value": ["18"],- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),build(:event,name: "Purchase","meta.key": ["cost"],"meta.value": ["14"],- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),build(:event,name: "Purchase","meta.key": ["cost"],"meta.value": ["14"],- domain: site.domain,timestamp: ~N[2021-01-01 00:26:00])])@@ -1135,14 +1065,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doinsert(:goal, %{domain: site.domain, event_name: "Purchase"})insert(:goal, %{domain: site.domain, page_path: "/test"})- populate_stats([+ populate_stats(site, [build(:pageview,- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00],pathname: "/test"),build(:pageview,- domain: site.domain,timestamp: ~N[2021-01-01 00:00:01],pathname: "/test","meta.key": ["method"],@@ -1150,14 +1078,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do),build(:event,name: "404",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:02],"meta.key": ["method"],"meta.value": ["HTTP"]),build(:event,name: "Purchase",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:02],"meta.key": ["method"],"meta.value": ["HTTPS"]@@ -1165,14 +1091,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest dobuild(:event,name: "404",timestamp: ~N[2021-01-01 00:00:03],- domain: site.domain,"meta.key": ["OS", "method"],"meta.value": ["Linux", "HTTP"]),build(:event,name: "404",timestamp: ~N[2021-01-01 00:00:04],- domain: site.domain,"meta.key": ["version"],"meta.value": ["1"])@@ -1213,29 +1137,25 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest dodescribe "filtering" dotest "event:page filter for breakdown by session props", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,pathname: "/ignore",browser: "Chrome",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,pathname: "/plausible.io",browser: "Chrome",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,pathname: "/plausible.io",browser: "Chrome",- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),build(:pageview,browser: "Safari",pathname: "/plausible.io",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00])])@@ -1409,25 +1329,21 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "IN filter for event:page", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,pathname: "/ignore",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,pathname: "/plausible.io",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,pathname: "/plausible.io",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,pathname: "/important-page",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00])])@@ -1450,29 +1366,25 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "IN filter for visit:browser", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,pathname: "/ignore",browser: "Firefox",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,pathname: "/plausible.io",browser: "Chrome",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,pathname: "/plausible.io",browser: "Safari",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,pathname: "/important-page",browser: "Safari",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00])])@@ -1495,25 +1407,21 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "IN filter for visit:entry_page", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,pathname: "/ignore",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,pathname: "/plausible.io",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,pathname: "/plausible.io",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,pathname: "/important-page",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00])])@@ -1537,25 +1445,21 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "IN filter for event:name", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:event,name: "Signup",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,name: "Signup",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,name: "Login",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,name: "Irrelevant",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00])])@@ -1603,10 +1507,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest dodescribe "pagination" dotest "can limit results", %{conn: conn, site: site} do- populate_stats([- build(:pageview, pathname: "/a", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, pathname: "/b", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, pathname: "/c", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ populate_stats(site, [+ build(:pageview, pathname: "/a", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, pathname: "/b", timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview, pathname: "/c", timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -1623,11 +1527,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "does not repeat results", %{conn: conn, site: site} do- populate_stats([- build(:pageview, %{domain: site.domain, "meta.key": ["item"], "meta.value": ["apple"]}),- build(:pageview, %{domain: site.domain, "meta.key": ["item"], "meta.value": ["kiwi"]}),- build(:pageview, %{domain: site.domain, "meta.key": ["item"], "meta.value": ["pineapple"]}),- build(:pageview, %{domain: site.domain, "meta.key": ["item"], "meta.value": ["grapes"]})+ populate_stats(site, [+ build(:pageview, %{"meta.key": ["item"], "meta.value": ["apple"]}),+ build(:pageview, %{"meta.key": ["item"], "meta.value": ["kiwi"]}),+ build(:pageview, %{"meta.key": ["item"], "meta.value": ["pineapple"]}),+ build(:pageview, %{"meta.key": ["item"], "meta.value": ["grapes"]})])params = %{@@ -1693,10 +1597,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "can paginate results", %{conn: conn, site: site} do- populate_stats([- build(:pageview, pathname: "/a", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, pathname: "/b", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview, pathname: "/c", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ populate_stats(site, [+ build(:pageview, pathname: "/a", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, pathname: "/b", timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview, pathname: "/c", timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -1820,13 +1724,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "filter by custom event property", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:event,name: "Purchase","meta.key": ["package"],"meta.value": ["business"],browser: "Chrome",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,@@ -1834,7 +1737,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do"meta.key": ["package"],"meta.value": ["business"],browser: "Safari",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,@@ -1842,7 +1744,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do"meta.key": ["package"],"meta.value": ["business"],browser: "Safari",- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),build(:event,@@ -1850,7 +1751,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do"meta.key": ["package"],"meta.value": ["personal"],browser: "IE",- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00])])@@ -1873,23 +1773,20 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doendtest "all metrics for breakdown by event prop", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,user_id: 1,pathname: "/",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,user_id: 1,pathname: "/plausible.io",- domain: site.domain,timestamp: ~N[2021-01-01 00:10:00]),- build(:pageview, pathname: "/", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),build(:pageview,pathname: "/plausible.io",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00])])diff --git a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exsindex b84c3cf1c2..b0e1f6109b 100644--- a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs@@ -261,9 +261,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest doendtest "shows last 7 days of visitors", %{conn: conn, site: site} do- populate_stats([- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-07 23:59:00])+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-07 23:59:00])])conn =@@ -287,10 +287,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest doendtest "shows last 6 months of visitors", %{conn: conn, site: site} do- populate_stats([- build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -313,11 +313,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest doendtest "shows last 12 months of visitors", %{conn: conn, site: site} do- populate_stats([- build(:pageview, domain: site.domain, timestamp: ~N[2020-02-01 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2020-02-01 00:00:00]),+ build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -346,11 +346,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest doendtest "shows last 12 months of visitors with interval daily", %{conn: conn, site: site} do- populate_stats([- build(:pageview, domain: site.domain, timestamp: ~N[2020-02-01 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2020-02-01 00:00:00]),+ build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -365,10 +365,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest doendtest "shows a custom range with daily interval", %{conn: conn, site: site} do- populate_stats([- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-02 00:00:00])+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-02 00:00:00])])conn =@@ -387,11 +387,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest doendtest "shows a custom range with monthly interval", %{conn: conn, site: site} do- populate_stats([- build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2020-12-01 00:00:00]),- build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2020-12-01 00:05:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-02 00:00:00])+ populate_stats(site, [+ build(:pageview, user_id: @user_id, timestamp: ~N[2020-12-01 00:00:00]),+ build(:pageview, user_id: @user_id, timestamp: ~N[2020-12-01 00:05:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-02 00:00:00])])conn =@@ -425,13 +425,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest dodescribe "filters" dotest "can filter by source", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,referrer_source: "Google",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -447,11 +446,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest doendtest "can filter by no source/referrer", %{conn: conn, site: site} do- populate_stats([- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,referrer_source: "Google",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00])])@@ -469,13 +467,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest doendtest "can filter by referrer", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,referrer: "https://facebook.com",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -491,13 +488,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest doendtest "can filter by utm_medium", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,utm_medium: "social",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -513,13 +509,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest doendtest "can filter by utm_source", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,utm_source: "Twitter",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -535,13 +530,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest doendtest "can filter by utm_campaign", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,utm_campaign: "profile",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -557,13 +551,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest doendtest "can filter by device type", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,screen_size: "Desktop",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -579,20 +572,18 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest doendtest "can filter by browser", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,browser: "Chrome",browser_version: "56.1",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,browser: "Chrome",browser_version: "55",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -608,26 +599,23 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest doendtest "can filter by operating system", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,operating_system: "Mac",operating_system_version: "10.5",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,operating_system: "Something else",operating_system_version: "10.5",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,operating_system: "Mac",operating_system_version: "10.4",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -643,22 +631,20 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest doendtest "can filter by country", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,user_id: @user_id,country_code: "EE",operating_system_version: "10.5",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,user_id: @user_id,country_code: "EE",operating_system_version: "10.5",- domain: site.domain,timestamp: ~N[2021-01-01 00:15:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -685,26 +671,22 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest doconn: conn,site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,pathname: "/hello",user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:05:00]),build(:pageview,pathname: "/hello",- domain: site.domain,timestamp: ~N[2021-01-01 05:00:00]),build(:pageview,pathname: "/goobye",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00])])@@ -732,13 +714,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest doendtest "can filter by event:name", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:event,name: "Signup",- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])conn =@@ -754,33 +735,29 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest doendtest "filter by custom event property", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:event,name: "Purchase","meta.key": ["package"],"meta.value": ["business"],- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,name: "Purchase","meta.key": ["package"],"meta.value": ["business"],- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:event,name: "Purchase","meta.key": ["package"],"meta.value": ["personal"],- domain: site.domain,timestamp: ~N[2021-01-01 00:25:00]),build(:event,name: "Purchase","meta.key": ["package"],"meta.value": ["business"],- domain: site.domain,timestamp: ~N[2021-01-02 00:25:00])])@@ -801,19 +778,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest dodescribe "metrics" dotest "shows pageviews,visits,views_per_visit for last 7d", %{conn: conn, site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:05:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-07 23:59:00])+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-07 23:59:00])])conn =@@ -876,30 +851,26 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest doconn: conn,site: site} do- populate_stats([+ populate_stats(site, [build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-01 00:05:00]),build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-03 00:00:00]),build(:pageview,user_id: @user_id,- domain: site.domain,timestamp: ~N[2021-01-03 00:01:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-03 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-03 00:00:00]),- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-07 23:59:00])+ build(:pageview, timestamp: ~N[2021-01-03 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-03 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-07 23:59:00])])conn =diff --git a/test/plausible_web/controllers/api/internal_controller_test.exs b/test/plausible_web/controllers/api/internal_controller_test.exsindex 65821f796b..0c8b35721d 100644--- a/test/plausible_web/controllers/api/internal_controller_test.exs+++ b/test/plausible_web/controllers/api/internal_controller_test.exs@@ -14,7 +14,7 @@ defmodule PlausibleWeb.Api.InternalControllerTest dotest "is READY when site has at least 1 pageview", %{conn: conn, user: user} dosite = insert(:site, members: [user])- Plausible.TestUtils.create_pageviews([%{domain: site.domain}])+ Plausible.TestUtils.create_pageviews([%{site: site}])conn = get(conn, "/api/#{site.domain}/status")diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exsindex ffc38b7a12..ef2d482b36 100644--- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs@@ -653,6 +653,41 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest doinsert(:goal, %{domain: site.domain, page_path: "/*"})insert(:goal, %{domain: site.domain, page_path: "/**"})+ populate_stats(site, [+ build(:pageview,+ pathname: "/hum",+ timestamp: ~N[2019-07-01 23:00:00]+ ),+ build(:pageview,+ pathname: "/register",+ timestamp: ~N[2019-07-01 23:00:00]+ ),+ build(:pageview,+ pathname: "/reg",+ timestamp: ~N[2019-07-01 23:00:00]+ ),+ build(:pageview,+ pathname: "/billing/success",+ timestamp: ~N[2019-07-01 23:00:00]+ ),+ build(:pageview,+ pathname: "/billing/upgrade/success",+ timestamp: ~N[2019-07-01 23:00:00]+ ),+ build(:pageview,+ pathname: "/signup/new",+ timestamp: ~N[2019-07-01 23:00:00]+ ),+ build(:pageview,+ pathname: "/signup/new/2",+ timestamp: ~N[2019-07-01 23:00:00]+ ),+ build(:pageview,+ pathname: "/signup/new/3",+ timestamp: ~N[2019-07-01 23:00:00]+ )+ ])+conn =get(conn,diff --git a/test/plausible_web/controllers/api/stats_controller/current_visitors_test.exs b/test/plausible_web/controllers/api/stats_controller/current_visitors_test.exsindex 23b1b26528..1bf1a7286a 100644--- a/test/plausible_web/controllers/api/stats_controller/current_visitors_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/current_visitors_test.exs@@ -5,6 +5,11 @@ defmodule PlausibleWeb.Api.StatsController.CurrentVisitorsTest dosetup [:create_user, :log_in, :create_site]test "returns unique users in the last 5 minutes", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, user_id: 123),+ build(:pageview, user_id: 456)+ ])+conn = get(conn, "/api/stats/#{site.domain}/current-visitors")assert json_response(conn, 200) == 2diff --git a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exsindex 6742f42a73..eef27ce2cc 100644--- a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs@@ -5,23 +5,37 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest dosetup [:create_user, :log_in, :create_site]test "returns suggestions for pages without a query", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/"),+ build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/register"),+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/contact"),+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/irrelevant")+ ])+conn = get(conn, "/api/stats/#{site.domain}/suggestions/page?period=month&date=2019-01-01")assert json_response(conn, 200) == [%{"label" => "/", "value" => "/"},- %{"label" => "/register", "value" => "/register"},%{"label" => "/contact", "value" => "/contact"},- %{"label" => "/irrelevant", "value" => "/irrelevant"}+ %{"label" => "/irrelevant", "value" => "/irrelevant"},+ %{"label" => "/register", "value" => "/register"}]endtest "returns suggestions for pages with a query", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/"),+ build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/register"),+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/contact"),+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/irrelevant")+ ])+conn =get(conn, "/api/stats/#{site.domain}/suggestions/page?period=month&date=2019-01-01&q=re")assert json_response(conn, 200) == [- %{"label" => "/register", "value" => "/register"},- %{"label" => "/irrelevant", "value" => "/irrelevant"}+ %{"label" => "/irrelevant", "value" => "/irrelevant"},+ %{"label" => "/register", "value" => "/register"}]end@@ -42,16 +56,25 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest doendtest "returns suggestions for sources", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2019-01-01 23:00:00], referrer_source: "Bing"),+ build(:pageview, timestamp: ~N[2019-01-01 23:00:00], referrer_source: "10words")+ ])+conn =get(conn, "/api/stats/#{site.domain}/suggestions/source?period=month&date=2019-01-01")assert json_response(conn, 200) == [- %{"label" => "10words", "value" => "10words"},- %{"label" => "Bing", "value" => "Bing"}+ %{"label" => "Bing", "value" => "Bing"},+ %{"label" => "10words", "value" => "10words"}]endtest "returns suggestions for countries", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/", country_code: "US")+ ])+conn =get(conn,@@ -106,6 +129,10 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest doendtest "returns suggestions for screen sizes", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/", screen_size: "Desktop")+ ])+conn =get(conn, "/api/stats/#{site.domain}/suggestions/screen?period=month&date=2019-01-01")@@ -113,6 +140,10 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest doendtest "returns suggestions for browsers", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/", browser: "Chrome")+ ])+conn =get(conn, "/api/stats/#{site.domain}/suggestions/browser?period=month&date=2019-01-01")@@ -122,6 +153,14 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest dotest "returns suggestions for browser versions", %{conn: conn, site: site} dofilters = Jason.encode!(%{browser: "Chrome"})+ populate_stats(site, [+ build(:pageview,+ timestamp: ~N[2019-01-01 00:00:00],+ browser: "Chrome",+ browser_version: "78.0"+ )+ ])+conn =get(conn,@@ -132,6 +171,10 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest doendtest "returns suggestions for OS", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2019-01-01 00:00:00], operating_system: "Mac")+ ])+conn = get(conn, "/api/stats/#{site.domain}/suggestions/os?period=month&date=2019-01-01")assert json_response(conn, 200) == [%{"value" => "Mac", "label" => "Mac"}]@@ -140,6 +183,14 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest dotest "returns suggestions for OS versions", %{conn: conn, site: site} dofilters = Jason.encode!(%{os: "Mac"})+ populate_stats(site, [+ build(:pageview,+ timestamp: ~N[2019-01-01 00:00:00],+ operating_system: "Mac",+ operating_system_version: "10.15"+ )+ ])+conn =get(conn,@@ -162,6 +213,14 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest doendtest "returns suggestions for referrers", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ timestamp: ~N[2019-01-01 23:00:00],+ pathname: "/",+ referrer: "10words.com/page1"+ )+ ])+conn =get(conn, "/api/stats/#{site.domain}/suggestions/referrer?period=month&date=2019-01-01")diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exsindex eee7f47652..7ab6a20578 100644--- a/test/plausible_web/controllers/site_controller_test.exs+++ b/test/plausible_web/controllers/site_controller_test.exs@@ -41,10 +41,12 @@ defmodule PlausibleWeb.SiteControllerTest doendtest "lists all of your sites with last 24h visitors", %{conn: conn, user: user} do- insert(:site, members: [user], domain: "test-site.com")+ site = insert(:site, members: [user])++ populate_stats(site, [build(:pageview), build(:pageview), build(:pageview)])conn = get(conn, "/sites")- assert html_response(conn, 200) =~ "test-site.com"+ assert html_response(conn, 200) =~ site.domainassert html_response(conn, 200) =~ "<b>3</b> visitors in last 24h"enddiff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exsindex 9ec925d1e7..0c7d32e83b 100644--- a/test/plausible_web/controllers/stats_controller_test.exs+++ b/test/plausible_web/controllers/stats_controller_test.exs@@ -4,9 +4,10 @@ defmodule PlausibleWeb.StatsControllerTest dodescribe "GET /:website - anonymous user" dotest "public site - shows site stats", %{conn: conn} do- insert(:site, domain: "public-site.io", public: true)+ site = insert(:site, public: true)+ populate_stats(site, [build(:pageview)])- conn = get(conn, "/public-site.io")+ conn = get(conn, "/#{site.domain}")assert html_response(conn, 200) =~ "stats-react-container"end@@ -27,6 +28,7 @@ defmodule PlausibleWeb.StatsControllerTest dosetup [:create_user, :log_in, :create_site]test "can view stats of a website I've created", %{conn: conn, site: site} do+ populate_stats(site, [build(:pageview)])conn = get(conn, "/" <> site.domain)assert html_response(conn, 200) =~ "stats-react-container"enddiff --git a/test/support/clickhouse_setup.ex b/test/support/clickhouse_setup.exdeleted file mode 100644index ef4fa03e1b..0000000000--- a/test/support/clickhouse_setup.ex+++ /dev/null@@ -1,337 +0,0 @@-defmodule Plausible.Test.ClickhouseSetup do- @conversion_1_session_id 123- @conversion_2_session_id 234-- def run() do- Plausible.TestUtils.create_events([- %{- name: "pageview",- domain: "test-site.com",- pathname: "/",- country_code: "EE",- browser: "Chrome",- browser_version: "78.0",- operating_system: "Mac",- operating_system_version: "10.15",- screen_size: "Desktop",- referrer_source: "10words",- referrer: "10words.com/page1",- timestamp: ~N[2019-01-01 00:00:00],- session_id: @conversion_1_session_id- },- %{- name: "pageview",- domain: "test-site.com",- pathname: "/",- country_code: "EE",- browser: "Chrome",- browser_version: "78.0",- operating_system: "Mac",- operating_system_version: "10.15",- screen_size: "Desktop",- referrer_source: "10words",- referrer: "10words.com/page2",- timestamp: ~N[2019-01-01 00:00:00]- },- %{- name: "pageview",- domain: "test-site.com",- pathname: "/",- country_code: "US",- browser: "Chrome",- browser_version: "79.0",- operating_system: "Mac",- operating_system_version: "10.16",- screen_size: "Laptop",- referrer_source: "",- referrer: "",- timestamp: ~N[2019-01-01 00:00:00]- },- %{- name: "pageview",- domain: "test-site.com",- pathname: "/contact",- country_code: "GB",- browser: "Firefox",- operating_system: "Android",- screen_size: "Mobile",- referrer_source: "Bing",- timestamp: ~N[2019-01-01 00:00:00]- },- %{name: "pageview", domain: "test-site.com", timestamp: ~N[2019-01-31 00:00:00]},- %{- name: "Signup",- domain: "test-site.com",- session_id: @conversion_1_session_id,- timestamp: ~N[2019-01-01 01:00:00],- "meta.key": ["variant"],- "meta.value": ["A"]- },- %{- name: "Signup",- domain: "test-site.com",- session_id: @conversion_1_session_id,- timestamp: ~N[2019-01-01 02:00:00],- "meta.key": ["variant"],- "meta.value": ["B"]- },- %{- name: "Signup",- domain: "test-site.com",- session_id: @conversion_2_session_id,- timestamp: ~N[2019-01-01 02:00:00],- "meta.key": ["variant"],- "meta.value": ["B"]- },- %{- name: "pageview",- pathname: "/register",- domain: "test-site.com",- session_id: @conversion_1_session_id,- timestamp: ~N[2019-01-01 23:00:00]- },- %{- name: "pageview",- pathname: "/register",- domain: "test-site.com",- session_id: @conversion_2_session_id,- timestamp: ~N[2019-01-01 23:00:00]- },- %{- name: "pageview",- pathname: "/irrelevant",- domain: "test-site.com",- session_id: @conversion_1_session_id,- timestamp: ~N[2019-01-01 23:00:01]- },- %{- name: "pageview",- domain: "test-site.com",- referrer_source: "Google",- timestamp: ~N[2019-02-01 01:00:00]- },- %{- name: "pageview",- domain: "test-site.com",- referrer_source: "Google",- timestamp: ~N[2019-02-01 02:00:00]- },- %{- name: "pageview",- domain: "test-site.com",- referrer: "t.co/some-link",- referrer_source: "Twitter",- timestamp: ~N[2019-03-01 01:00:00]- },- %{- name: "pageview",- domain: "test-site.com",- referrer: "t.co/some-link",- referrer_source: "Twitter",- timestamp: ~N[2019-03-01 01:00:00]- },- %{- name: "pageview",- domain: "test-site.com",- referrer: "t.co/nonexistent-link",- referrer_source: "Twitter",- timestamp: ~N[2019-03-01 02:00:00]- },- %{name: "pageview", domain: "test-site.com"},- %{- name: "pageview",- domain: "test-site.com",- timestamp: Timex.now() |> Timex.shift(minutes: -3)- },- %{- name: "pageview",- domain: "test-site.com",- timestamp: Timex.now() |> Timex.shift(minutes: -6)- },- %{name: "pageview", domain: "tz-test.com", timestamp: ~N[2019-01-01 00:00:00]},- %{name: "pageview", domain: "public-site.io"},- %{- name: "pageview",- domain: "fetch-tweets-test.com",- referrer: "t.co/a-link",- referrer_source: "Twitter"- },- %{- name: "pageview",- domain: "fetch-tweets-test.com",- referrer: "t.co/b-link",- referrer_source: "Twitter",- timestamp: Timex.now() |> Timex.shift(days: -5)- },- %{- name: "pageview",- pathname: "/register",- domain: "test-site.com",- timestamp: ~N[2019-07-01 23:00:00]- },- %{- name: "pageview",- pathname: "/signup/new",- domain: "test-site.com",- timestamp: ~N[2019-07-01 23:00:00]- },- %{- name: "pageview",- pathname: "/billing/success",- domain: "test-site.com",- timestamp: ~N[2019-07-01 23:00:00]- },- %{- name: "pageview",- pathname: "/",- domain: "test-site.com",- timestamp: ~N[2019-07-01 23:00:00]- },- %{- name: "pageview",- pathname: "/reg",- domain: "test-site.com",- timestamp: ~N[2019-07-01 23:00:00]- },- %{- name: "pageview",- pathname: "/signup/new/2",- domain: "test-site.com",- timestamp: ~N[2019-07-01 23:00:00]- },- %{- name: "pageview",- pathname: "/signup/new/3",- domain: "test-site.com",- timestamp: ~N[2019-07-01 23:00:00]- },- %{- name: "pageview",- pathname: "/billing/upgrade/success",- domain: "test-site.com",- timestamp: ~N[2019-07-01 23:00:00]- }- ])-- Plausible.TestUtils.create_sessions([- %{- domain: "test-site.com",- entry_page: "/",- exit_page: "/",- referrer_source: "10words",- referrer: "10words.com/page1",- utm_medium: "listing",- country_code: "US",- screen_size: "Desktop",- browser: "Chrome",- browser_version: "78.0",- operating_system: "Mac",- operating_system_version: "10.15",- session_id: @conversion_1_session_id,- is_bounce: true,- duration: 100,- start: ~N[2019-01-01 02:00:00],- timestamp: ~N[2019-01-01 02:00:00]- },- %{- domain: "test-site.com",- entry_page: "/",- exit_page: "/",- referrer_source: "10words",- referrer: "10words.com/page1",- utm_medium: "listing",- session_id: @conversion_2_session_id,- is_bounce: false,- duration: 0,- start: ~N[2019-01-01 02:00:00],- timestamp: ~N[2019-01-01 02:00:00]- },- %{- domain: "test-site.com",- entry_page: "/",- exit_page: "/",- referrer_source: "Bing",- referrer: "",- utm_medium: "search",- is_bounce: false,- duration: 100,- start: ~N[2019-01-01 03:00:00],- timestamp: ~N[2019-01-01 03:00:00]- },- %{- domain: "test-site.com",- entry_page: "/",- exit_page: "/",- referrer_source: "Google",- referrer: "",- is_bounce: false,- start: ~N[2019-02-01 01:00:00],- timestamp: ~N[2019-02-01 01:00:00]- },- %{- domain: "test-site.com",- entry_page: "/",- exit_page: "/",- referrer_source: "Google",- referrer: "",- is_bounce: false,- start: ~N[2019-02-01 02:00:00],- timestamp: ~N[2019-02-01 02:00:00]- },- %{- domain: "test-site.com",- entry_page: "/",- exit_page: "/",- referrer: "t.co/some-link",- referrer_source: "Twitter",- start: ~N[2019-03-01 01:00:00],- timestamp: ~N[2019-03-01 01:00:00]- },- %{- domain: "test-site.com",- entry_page: "/",- exit_page: "/",- referrer: "t.co/some-link",- referrer_source: "Twitter",- start: ~N[2019-03-01 01:00:00],- timestamp: ~N[2019-03-01 01:00:00]- },- %{- domain: "test-site.com",- entry_page: "/",- exit_page: "/",- referrer: "t.co/nonexistent-link",- referrer_source: "Twitter",- start: ~N[2019-03-01 02:00:00],- timestamp: ~N[2019-03-01 02:00:00]- },- %{- domain: "test-site.com",- entry_page: "/",- exit_page: "/",- referrer_source: "Bing",- referrer: "bing.com",- start: Timex.now() |> Timex.shift(minutes: -1),- timestamp: Timex.now() |> Timex.shift(minutes: -1)- },- %{- domain: "test-site.com",- entry_page: "/",- exit_page: "/exit",- referrer_source: "10words",- referrer: "10words.com",- start: Timex.now() |> Timex.shift(minutes: -2),- timestamp: Timex.now() |> Timex.shift(minutes: -2)- },- %{- domain: "test-site.com",- entry_page: "/",- exit_page: "/exit",- referrer_source: "10words",- referrer: "10words.com",- start: Timex.now() |> Timex.shift(minutes: -3),- timestamp: Timex.now() |> Timex.shift(minutes: -3)- }- ])- end-enddiff --git a/test/support/factory.ex b/test/support/factory.exindex 629cdbb075..c2e9ecf768 100644--- a/test/support/factory.ex+++ b/test/support/factory.ex@@ -38,19 +38,35 @@ defmodule Plausible.Factory dodef ch_session_factory dohostname = sequence(:domain, &"example-#{&1}.com")- %Plausible.ClickhouseSession{- sign: 1,- session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),- user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),- hostname: hostname,- domain: hostname,- entry_page: "/",- pageviews: 1,- events: 1,- start: Timex.now(),- timestamp: Timex.now(),- is_bounce: false- }+ if Plausible.v2?() do+ %Plausible.ClickhouseSessionV2{+ sign: 1,+ session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),+ user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),+ hostname: hostname,+ site_id: Enum.random(1000..10_000),+ entry_page: "/",+ pageviews: 1,+ events: 1,+ start: Timex.now(),+ timestamp: Timex.now(),+ is_bounce: false+ }+ else+ %Plausible.ClickhouseSession{+ sign: 1,+ session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),+ user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),+ hostname: hostname,+ domain: hostname,+ entry_page: "/",+ pageviews: 1,+ events: 1,+ start: Timex.now(),+ timestamp: Timex.now(),+ is_bounce: false+ }+ endenddef pageview_factory do@@ -65,14 +81,25 @@ defmodule Plausible.Factory dodef event_factory dohostname = sequence(:domain, &"example-#{&1}.com")- %Plausible.ClickhouseEvent{- hostname: hostname,- domain: hostname,- pathname: "/",- timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second),- user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),- session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate())- }+ if Plausible.v2?() do+ %Plausible.ClickhouseEventV2{+ hostname: hostname,+ site_id: Enum.random(1000..10_000),+ pathname: "/",+ timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second),+ user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),+ session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate())+ }+ else+ %Plausible.ClickhouseEvent{+ hostname: hostname,+ domain: hostname,+ pathname: "/",+ timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second),+ user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),+ session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate())+ }+ endenddef goal_factory dodiff --git a/test/support/test_utils.ex b/test/support/test_utils.exindex a129fd2ade..0cf49d40ef 100644--- a/test/support/test_utils.ex+++ b/test/support/test_utils.ex@@ -37,7 +37,6 @@ defmodule Plausible.TestUtils dodef create_site(%{user: user}) dosite =Factory.insert(:site,- domain: "test-site.com",members: [user])@@ -73,30 +72,63 @@ defmodule Plausible.TestUtils dodef create_pageviews(pageviews) dopageviews =Enum.map(pageviews, fn pageview ->+ pageview =+ if Plausible.v2?() do+ pageview+ |> Map.delete(:site)+ |> Map.put(:site_id, pageview.site.id)+ else+ pageview+ |> Map.delete(:site)+ |> Map.put(:domain, pageview.site.domain)+ end+Factory.build(:pageview, pageview)|> Map.from_struct()|> Map.delete(:__meta__)|> update_in([:timestamp], &to_naive_truncate/1)end)- Plausible.IngestRepo.insert_all(Plausible.ClickhouseEvent, pageviews)+ if Plausible.v2?() do+ Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, pageviews)+ else+ Plausible.IngestRepo.insert_all(Plausible.ClickhouseEvent, pageviews)+ endenddef create_events(events) doevents =Enum.map(events, fn event ->+ event =+ if Plausible.v2?() do+ Map.delete(event, :domain)+ else+ Map.delete(event, :site_id)+ end+Factory.build(:event, event)|> Map.from_struct()|> Map.delete(:__meta__)|> update_in([:timestamp], &to_naive_truncate/1)end)- Plausible.IngestRepo.insert_all(Plausible.ClickhouseEvent, events)+ if Plausible.v2?() do+ Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, events)+ else+ Plausible.IngestRepo.insert_all(Plausible.ClickhouseEvent, events)+ endenddef create_sessions(sessions) dosessions =Enum.map(sessions, fn session ->+ session =+ if Plausible.v2?() do+ Map.delete(session, :domain)+ else+ Map.delete(session, :site_id)+ end+Factory.build(:ch_session, session)|> Map.from_struct()|> Map.delete(:__meta__)@@ -104,7 +136,11 @@ defmodule Plausible.TestUtils do|> update_in([:start], &to_naive_truncate/1)end)- Plausible.IngestRepo.insert_all(Plausible.ClickhouseSession, sessions)+ if Plausible.v2?() do+ Plausible.IngestRepo.insert_all(Plausible.ClickhouseSessionV2, sessions)+ else+ Plausible.IngestRepo.insert_all(Plausible.ClickhouseSession, sessions)+ endenddef log_in(%{user: user, conn: conn}) do@@ -134,6 +170,9 @@ defmodule Plausible.TestUtils dodef populate_stats(site, events) doEnum.map(events, fn event ->case event do+ %Plausible.ClickhouseEventV2{} ->+ Map.put(event, :site_id, site.id)+%Plausible.ClickhouseEvent{} ->Map.put(event, :domain, site.domain)@@ -158,6 +197,9 @@ defmodule Plausible.TestUtils doend)|> Enum.split_with(fn event ->case event do+ %Plausible.ClickhouseEventV2{} ->+ true+%Plausible.ClickhouseEvent{} ->true@@ -174,11 +216,22 @@ defmodule Plausible.TestUtils dosessions =Enum.reduce(events, %{}, fn event, sessions ->session_id = Plausible.Session.CacheStore.on_event(event, nil)- Map.put(sessions, {event.domain, event.user_id}, session_id)++ if Plausible.v2?() do+ Map.put(sessions, {event.site_id, event.user_id}, session_id)+ else+ Map.put(sessions, {event.domain, event.user_id}, session_id)+ endend)Enum.each(events, fn event ->- event = Map.put(event, :session_id, sessions[{event.domain, event.user_id}])+ event =+ if Plausible.v2?() do+ Map.put(event, :session_id, sessions[{event.site_id, event.user_id}])+ else+ Map.put(event, :session_id, sessions[{event.domain, event.user_id}])+ end+Plausible.Event.WriteBuffer.insert(event)end)diff --git a/test/test_helper.exs b/test/test_helper.exsindex 62161c24ab..4d594919b2 100644--- a/test/test_helper.exs+++ b/test/test_helper.exs@@ -1,7 +1,14 @@{:ok, _} = Application.ensure_all_started(:ex_machina)-Plausible.Test.ClickhouseSetup.run()Mox.defmock(Plausible.HTTPClient.Mock, for: Plausible.HTTPClient.Interface)FunWithFlags.enable(:visits_metric)ExUnit.start(exclude: :slow)Application.ensure_all_started(:double)Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual)++if Plausible.v2?() do+ IO.puts("Running tests against v2 schema")+else+ IO.puts(+ "Running tests against v1 schema. Use: `V2_MIGRATION_DONE=1 mix test` for secondary run."+ )+enddiff --git a/test/workers/send_check_stats_emails_test.exs b/test/workers/send_check_stats_emails_test.exsindex 8a021d5137..b247e119c5 100644--- a/test/workers/send_check_stats_emails_test.exs+++ b/test/workers/send_check_stats_emails_test.exs@@ -25,6 +25,8 @@ defmodule Plausible.Workers.SendCheckStatsEmailsTest dotest "does not send an email if the user has configured a weekly report" douser = insert(:user, inserted_at: days_ago(9), last_seen: days_ago(7))site = insert(:site, domain: "test-site.com", members: [user])++ populate_stats(site, [build(:pageview)])insert(:weekly_report, site: site, recipients: ["[email protected]"])perform_job(SendCheckStatsEmails, %{})@@ -34,7 +36,8 @@ defmodule Plausible.Workers.SendCheckStatsEmailsTest dotest "sends an email after a week of signup if the user hasn't logged in" douser = insert(:user, inserted_at: days_ago(8), last_seen: days_ago(8))- insert(:site, domain: "test-site.com", members: [user])+ site = insert(:site, domain: "test-site.com", members: [user])+ populate_stats(site, [build(:pageview)])perform_job(SendCheckStatsEmails, %{})diff --git a/test/workers/send_email_report_test.exs b/test/workers/send_email_report_test.exsindex 0b22fd5577..ed3a77cb60 100644--- a/test/workers/send_email_report_test.exs+++ b/test/workers/send_email_report_test.exs@@ -39,17 +39,17 @@ defmodule Plausible.Workers.SendEmailReportTest docreate_pageviews([# Sunday before last, not counted- %{domain: site.domain, timestamp: Timezone.convert(sunday_before_last, "UTC")},+ %{site: site, timestamp: Timezone.convert(sunday_before_last, "UTC")},# Sunday before last, not counted- %{domain: site.domain, timestamp: Timezone.convert(sunday_before_last, "UTC")},+ %{site: site, timestamp: Timezone.convert(sunday_before_last, "UTC")},# Last monday, counted- %{domain: site.domain, timestamp: Timezone.convert(last_monday, "UTC")},+ %{site: site, timestamp: Timezone.convert(last_monday, "UTC")},# Last sunday, counted- %{domain: site.domain, timestamp: Timezone.convert(last_sunday, "UTC")},+ %{site: site, timestamp: Timezone.convert(last_sunday, "UTC")},# This monday, not counted- %{domain: site.domain, timestamp: Timezone.convert(this_monday, "UTC")},+ %{site: site, timestamp: Timezone.convert(this_monday, "UTC")},# This monday, not counted- %{domain: site.domain, timestamp: Timezone.convert(this_monday, "UTC")}+ %{site: site, timestamp: Timezone.convert(this_monday, "UTC")}])perform_job(SendEmailReport, %{"site_id" => site.id, "interval" => "weekly"})diff --git a/test/workers/send_site_setup_emails_test.exs b/test/workers/send_site_setup_emails_test.exsindex 3634f06609..64862d2ab6 100644--- a/test/workers/send_site_setup_emails_test.exs+++ b/test/workers/send_site_setup_emails_test.exs@@ -41,7 +41,9 @@ defmodule Plausible.Workers.SendSiteSetupEmailsTest dotest "sends the setup completed email as soon as possible" douser = insert(:user)- insert(:site, members: [user], domain: "test-site.com")+ site = insert(:site, members: [user])++ populate_stats(site, [build(:pageview)])perform_job(SendSiteSetupEmails, %{})@@ -62,7 +64,7 @@ defmodule Plausible.Workers.SendSiteSetupEmailsTest dosubject: "Your Plausible setup: Waiting for the first page views")- create_pageviews([%{domain: site.domain}])+ create_pageviews([%{site: site}])perform_job(SendSiteSetupEmails, %{})assert_email_delivered_with(diff --git a/test/workers/send_trial_notifications_test.exs b/test/workers/send_trial_notifications_test.exsindex c056f919f7..768c38a7bd 100644--- a/test/workers/send_trial_notifications_test.exs+++ b/test/workers/send_trial_notifications_test.exs@@ -44,7 +44,7 @@ defmodule Plausible.Workers.SendTrialNotificationsTest do])- populate_stats(site, [build(:pageview, domain: site.domain)])+ populate_stats(site, [build(:pageview)])perform_job(SendTrialNotifications, %{})@@ -55,7 +55,7 @@ defmodule Plausible.Workers.SendTrialNotificationsTest dotest "sends a reminder 7 days before trial ends (16 days after user signed up)" douser = insert(:user, trial_expiry_date: Timex.now() |> Timex.shift(days: 7))site = insert(:site, members: [user])- populate_stats(site, [build(:pageview, domain: site.domain)])+ populate_stats(site, [build(:pageview)])perform_job(SendTrialNotifications, %{})@@ -67,9 +67,9 @@ defmodule Plausible.Workers.SendTrialNotificationsTest dosite = insert(:site, members: [user])populate_stats(site, [- build(:pageview, domain: site.domain),- build(:pageview, domain: site.domain),- build(:pageview, domain: site.domain)+ build(:pageview),+ build(:pageview),+ build(:pageview)])perform_job(SendTrialNotifications, %{})@@ -82,9 +82,9 @@ defmodule Plausible.Workers.SendTrialNotificationsTest dosite = insert(:site, members: [user])populate_stats(site, [- build(:pageview, domain: site.domain),- build(:pageview, domain: site.domain),- build(:pageview, domain: site.domain)+ build(:pageview),+ build(:pageview),+ build(:pageview)])perform_job(SendTrialNotifications, %{})@@ -115,9 +115,9 @@ defmodule Plausible.Workers.SendTrialNotificationsTest dosite = insert(:site, members: [user])populate_stats(site, [- build(:pageview, domain: site.domain),- build(:pageview, domain: site.domain),- build(:pageview, domain: site.domain)+ build(:pageview),+ build(:pageview),+ build(:pageview)])perform_job(SendTrialNotifications, %{})@@ -130,9 +130,9 @@ defmodule Plausible.Workers.SendTrialNotificationsTest dosite = insert(:site, members: [user])populate_stats(site, [- build(:pageview, domain: site.domain),- build(:pageview, domain: site.domain),- build(:pageview, domain: site.domain)+ build(:pageview),+ build(:pageview),+ build(:pageview)])insert(:subscription, user: user)