Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Conditionally support switching between v1 and v2 clickhouse schemas (#2780)

hq1d2f2c69387bd
diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml
index 540b8a47c0..003689b0c0 100644
--- a/.github/workflows/elixir.yml
+++ b/.github/workflows/elixir.yml
@@ -70,8 +70,12 @@ jobs:
run: npm run deploy --prefix ./tracker
- name: Check Credo Warnings
run: mix credo diff --from-git-merge-base origin/master
- - name: Run tests
+ - name: Run tests (against v1 schema)
run: mix test --include slow --max-failures 1 --warnings-as-errors
+ - name: Run tests (against v2 schema)
+ run: mix test --include slow --max-failures 1 --warnings-as-errors
+ env:
+ V2_MIGRATION_DONE: 1
- name: Check Dialyzer
run: mix dialyzer
env:
diff --git a/config/runtime.exs b/config/runtime.exs
index 744b66e1cc..c937bd1c7d 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -99,6 +99,8 @@ ch_db_url =
|> get_var_from_path_or_env("CLICKHOUSE_MAX_BUFFER_SIZE", "10000")
|> Integer.parse()
+v2_migration_done = get_var_from_path_or_env(config_dir, "V2_MIGRATION_DONE")
+
### Mandatory params End
build_metadata_raw = get_var_from_path_or_env(config_dir, "BUILD_METADATA", "{}")
@@ -131,6 +133,8 @@ runtime_metadata = [
config :plausible, :runtime_metadata, runtime_metadata
+config :plausible, :v2_migration_done, v2_migration_done
+
sentry_dsn = get_var_from_path_or_env(config_dir, "SENTRY_DSN")
honeycomb_api_key = get_var_from_path_or_env(config_dir, "HONEYCOMB_API_KEY")
honeycomb_dataset = get_var_from_path_or_env(config_dir, "HONEYCOMB_DATASET")
diff --git a/lib/plausible.ex b/lib/plausible.ex
index ebdaaaaedb..0dbc04bf0d 100644
--- a/lib/plausible.ex
+++ b/lib/plausible.ex
@@ -6,4 +6,9 @@ defmodule Plausible do
Contexts are also responsible for managing your data, regardless
if it comes from the database, an external API or others.
"""
+
+ @spec v2?() :: boolean()
+ def v2?() do
+ Plausible.DataMigration.NumericIDs.ready?()
+ end
end
diff --git a/lib/plausible/billing/billing.ex b/lib/plausible/billing/billing.ex
index 2c09d8bce0..9967c8a87f 100644
--- a/lib/plausible/billing/billing.ex
+++ b/lib/plausible/billing/billing.ex
@@ -113,17 +113,36 @@ defmodule Plausible.Billing do
def last_two_billing_months_usage(user, today \\ Timex.today()) do
{first, second} = last_two_billing_cycles(user, today)
- domains = Plausible.Sites.owned_sites_domains(user)
- usage_for_sites = fn domains, date_range ->
- {pageviews, custom_events} = Plausible.Stats.Clickhouse.usage_breakdown(domains, date_range)
- pageviews + custom_events
- end
+ if Plausible.v2?() do
+ site_ids = Plausible.Sites.owned_site_ids(user)
- {
- usage_for_sites.(domains, first),
- usage_for_sites.(domains, second)
- }
+ usage_for_sites = fn site_ids, date_range ->
+ {pageviews, custom_events} =
+ Plausible.Stats.Clickhouse.usage_breakdown(site_ids, date_range)
+
+ pageviews + custom_events
+ end
+
+ {
+ usage_for_sites.(site_ids, first),
+ usage_for_sites.(site_ids, second)
+ }
+ else
+ domains = Plausible.Sites.owned_sites_domains(user)
+
+ usage_for_sites = fn domains, date_range ->
+ {pageviews, custom_events} =
+ Plausible.Stats.Clickhouse.usage_breakdown(domains, date_range)
+
+ pageviews + custom_events
+ end
+
+ {
+ usage_for_sites.(domains, first),
+ usage_for_sites.(domains, second)
+ }
+ end
end
def last_two_billing_cycles(user, today \\ Timex.today()) do
@@ -147,8 +166,13 @@ defmodule Plausible.Billing do
end
def usage_breakdown(user) do
- domains = Plausible.Sites.owned_sites_domains(user)
- Plausible.Stats.Clickhouse.usage_breakdown(domains)
+ if Plausible.v2?() do
+ site_ids = Plausible.Sites.owned_site_ids(user)
+ Plausible.Stats.Clickhouse.usage_breakdown(site_ids)
+ else
+ domains = Plausible.Sites.owned_sites_domains(user)
+ Plausible.Stats.Clickhouse.usage_breakdown(domains)
+ end
end
@doc """
diff --git a/lib/plausible/data_migration.ex b/lib/plausible/data_migration.ex
index 611826cef2..be38859286 100644
--- a/lib/plausible/data_migration.ex
+++ b/lib/plausible/data_migration.ex
@@ -13,7 +13,7 @@ defmodule Plausible.DataMigration do
@repo repo
def run_sql_confirm(name, assigns \\ []) do
- query = unwrap(name, assigns)
+ query = unwrap_with_io(name, assigns)
confirm("Execute?", fn -> do_run(name, query) end)
end
@@ -30,7 +30,9 @@ defmodule Plausible.DataMigration do
end
defp unwrap(name, assigns) do
- "priv/data_migrations"
+ :plausible
+ |> :code.priv_dir()
+ |> Path.join("data_migrations")
|> Path.join(@dir)
|> Path.join("sql")
|> Path.join(name <> ".sql.eex")
diff --git a/lib/plausible/data_migration/numeric_ids.ex b/lib/plausible/data_migration/numeric_ids.ex
index 8782df7a2c..4774f465b6 100644
--- a/lib/plausible/data_migration/numeric_ids.ex
+++ b/lib/plausible/data_migration/numeric_ids.ex
@@ -7,6 +7,10 @@ defmodule Plausible.DataMigration.NumericIDs do
@table_settings "SETTINGS index_granularity = 8192, storage_policy = 'tiered'"
+ def ready?() do
+ Application.get_env(:plausible, :v2_migration_done) || false
+ end
+
# credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity
def run(opts \\ []) do
interactive? = Keyword.get(opts, :interactive?, true)
@@ -65,7 +69,7 @@ defmodule Plausible.DataMigration.NumericIDs do
- table_settings: #{table_settings}
- db url: #{db_url}
- cluster?: #{cluster?}
- - partitions to do: #{inspect(partitions, pretty: true, width: 80)}
+ - partitions to do: #{inspect(partitions, pretty: true, limit: :infinity, width: 80)}
- start from: #{start_from}
- stop at: #{stop_at}
""")
diff --git a/lib/plausible/event/write_buffer.ex b/lib/plausible/event/write_buffer.ex
index ccb3fcb25f..f6af74b17e 100644
--- a/lib/plausible/event/write_buffer.ex
+++ b/lib/plausible/event/write_buffer.ex
@@ -64,7 +64,12 @@ defmodule Plausible.Event.WriteBuffer do
events ->
Logger.info("Flushing #{length(events)} events")
events = Enum.map(events, &(Map.from_struct(&1) |> Map.delete(:__meta__)))
- IngestRepo.insert_all(Plausible.ClickhouseEvent, events)
+
+ if Plausible.v2?() do
+ IngestRepo.insert_all(Plausible.ClickhouseEventV2, events)
+ else
+ IngestRepo.insert_all(Plausible.ClickhouseEvent, events)
+ end
end
end
diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex
index 72b3253b00..00a475892b 100644
--- a/lib/plausible/ingestion/event.ex
+++ b/lib/plausible/ingestion/event.ex
@@ -7,9 +7,11 @@ defmodule Plausible.Ingestion.Event do
"""
alias Plausible.Ingestion.Request
alias Plausible.ClickhouseEvent
+ alias Plausible.ClickhouseEventV2
alias Plausible.Site.GateKeeper
defstruct domain: nil,
+ site_id: nil,
clickhouse_event_attrs: %{},
clickhouse_event: nil,
dropped?: false,
@@ -26,8 +28,9 @@ defmodule Plausible.Ingestion.Event do
@type t() :: %__MODULE__{
domain: String.t() | nil,
+ site_id: pos_integer() | nil,
clickhouse_event_attrs: map(),
- clickhouse_event: %ClickhouseEvent{} | nil,
+ clickhouse_event: %ClickhouseEvent{} | %ClickhouseEventV2{} | nil,
dropped?: boolean(),
drop_reason: drop_reason(),
request: Request.t(),
@@ -43,10 +46,10 @@ defmodule Plausible.Ingestion.Event do
else
Enum.reduce(domains, [], fn domain, acc ->
case GateKeeper.check(domain) do
- :allow ->
+ {:allow, site_id} ->
processed =
domain
- |> new(request)
+ |> new(site_id, request)
|> process_unless_dropped(pipeline())
[processed | acc]
@@ -117,6 +120,10 @@ defmodule Plausible.Ingestion.Event do
struct!(__MODULE__, domain: domain, request: request)
end
+ defp new(domain, site_id, request) do
+ struct!(__MODULE__, domain: domain, site_id: site_id, request: request)
+ end
+
defp drop(%__MODULE__{} = event, reason, attrs \\ []) do
fields =
attrs
@@ -156,6 +163,7 @@ defmodule Plausible.Ingestion.Event do
defp put_basic_info(%__MODULE__{} = event) do
update_attrs(event, %{
domain: event.domain,
+ site_id: event.site_id,
timestamp: event.request.timestamp,
name: event.request.event_name,
hostname: event.request.hostname,
@@ -217,9 +225,15 @@ defmodule Plausible.Ingestion.Event do
defp validate_clickhouse_event(%__MODULE__{} = event) do
clickhouse_event =
- event
- |> Map.fetch!(:clickhouse_event_attrs)
- |> ClickhouseEvent.new()
+ if Plausible.v2?() do
+ event
+ |> Map.fetch!(:clickhouse_event_attrs)
+ |> ClickhouseEventV2.new()
+ else
+ event
+ |> Map.fetch!(:clickhouse_event_attrs)
+ |> ClickhouseEvent.new()
+ end
case Ecto.Changeset.apply_action(clickhouse_event, nil) do
{:ok, valid_clickhouse_event} ->
diff --git a/lib/plausible/session/cache_store.ex b/lib/plausible/session/cache_store.ex
index b48e1bc229..3563673ed3 100644
--- a/lib/plausible/session/cache_store.ex
+++ b/lib/plausible/session/cache_store.ex
@@ -22,7 +22,14 @@ defmodule Plausible.Session.CacheStore do
defp find_session(_domain, nil), do: nil
defp find_session(event, user_id) do
- case Cachex.get(:sessions, {event.domain, user_id}) do
+ from_cache =
+ if Plausible.v2?() do
+ Cachex.get(:sessions, {event.site_id, user_id})
+ else
+ Cachex.get(:sessions, {event.domain, user_id})
+ end
+
+ case from_cache do
{:ok, nil} ->
nil
@@ -38,7 +45,13 @@ defmodule Plausible.Session.CacheStore do
end
defp persist_session(session) do
- key = {session.domain, session.user_id}
+ key =
+ if Plausible.v2?() do
+ {session.site_id, session.user_id}
+ else
+ {session.domain, session.user_id}
+ end
+
Cachex.put(:sessions, key, session, ttl: :timer.minutes(30))
session
end
@@ -68,38 +81,74 @@ defmodule Plausible.Session.CacheStore do
end
defp new_session_from_event(event) do
- %Plausible.ClickhouseSession{
- sign: 1,
- session_id: Plausible.ClickhouseSession.random_uint64(),
- hostname: event.hostname,
- domain: event.domain,
- user_id: event.user_id,
- entry_page: event.pathname,
- exit_page: event.pathname,
- is_bounce: true,
- duration: 0,
- pageviews: if(event.name == "pageview", do: 1, else: 0),
- events: 1,
- referrer: event.referrer,
- referrer_source: event.referrer_source,
- utm_medium: event.utm_medium,
- utm_source: event.utm_source,
- utm_campaign: event.utm_campaign,
- utm_content: event.utm_content,
- utm_term: event.utm_term,
- country_code: event.country_code,
- subdivision1_code: event.subdivision1_code,
- subdivision2_code: event.subdivision2_code,
- city_geoname_id: event.city_geoname_id,
- screen_size: event.screen_size,
- operating_system: event.operating_system,
- operating_system_version: event.operating_system_version,
- browser: event.browser,
- browser_version: event.browser_version,
- timestamp: event.timestamp,
- start: event.timestamp,
- "entry_meta.key": Map.get(event, :"meta.key"),
- "entry_meta.value": Map.get(event, :"meta.value")
- }
+ if Plausible.v2?() do
+ %Plausible.ClickhouseSessionV2{
+ sign: 1,
+ session_id: Plausible.ClickhouseSession.random_uint64(),
+ hostname: event.hostname,
+ site_id: event.site_id,
+ user_id: event.user_id,
+ entry_page: event.pathname,
+ exit_page: event.pathname,
+ is_bounce: true,
+ duration: 0,
+ pageviews: if(event.name == "pageview", do: 1, else: 0),
+ events: 1,
+ referrer: event.referrer,
+ referrer_source: event.referrer_source,
+ utm_medium: event.utm_medium,
+ utm_source: event.utm_source,
+ utm_campaign: event.utm_campaign,
+ utm_content: event.utm_content,
+ utm_term: event.utm_term,
+ country_code: event.country_code,
+ subdivision1_code: event.subdivision1_code,
+ subdivision2_code: event.subdivision2_code,
+ city_geoname_id: event.city_geoname_id,
+ screen_size: event.screen_size,
+ operating_system: event.operating_system,
+ operating_system_version: event.operating_system_version,
+ browser: event.browser,
+ browser_version: event.browser_version,
+ timestamp: event.timestamp,
+ start: event.timestamp,
+ "entry_meta.key": Map.get(event, :"meta.key"),
+ "entry_meta.value": Map.get(event, :"meta.value")
+ }
+ else
+ %Plausible.ClickhouseSession{
+ sign: 1,
+ session_id: Plausible.ClickhouseSession.random_uint64(),
+ hostname: event.hostname,
+ domain: event.domain,
+ user_id: event.user_id,
+ entry_page: event.pathname,
+ exit_page: event.pathname,
+ is_bounce: true,
+ duration: 0,
+ pageviews: if(event.name == "pageview", do: 1, else: 0),
+ events: 1,
+ referrer: event.referrer,
+ referrer_source: event.referrer_source,
+ utm_medium: event.utm_medium,
+ utm_source: event.utm_source,
+ utm_campaign: event.utm_campaign,
+ utm_content: event.utm_content,
+ utm_term: event.utm_term,
+ country_code: event.country_code,
+ subdivision1_code: event.subdivision1_code,
+ subdivision2_code: event.subdivision2_code,
+ city_geoname_id: event.city_geoname_id,
+ screen_size: event.screen_size,
+ operating_system: event.operating_system,
+ operating_system_version: event.operating_system_version,
+ browser: event.browser,
+ browser_version: event.browser_version,
+ timestamp: event.timestamp,
+ start: event.timestamp,
+ "entry_meta.key": Map.get(event, :"meta.key"),
+ "entry_meta.value": Map.get(event, :"meta.value")
+ }
+ end
end
end
diff --git a/lib/plausible/session/write_buffer.ex b/lib/plausible/session/write_buffer.ex
index de23a6802c..598dc83933 100644
--- a/lib/plausible/session/write_buffer.ex
+++ b/lib/plausible/session/write_buffer.ex
@@ -69,7 +69,11 @@ defmodule Plausible.Session.WriteBuffer do
|> Enum.map(&(Map.from_struct(&1) |> Map.delete(:__meta__)))
|> Enum.reverse()
- IngestRepo.insert_all(Plausible.ClickhouseSession, sessions)
+ if Plausible.v2?() do
+ IngestRepo.insert_all(Plausible.ClickhouseSessionV2, sessions)
+ else
+ IngestRepo.insert_all(Plausible.ClickhouseSession, sessions)
+ end
end
end
diff --git a/lib/plausible/site/admin.ex b/lib/plausible/site/admin.ex
index c2f29053e0..032f196624 100644
--- a/lib/plausible/site/admin.ex
+++ b/lib/plausible/site/admin.ex
@@ -84,26 +84,32 @@ defmodule Plausible.SiteAdmin do
end
def transfer_data([from_site], params) do
- to_site = Repo.get_by(Plausible.Site, domain: params["domain"])
+ if Plausible.v2?() do
+ {:error, "Transfers temporarily unspported with v2"}
+ else
+ to_site = Repo.get_by(Plausible.Site, domain: params["domain"])
- if to_site do
- event_q = event_transfer_query(from_site.domain, to_site.domain)
- {:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, event_q, [], timeout: 30_000)
+ if to_site do
+ event_q = event_transfer_query(from_site.domain, to_site.domain)
+ {:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, event_q, [], timeout: 30_000)
- session_q = session_transfer_query(from_site.domain, to_site.domain)
- {:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, session_q, [], timeout: 30_000)
+ session_q = session_transfer_query(from_site.domain, to_site.domain)
- start_date = Plausible.Stats.Clickhouse.pageview_start_date_local(from_site)
+ {:ok, _} =
+ Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, session_q, [], timeout: 30_000)
- {:ok, _} =
- to_site
- |> Plausible.Site.set_stats_start_date(start_date)
- |> Plausible.Site.set_native_stats_start_at(from_site.native_stats_start_at)
- |> Repo.update()
+ start_date = Plausible.Stats.Clickhouse.pageview_start_date_local(from_site)
- :ok
- else
- {:error, "Cannot transfer to non-existing domain"}
+ {:ok, _} =
+ to_site
+ |> Plausible.Site.set_stats_start_date(start_date)
+ |> Plausible.Site.set_native_stats_start_at(from_site.native_stats_start_at)
+ |> Repo.update()
+
+ :ok
+ else
+ {:error, "Cannot transfer to non-existing domain"}
+ end
end
end
diff --git a/lib/plausible/site/gate_keeper.ex b/lib/plausible/site/gate_keeper.ex
index d0e4fe6efb..b7dcc65d1f 100644
--- a/lib/plausible/site/gate_keeper.ex
+++ b/lib/plausible/site/gate_keeper.ex
@@ -1,9 +1,9 @@
defmodule Plausible.Site.GateKeeper do
@type policy() :: :allow | :not_found | :block | :throttle
+ @type site_id() :: pos_integer()
@policy_for_non_existing_sites :not_found
- @policy_on_rate_limiting_backend_error :allow
- @type t() :: :allow | {:deny, policy()}
+ @type t() :: {:allow, site_id()} | {:deny, policy()}
@moduledoc """
Thin wrapper around Hammer for gate keeping domain-specific events
@@ -23,7 +23,7 @@ defmodule Plausible.Site.GateKeeper do
The module defines two policies outside the regular bucket inspection:
* when the the site is not found in cache: #{@policy_for_non_existing_sites}
* when the underlying rate limiting mechanism returns
- an internal error: #{@policy_on_rate_limiting_backend_error}
+ an internal error: :allow
"""
alias Plausible.Site
alias Plausible.Site.Cache
@@ -33,7 +33,7 @@ defmodule Plausible.Site.GateKeeper do
@spec check(String.t(), Keyword.t()) :: t()
def check(domain, opts \\ []) when is_binary(domain) do
case policy(domain, opts) do
- :allow -> :allow
+ {:allow, site_id} -> {:allow, site_id}
other -> {:deny, other}
end
end
@@ -56,15 +56,15 @@ defmodule Plausible.Site.GateKeeper do
result
end
- defp check_rate_limit(%Site{ingest_rate_limit_threshold: nil}, _opts) do
- :allow
+ defp check_rate_limit(%Site{id: site_id, ingest_rate_limit_threshold: nil}, _opts) do
+ {:allow, site_id}
end
defp check_rate_limit(%Site{ingest_rate_limit_threshold: 0}, _opts) do
:block
end
- defp check_rate_limit(%Site{ingest_rate_limit_threshold: threshold} = site, opts)
+ defp check_rate_limit(%Site{id: site_id, ingest_rate_limit_threshold: threshold} = site, opts)
when is_integer(threshold) do
key = Keyword.get(opts, :key, key(site.domain))
scale_ms = site.ingest_rate_limit_scale_seconds * 1_000
@@ -74,14 +74,14 @@ defmodule Plausible.Site.GateKeeper do
:throttle
{:allow, _} ->
- :allow
+ {:allow, site_id}
{:error, reason} ->
Logger.error(
- "Error checking rate limit for '#{key}': #{inspect(reason)}. Falling back to: #{@policy_on_rate_limiting_backend_error}"
+ "Error checking rate limit for '#{key}': #{inspect(reason)}. Falling back to: :allow"
)
- @policy_on_rate_limiting_backend_error
+ {:allow, site_id}
end
end
end
diff --git a/lib/plausible/sites.ex b/lib/plausible/sites.ex
index e791986089..d936f7b747 100644
--- a/lib/plausible/sites.ex
+++ b/lib/plausible/sites.ex
@@ -1,6 +1,5 @@
defmodule Plausible.Sites do
use Plausible.Repo
- alias Plausible.ClickhouseRepo
alias Plausible.Site
alias Plausible.Site.SharedLink
import Ecto.Query
@@ -77,11 +76,6 @@ defmodule Plausible.Sites do
!!stats_start_date(site)
end
- def has_events?(domain) do
- q = from e in "events", where: e.domain == ^domain, select: true, limit: 1
- ClickhouseRepo.one(q) == true
- end
-
def create_shared_link(site, name, password \\ nil) do
changes =
SharedLink.changeset(
@@ -158,6 +152,13 @@ defmodule Plausible.Sites do
|> Repo.all()
end
+ def owned_site_ids(user) do
+ user
+ |> owned_sites_query()
+ |> select([site], site.id)
+ |> Repo.all()
+ end
+
defp owned_sites_query(user) do
from(s in Site,
join: sm in Site.Membership,
diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex
index 225fc30294..7920ac8e7d 100644
--- a/lib/plausible/stats/base.ex
+++ b/lib/plausible/stats/base.ex
@@ -35,11 +35,19 @@ defmodule Plausible.Stats.Base do
{first_datetime, last_datetime} = utc_boundaries(query, site)
q =
- from(
- e in "events",
- where: e.domain == ^site.domain,
- where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
- )
+ if Plausible.v2?() do
+ from(
+ e in "events_v2",
+ where: e.site_id == ^site.id,
+ where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
+ )
+ else
+ from(
+ e in "events",
+ where: e.domain == ^site.domain,
+ where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
+ )
+ end
|> add_sample_hint(query)
q =
@@ -190,11 +198,19 @@ defmodule Plausible.Stats.Base do
{first_datetime, last_datetime} = utc_boundaries(query, site)
sessions_q =
- from(
- s in "sessions",
- where: s.domain == ^site.domain,
- where: s.start >= ^first_datetime and s.start < ^last_datetime
- )
+ if Plausible.v2?() do
+ from(
+ s in "sessions_v2",
+ where: s.site_id == ^site.id,
+ where: s.start >= ^first_datetime and s.start < ^last_datetime
+ )
+ else
+ from(
+ s in "sessions",
+ where: s.domain == ^site.domain,
+ where: s.start >= ^first_datetime and s.start < ^last_datetime
+ )
+ end
|> add_sample_hint(query)
|> filter_by_entry_props(query)
diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex
index 66f0ea07b8..f06b44e9cd 100644
--- a/lib/plausible/stats/breakdown.ex
+++ b/lib/plausible/stats/breakdown.ex
@@ -312,7 +312,7 @@ defmodule Plausible.Stats.Breakdown do
end
defp do_group_by(
- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events", _}}} = q,
+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,
"event:props:" <> prop
) do
q =
@@ -336,7 +336,7 @@ defmodule Plausible.Stats.Breakdown do
end
defp do_group_by(
- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events", _}}} = q,
+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,
"event:name"
) do
from(
@@ -348,7 +348,7 @@ defmodule Plausible.Stats.Breakdown do
end
defp do_group_by(
- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events", _}}} = q,
+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,
"event:page"
) do
from(
@@ -360,7 +360,7 @@ defmodule Plausible.Stats.Breakdown do
end
defp do_group_by(
- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events", _}}} = q,
+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,
"event:page_match"
) do
case Map.get(q, :__private_match_sources__) do
diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex
index a23b48d65f..5ce0ed4360 100644
--- a/lib/plausible/stats/clickhouse.ex
+++ b/lib/plausible/stats/clickhouse.ex
@@ -8,12 +8,23 @@ defmodule Plausible.Stats.Clickhouse do
@spec pageview_start_date_local(Plausible.Site.t()) :: Date.t() | nil
def pageview_start_date_local(site) do
datetime =
- ClickhouseRepo.one(
- from e in "events",
- select: fragment("min(?)", e.timestamp),
- where: e.domain == ^site.domain,
- where: e.timestamp >= ^site.native_stats_start_at
- )
+ if Plausible.v2?() do
+ ClickhouseRepo.one(
+ from(e in "events_v2",
+ select: fragment("min(?)", e.timestamp),
+ where: e.site_id == ^site.id,
+ where: e.timestamp >= ^site.native_stats_start_at
+ )
+ )
+ else
+ ClickhouseRepo.one(
+ from(e in "events",
+ select: fragment("min(?)", e.timestamp),
+ where: e.domain == ^site.domain,
+ where: e.timestamp >= ^site.native_stats_start_at
+ )
+ )
+ end
case datetime do
# no stats for this domain yet
@@ -29,28 +40,29 @@ defmodule Plausible.Stats.Clickhouse do
def imported_pageview_count(site) do
Plausible.ClickhouseRepo.one(
- from i in "imported_visitors",
+ from(i in "imported_visitors",
where: i.site_id == ^site.id,
select: sum(i.pageviews)
+ )
)
end
- def usage_breakdown(domains) do
+ def usage_breakdown(domains_or_site_ids) do
range =
Date.range(
Timex.shift(Timex.today(), days: -30),
Timex.today()
)
- usage_breakdown(domains, range)
+ usage_breakdown(domains_or_site_ids, range)
end
- def usage_breakdown(domains, date_range) do
+ def usage_breakdown([d | _] = domains, date_range) when is_binary(d) do
Enum.chunk_every(domains, 300)
|> Enum.reduce({0, 0}, fn domains, {pageviews_total, custom_events_total} ->
{chunk_pageviews, chunk_custom_events} =
ClickhouseRepo.one(
- from e in "events",
+ from(e in "events",
where: e.domain in ^domains,
where: fragment("toDate(?)", e.timestamp) >= ^date_range.first,
where: fragment("toDate(?)", e.timestamp) <= ^date_range.last,
@@ -58,12 +70,35 @@ defmodule Plausible.Stats.Clickhouse do
fragment("countIf(? = 'pageview')", e.name),
fragment("countIf(? != 'pageview')", e.name)
}
+ )
)
{pageviews_total + chunk_pageviews, custom_events_total + chunk_custom_events}
end)
end
+ def usage_breakdown([sid | _] = site_ids, date_range) when is_integer(sid) do
+ Enum.chunk_every(site_ids, 300)
+ |> Enum.reduce({0, 0}, fn site_ids, {pageviews_total, custom_events_total} ->
+ {chunk_pageviews, chunk_custom_events} =
+ ClickhouseRepo.one(
+ from(e in "events_v2",
+ where: e.site_id in ^site_ids,
+ where: fragment("toDate(?)", e.timestamp) >= ^date_range.first,
+ where: fragment("toDate(?)", e.timestamp) <= ^date_range.last,
+ select: {
+ fragment("countIf(? = 'pageview')", e.name),
+ fragment("countIf(? != 'pageview')", e.name)
+ }
+ )
+ )
+
+ {pageviews_total + chunk_pageviews, custom_events_total + chunk_custom_events}
+ end)
+ end
+
+ def usage_breakdown([], _date_range), do: {0, 0}
+
def top_sources(site, query, limit, page, show_noref \\ false, include_details) do
offset = (page - 1) * limit
@@ -151,47 +186,87 @@ defmodule Plausible.Stats.Clickhouse do
def current_visitors(site, query) do
Plausible.ClickhouseRepo.one(
- from e in base_query(site, query),
+ from(e in base_query(site, query),
select: uniq(e.user_id)
+ )
)
end
def has_pageviews?(site) do
- ClickhouseRepo.exists?(
- from(e in "events",
- where:
- e.domain == ^site.domain and
- e.name == "pageview" and
- e.timestamp >=
- ^site.native_stats_start_at
+ if Plausible.v2?() do
+ ClickhouseRepo.exists?(
+ from(e in "events_v2",
+ where:
+ e.site_id == ^site.id and
+ e.name == "pageview" and
+ e.timestamp >=
+ ^site.native_stats_start_at
+ )
)
- )
+ else
+ ClickhouseRepo.exists?(
+ from(e in "events",
+ where:
+ e.domain == ^site.domain and
+ e.name == "pageview" and
+ e.timestamp >=
+ ^site.native_stats_start_at
+ )
+ )
+ end
end
def last_24h_visitors([]), do: %{}
def last_24h_visitors(sites) do
- domains = Enum.map(sites, & &1.domain)
-
- ClickhouseRepo.all(
- from e in "events",
- group_by: e.domain,
- where: e.domain in ^domains,
- where: e.timestamp > fragment("now() - INTERVAL 24 HOUR"),
- select: {e.domain, fragment("uniq(user_id)")}
- )
- |> Enum.into(%{})
+ if Plausible.v2?() do
+ site_id_to_domain_mapping = for site <- sites, do: {site.id, site.domain}, into: %{}
+
+ ClickhouseRepo.all(
+ from(e in "events_v2",
+ group_by: e.site_id,
+ where: e.site_id in ^Map.keys(site_id_to_domain_mapping),
+ where: e.timestamp > fragment("now() - INTERVAL 24 HOUR"),
+ select: {e.site_id, fragment("uniq(user_id)")}
+ )
+ )
+ |> Enum.map(fn {site_id, user_id} ->
+ {site_id_to_domain_mapping[site_id], user_id}
+ end)
+ |> Enum.into(%{})
+ else
+ domains = Enum.map(sites, & &1.domain)
+
+ ClickhouseRepo.all(
+ from(e in "events",
+ group_by: e.domain,
+ where: e.domain in ^domains,
+ where: e.timestamp > fragment("now() - INTERVAL 24 HOUR"),
+ select: {e.domain, fragment("uniq(user_id)")}
+ )
+ )
+ |> Enum.into(%{})
+ end
end
+ # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity
defp base_session_query(site, query) do
{first_datetime, last_datetime} = utc_boundaries(query, site)
q =
- from(s in "sessions",
- hints: ["SAMPLE 10000000"],
- where: s.domain == ^site.domain,
- where: s.timestamp >= ^first_datetime and s.start < ^last_datetime
- )
+ if Plausible.v2?() do
+ from(s in "sessions_v2",
+ hints: ["SAMPLE 10000000"],
+ where: s.site_id == ^site.id,
+ where: s.timestamp >= ^first_datetime and s.start < ^last_datetime
+ )
+ else
+ from(s in "sessions",
+ hints: ["SAMPLE 10000000"],
+ where: s.domain == ^site.domain,
+ where: s.timestamp >= ^first_datetime and s.start < ^last_datetime
+ )
+ end
q =
if query.filters["source"] do
@@ -302,15 +377,24 @@ defmodule Plausible.Stats.Clickhouse do
end
end
+ # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity
defp base_query_bare(site, query) do
{first_datetime, last_datetime} = utc_boundaries(query, site)
q =
- from(e in "events",
- hints: ["SAMPLE 10000000"],
- where: e.domain == ^site.domain,
- where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
- )
+ if Plausible.v2?() do
+ from(e in "events_v2",
+ hints: ["SAMPLE 10000000"],
+ where: e.site_id == ^site.id,
+ where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
+ )
+ else
+ from(e in "events",
+ hints: ["SAMPLE 10000000"],
+ where: e.domain == ^site.domain,
+ where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
+ )
+ end
q =
if query.filters["screen"] do
diff --git a/lib/plausible/stats/current_visitors.ex b/lib/plausible/stats/current_visitors.ex
index c254df889e..ef36b47956 100644
--- a/lib/plausible/stats/current_visitors.ex
+++ b/lib/plausible/stats/current_visitors.ex
@@ -8,11 +8,20 @@ defmodule Plausible.Stats.CurrentVisitors do
|> Timex.shift(minutes: -5)
|> NaiveDateTime.truncate(:second)
- ClickhouseRepo.one(
- from e in "events",
- where: e.domain == ^site.domain,
- where: e.timestamp >= ^first_datetime,
- select: uniq(e.user_id)
- )
+ if Plausible.v2?() do
+ ClickhouseRepo.one(
+ from e in "events_v2",
+ where: e.site_id == ^site.id,
+ where: e.timestamp >= ^first_datetime,
+ select: uniq(e.user_id)
+ )
+ else
+ ClickhouseRepo.one(
+ from e in "events",
+ where: e.domain == ^site.domain,
+ where: e.timestamp >= ^first_datetime,
+ select: uniq(e.user_id)
+ )
+ end
end
end
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index 9a1172d930..386df3b8a9 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -1161,9 +1161,14 @@ defmodule PlausibleWeb.Api.StatsController do
site = conn.assigns[:site]
with :ok <- validate_params(params) do
- query = Query.from(site, params) |> Filters.add_prefix()
-
- json(conn, Stats.filter_suggestions(site, query, params["filter_name"], params["q"]))
+ query =
+ Query.from(site, params)
+ |> Filters.add_prefix()
+
+ json(
+ conn,
+ Stats.filter_suggestions(site, query, params["filter_name"], params["q"])
+ )
else
{:error, message} when is_binary(message) -> bad_request(conn, message)
end
diff --git a/mix.lock b/mix.lock
index ce9f06c677..4f1024c679 100644
--- a/mix.lock
+++ b/mix.lock
@@ -10,7 +10,7 @@
"cachex": {:hex, :cachex, "3.4.0", "868b2959ea4aeb328c6b60ff66c8d5123c083466ad3c33d3d8b5f142e13101fb", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "370123b1ab4fba4d2965fb18f87fd758325709787c8c5fce35b3fe80645ccbe5"},
"castore": {:hex, :castore, "0.1.22", "4127549e411bedd012ca3a308dede574f43819fe9394254ca55ab4895abfa1a2", [:mix], [], "hexpm", "c17576df47eb5aa1ee40cc4134316a99f5cad3e215d5c77b8dd3cfef12a22cac"},
"certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"},
- "ch": {:git, "https://github.com/ruslandoga/ch.git", "5894f81e246c049b9db5186497189cce226947aa", []},
+ "ch": {:git, "https://github.com/plausible/ch.git", "f411daa07c6310d3308f81397905df65330aeb64", []},
"chatterbox": {:hex, :ts_chatterbox, "0.13.0", "6f059d97bcaa758b8ea6fffe2b3b81362bd06b639d3ea2bb088335511d691ebf", [:rebar3], [{:hpack, "~> 0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "b93d19104d86af0b3f2566c4cba2a57d2e06d103728246ba1ac6c3c0ff010aa7"},
"chto": {:git, "https://github.com/plausible/chto.git", "316421ede947541bac32a853d70b0ca8ba003220", []},
"combination": {:hex, :combination, "0.0.3", "746aedca63d833293ec6e835aa1f34974868829b1486b1e1cb0685f0b2ae1f41", [:mix], [], "hexpm", "72b099f463df42ef7dc6371d250c7070b57b6c5902853f69deb894f79eda18ca"},
diff --git a/priv/data_migrations/NumericIDs/sql/list-partitions.sql.eex b/priv/data_migrations/NumericIDs/sql/list-partitions.sql.eex
index c945e73764..9522395a30 100644
--- a/priv/data_migrations/NumericIDs/sql/list-partitions.sql.eex
+++ b/priv/data_migrations/NumericIDs/sql/list-partitions.sql.eex
@@ -1,4 +1,4 @@
-SELECT partition
+SELECT distinct(partition)
FROM system.parts
WHERE table = 'events'
<%= if @start_from do %>AND partition >= '<%= @start_from %>'<% end %>
diff --git a/test/plausible/auth/auth_test.exs b/test/plausible/auth/auth_test.exs
index ab9058bf85..98c79a146f 100644
--- a/test/plausible/auth/auth_test.exs
+++ b/test/plausible/auth/auth_test.exs
@@ -18,7 +18,11 @@ defmodule Plausible.AuthTest do
test "is true if user does have events" do
user = insert(:user)
- insert(:site, members: [user], domain: "test-site.com")
+ site = insert(:site, members: [user])
+
+ populate_stats(site, [
+ build(:pageview)
+ ])
assert Auth.has_active_sites?(user)
end
diff --git a/test/plausible/billing/billing_test.exs b/test/plausible/billing/billing_test.exs
index 9ca1111f1d..c44567768b 100644
--- a/test/plausible/billing/billing_test.exs
+++ b/test/plausible/billing/billing_test.exs
@@ -133,10 +133,10 @@ defmodule Plausible.BillingTest do
site = insert(:site, members: [user])
create_pageviews([
- %{domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]},
- %{domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]},
- %{domain: site.domain, timestamp: ~N[2020-11-01 00:00:00]},
- %{domain: site.domain, timestamp: ~N[2020-10-31 00:00:00]}
+ %{site: site, timestamp: ~N[2021-01-01 00:00:00]},
+ %{site: site, timestamp: ~N[2020-12-31 00:00:00]},
+ %{site: site, timestamp: ~N[2020-11-01 00:00:00]},
+ %{site: site, timestamp: ~N[2020-10-31 00:00:00]}
])
assert Billing.last_two_billing_months_usage(user, today) == {1, 1}
@@ -163,10 +163,10 @@ defmodule Plausible.BillingTest do
)
create_pageviews([
- %{domain: owner_site.domain, timestamp: ~N[2020-12-31 00:00:00]},
- %{domain: admin_site.domain, timestamp: ~N[2020-12-31 00:00:00]},
- %{domain: owner_site.domain, timestamp: ~N[2020-11-01 00:00:00]},
- %{domain: admin_site.domain, timestamp: ~N[2020-11-01 00:00:00]}
+ %{site: owner_site, timestamp: ~N[2020-12-31 00:00:00]},
+ %{site: admin_site, timestamp: ~N[2020-12-31 00:00:00]},
+ %{site: owner_site, timestamp: ~N[2020-11-01 00:00:00]},
+ %{site: admin_site, timestamp: ~N[2020-11-01 00:00:00]}
])
assert Billing.last_two_billing_months_usage(user, today) == {1, 1}
diff --git a/test/plausible/session/cache_store_test.exs b/test/plausible/session/cache_store_test.exs
index 6f14c52064..265d17df48 100644
--- a/test/plausible/session/cache_store_test.exs
+++ b/test/plausible/session/cache_store_test.exs
@@ -37,7 +37,13 @@ defmodule Plausible.Session.CacheStoreTest do
assert_receive({WriteBuffer, :insert, [sessions]})
assert [session] = sessions
assert session.hostname == event.hostname
- assert session.domain == event.domain
+
+ if Plausible.v2?() do
+ assert session.site_id == event.site_id
+ else
+ assert session.domain == event.domain
+ end
+
assert session.user_id == event.user_id
assert session.entry_page == event.pathname
assert session.exit_page == event.pathname
@@ -68,12 +74,9 @@ defmodule Plausible.Session.CacheStoreTest do
timestamp = Timex.now()
event1 = build(:event, name: "pageview", timestamp: timestamp |> Timex.shift(seconds: -10))
- event2 =
- build(:event,
- domain: event1.domain,
- user_id: event1.user_id,
- name: "pageview",
- timestamp: timestamp,
+ event2 = %{
+ event1
+ | timestamp: timestamp,
country_code: "US",
subdivision1_code: "SUB1",
subdivision2_code: "SUB2",
@@ -83,7 +86,7 @@ defmodule Plausible.Session.CacheStoreTest do
operating_system_version: "11",
browser: "Firefox",
browser_version: "10"
- )
+ }
CacheStore.on_event(event1, nil, buffer)
CacheStore.on_event(event2, nil, buffer)
@@ -107,13 +110,7 @@ defmodule Plausible.Session.CacheStoreTest do
timestamp = Timex.now()
event1 = build(:event, name: "pageview", timestamp: timestamp |> Timex.shift(seconds: 10))
- event2 =
- build(:event,
- domain: event1.domain,
- user_id: event1.user_id,
- name: "pageview",
- timestamp: timestamp
- )
+ event2 = %{event1 | timestamp: timestamp}
CacheStore.on_event(event1, nil, buffer)
CacheStore.on_event(event2, nil, buffer)
diff --git a/test/plausible/site/admin_test.exs b/test/plausible/site/admin_test.exs
index 9a34906890..99d5429c67 100644
--- a/test/plausible/site/admin_test.exs
+++ b/test/plausible/site/admin_test.exs
@@ -1,76 +1,98 @@
defmodule Plausible.SiteAdminTest do
use Plausible.DataCase, async: true
- alias Plausible.{SiteAdmin, ClickhouseRepo, ClickhouseEvent, ClickhouseSession}
+ alias Plausible.{
+ SiteAdmin,
+ ClickhouseRepo,
+ ClickhouseEvent,
+ ClickhouseSession
+ }
describe "transfer_data" do
test "event and session structs remain the same after transfer" do
- from_site = insert(:site)
- to_site = insert(:site)
+ if Plausible.v2?() do
+ :ok
+ else
+ from_site = insert(:site)
+ to_site = insert(:site)
- populate_stats(from_site, [build(:pageview)])
+ populate_stats(from_site, [build(:pageview)])
- event_before = get_event_by_domain(from_site.domain)
- session_before = get_session_by_domain(from_site.domain)
+ event_before = get_event_by_domain(from_site.domain)
+ session_before = get_session_by_domain(from_site.domain)
- SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
+ SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
- event_after = get_event_by_domain(to_site.domain)
- session_after = get_session_by_domain(to_site.domain)
+ event_after = get_event_by_domain(to_site.domain)
+ session_after = get_session_by_domain(to_site.domain)
- assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}
- assert session_before == %ClickhouseSession{session_after | transferred_from: ""}
- assert event_after.transferred_from == from_site.domain
- assert session_after.transferred_from == from_site.domain
+ assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}
+ assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}
+ assert session_before == %ClickhouseSession{session_after | transferred_from: ""}
+ assert event_after.transferred_from == from_site.domain
+ assert session_after.transferred_from == from_site.domain
+ end
end
test "transfers all events and sessions" do
- from_site = insert(:site)
- to_site = insert(:site)
-
- populate_stats(from_site, [
- build(:pageview, user_id: 123),
- build(:event, name: "Signup", user_id: 123),
- build(:pageview, user_id: 456),
- build(:event, name: "Signup", user_id: 789)
- ])
-
- SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
-
- transferred_events =
- ClickhouseRepo.all(
- from e in Plausible.ClickhouseEvent, where: e.domain == ^to_site.domain
- )
-
- transferred_sessions =
- ClickhouseRepo.all(
- from e in Plausible.ClickhouseSession, where: e.domain == ^to_site.domain
- )
-
- assert length(transferred_events) == 4
- assert length(transferred_sessions) == 3
+ if Plausible.v2?() do
+ :ok
+ else
+ from_site = insert(:site)
+ to_site = insert(:site)
+
+ populate_stats(from_site, [
+ build(:pageview, user_id: 123),
+ build(:event, name: "Signup", user_id: 123),
+ build(:pageview, user_id: 456),
+ build(:event, name: "Signup", user_id: 789)
+ ])
+
+ SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
+
+ transferred_events =
+ ClickhouseRepo.all(
+ from e in Plausible.ClickhouseEvent, where: e.domain == ^to_site.domain
+ )
+
+ transferred_sessions =
+ ClickhouseRepo.all(
+ from e in Plausible.ClickhouseSession, where: e.domain == ^to_site.domain
+ )
+
+ assert length(transferred_events) == 4
+ assert length(transferred_sessions) == 3
+ end
end
test "updates stats_start_date on site record" do
- from_site = insert(:site)
- to_site = insert(:site)
+ if Plausible.v2?() do
+ :ok
+ else
+ from_site = insert(:site)
+ to_site = insert(:site)
- populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])])
+ populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])])
- SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
+ SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
- assert Repo.reload(to_site).stats_start_date == ~D[2022-01-01]
+ assert Repo.reload(to_site).stats_start_date == ~D[2022-01-01]
+ end
end
test "updates native_stats_start_date on based on the from_site record" do
- from_site = insert(:site, native_stats_start_at: ~N[2022-01-01 01:00:00])
- to_site = insert(:site)
+ if Plausible.v2?() do
+ :ok
+ else
+ from_site = insert(:site, native_stats_start_at: ~N[2022-01-01 01:00:00])
+ to_site = insert(:site)
- populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])])
+ populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])])
- SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
+ SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
- assert Repo.reload(to_site).native_stats_start_at == ~N[2022-01-01 01:00:00]
+ assert Repo.reload(to_site).native_stats_start_at == ~N[2022-01-01 01:00:00]
+ end
end
test "session_transfer_query" do
diff --git a/test/plausible/site/gate_keeper_test.exs b/test/plausible/site/gate_keeper_test.exs
index e67b229bf5..44a1233a37 100644
--- a/test/plausible/site/gate_keeper_test.exs
+++ b/test/plausible/site/gate_keeper_test.exs
@@ -19,20 +19,21 @@ defmodule Plausible.Site.GateKeeperTest do
test "site from cache with no ingest_rate_limit_threshold is allowed", %{test: test, opts: opts} do
domain = "site1.example.com"
- add_site_and_refresh_cache(test, domain: domain)
- assert :allow = GateKeeper.check(domain, opts)
+ %{id: site_id} = add_site_and_refresh_cache(test, domain: domain)
+ assert {:allow, ^site_id} = GateKeeper.check(domain, opts)
end
test "rate limiting works with threshold", %{test: test, opts: opts} do
domain = "site1.example.com"
- add_site_and_refresh_cache(test,
- domain: domain,
- ingest_rate_limit_threshold: 1,
- ingest_rate_limit_scale_seconds: 60
- )
+ %{id: site_id} =
+ add_site_and_refresh_cache(test,
+ domain: domain,
+ ingest_rate_limit_threshold: 1,
+ ingest_rate_limit_scale_seconds: 60
+ )
- assert :allow = GateKeeper.check(domain, opts)
+ assert {:allow, ^site_id} = GateKeeper.check(domain, opts)
assert {:deny, :throttle} = GateKeeper.check(domain, opts)
assert {:deny, :throttle} = GateKeeper.check(domain, opts)
end
@@ -40,17 +41,18 @@ defmodule Plausible.Site.GateKeeperTest do
test "rate limiting works with scale window", %{test: test, opts: opts} do
domain = "site1.example.com"
- add_site_and_refresh_cache(test,
- domain: domain,
- ingest_rate_limit_threshold: 1,
- ingest_rate_limit_scale_seconds: 1
- )
+ %{id: site_id} =
+ add_site_and_refresh_cache(test,
+ domain: domain,
+ ingest_rate_limit_threshold: 1,
+ ingest_rate_limit_scale_seconds: 1
+ )
- assert :allow = GateKeeper.check(domain, opts)
+ assert {:allow, ^site_id} = GateKeeper.check(domain, opts)
Process.sleep(1)
assert {:deny, :throttle} = GateKeeper.check(domain, opts)
Process.sleep(1_000)
- assert :allow = GateKeeper.check(domain, opts)
+ assert {:allow, ^site_id} = GateKeeper.check(domain, opts)
end
test "rate limiting prioritises cache lookups", %{test: test, opts: opts} do
@@ -67,8 +69,9 @@ defmodule Plausible.Site.GateKeeperTest do
# We need some dummy site, otherwise the cache won't refresh in case the DB
# is completely empty
insert(:site)
+ deleted_site_id = site.id
- assert :allow = GateKeeper.check(domain, opts)
+ assert {:allow, ^deleted_site_id} = GateKeeper.check(domain, opts)
:ok = Cache.refresh_all(opts[:cache_opts])
assert {:deny, :not_found} = GateKeeper.check(domain, opts)
end
@@ -86,18 +89,19 @@ defmodule Plausible.Site.GateKeeperTest do
ingest_rate_limit_scale_seconds: 600
)
- assert :allow = GateKeeper.check(domain, opts)
+ site_id = site.id
+ assert {:allow, ^site_id} = GateKeeper.check(domain, opts)
assert {:deny, :throttle} = GateKeeper.check(domain, opts)
{:ok, :broken} = break_hammer(site)
log =
capture_log(fn ->
- assert :allow = GateKeeper.check(domain, opts)
+ assert {:allow, ^site_id} = GateKeeper.check(domain, opts)
end)
assert log =~ "Error checking rate limit for 'ingest:site:causingerrors.example.com'"
- assert log =~ "Falling back to: allow"
+ assert log =~ "Falling back to: :allow"
end
# We need a way to force Hammer to error-out on Hammer.check_rate/3.
diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exs
index 0c104954ba..547bc5911d 100644
--- a/test/plausible_web/controllers/api/external_controller_test.exs
+++ b/test/plausible_web/controllers/api/external_controller_test.exs
@@ -8,13 +8,13 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
describe "POST /api/event" do
setup do
- site1 = insert(:site)
- {:ok, domain: site1.domain}
+ site = insert(:site)
+ {:ok, site: site}
end
- test "records the event", %{conn: conn, domain: domain} do
+ test "records the event", %{conn: conn, site: site} do
params = %{
- domain: domain,
+ domain: site.domain,
name: "pageview",
url: "http://gigride.live/",
referrer: "http://m.facebook.com/"
@@ -25,17 +25,23 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("user-agent", @user_agent)
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert response(conn, 202) == "ok"
assert pageview.hostname == "gigride.live"
- assert pageview.domain == domain
+
+ if Plausible.v2?() do
+ assert pageview.site_id == site.id
+ else
+ assert pageview.domain == site.domain
+ end
+
assert pageview.pathname == "/"
end
- test "works with Content-Type: text/plain", %{conn: conn, domain: domain} do
+ test "works with Content-Type: text/plain", %{conn: conn, site: site} do
params = %{
- domain: domain,
+ domain: site.domain,
name: "pageview",
url: "http://gigride.live/"
}
@@ -45,11 +51,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("content-type", "text/plain")
|> post("/api/event", Jason.encode!(params))
- pageview = get_event(domain)
+ pageview = get_event(site)
assert response(conn, 202) == "ok"
assert pageview.hostname == "gigride.live"
- assert pageview.domain == domain
+
+ if Plausible.v2?() do
+ assert pageview.site_id == site.id
+ else
+ assert pageview.domain == site.domain
+ end
+
assert pageview.pathname == "/"
end
@@ -64,15 +76,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
test "can send to multiple dashboards by listing multiple domains - same timestamp", %{
conn: conn,
- domain: domain1
+ site: site1
} do
- domain2 = insert(:site).domain
+ site2 = insert(:site)
params = %{
name: "pageview",
url: "http://gigride.live/",
referrer: "http://m.facebook.com/",
- domain: "#{domain1},#{domain2}"
+ domain: "#{site1.domain},#{site2.domain}"
}
conn =
@@ -81,15 +93,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> post("/api/event", params)
assert response(conn, 202) == "ok"
- assert e1 = get_event(domain1)
- assert e2 = get_event(domain2)
+ assert e1 = get_event(site1)
+ assert e2 = get_event(site2)
assert NaiveDateTime.compare(e1.timestamp, e2.timestamp) == :eq
end
- test "timestamps differ when two events sent in a row", %{conn: conn, domain: domain} do
+ test "timestamps differ when two events sent in a row", %{conn: conn, site: site} do
params = %{
- domain: domain,
+ domain: site.domain,
name: "pageview",
url: "http://gigride.live/",
referrer: "http://m.facebook.com/"
@@ -110,90 +122,94 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("user-agent", @user_agent)
|> post("/api/event", params)
- [e1, e2] = get_events(domain)
+ [e1, e2] = get_events(site)
assert NaiveDateTime.compare(e1.timestamp, e2.timestamp) == :gt
end
- test "www. is stripped from domain", %{conn: conn, domain: domain} do
+ test "www. is stripped from domain", %{conn: conn, site: site} do
params = %{
name: "custom event",
url: "http://gigride.live/",
- domain: domain
+ domain: site.domain
}
conn
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
- assert pageview.domain == domain
+ if Plausible.v2?() do
+ assert pageview.site_id == site.id
+ else
+ assert pageview.domain == site.domain
+ end
end
- test "www. is stripped from hostname", %{conn: conn, domain: domain} do
+ test "www. is stripped from hostname", %{conn: conn, site: site} do
params = %{
name: "pageview",
url: "http://www.example.com/",
- domain: domain
+ domain: site.domain
}
conn
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert pageview.hostname == "example.com"
end
- test "empty path defaults to /", %{conn: conn, domain: domain} do
+ test "empty path defaults to /", %{conn: conn, site: site} do
params = %{
name: "pageview",
url: "http://www.example.com",
- domain: domain
+ domain: site.domain
}
conn
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert pageview.pathname == "/"
end
- test "trailing whitespace is removed", %{conn: conn, domain: domain} do
+ test "trailing whitespace is removed", %{conn: conn, site: site} do
params = %{
name: "pageview",
url: "http://www.example.com/path ",
- domain: domain
+ domain: site.domain
}
conn
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert pageview.pathname == "/path"
end
- test "bots and crawlers are ignored", %{conn: conn, domain: domain} do
+ test "bots and crawlers are ignored", %{conn: conn, site: site} do
params = %{
name: "pageview",
url: "http://www.example.com/",
- domain: domain
+ domain: site.domain
}
conn
|> put_req_header("user-agent", "generic crawler")
|> post("/api/event", params)
- assert get_event("external-controller-test-5.com") == nil
+ assert get_event(site) == nil
end
- test "Headless Chrome is ignored", %{conn: conn, domain: domain} do
+ test "Headless Chrome is ignored", %{conn: conn, site: site} do
params = %{
name: "pageview",
url: "http://www.example.com/",
- domain: domain
+ domain: site.domain
}
conn
@@ -203,14 +219,14 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
)
|> post("/api/event", params)
- assert get_event(domain) == nil
+ assert get_event(site) == nil
end
- test "parses user_agent", %{conn: conn, domain: domain} do
+ test "parses user_agent", %{conn: conn, site: site} do
params = %{
name: "pageview",
url: "http://gigride.live/",
- domain: domain
+ domain: site.domain
}
conn =
@@ -218,7 +234,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("user-agent", @user_agent)
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert response(conn, 202) == "ok"
assert pageview.operating_system == "Mac"
@@ -227,12 +243,12 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert pageview.browser_version == "70.0"
end
- test "parses referrer", %{conn: conn, domain: domain} do
+ test "parses referrer", %{conn: conn, site: site} do
params = %{
name: "pageview",
url: "http://gigride.live/",
referrer: "https://facebook.com",
- domain: domain
+ domain: site.domain
}
conn =
@@ -240,18 +256,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("user-agent", @user_agent)
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert response(conn, 202) == "ok"
assert pageview.referrer_source == "Facebook"
end
- test "strips trailing slash from referrer", %{conn: conn, domain: domain} do
+ test "strips trailing slash from referrer", %{conn: conn, site: site} do
params = %{
name: "pageview",
url: "http://gigride.live/",
referrer: "https://facebook.com/page/",
- domain: domain
+ domain: site.domain
}
conn =
@@ -259,16 +275,16 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("user-agent", @user_agent)
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert response(conn, 202) == "ok"
assert pageview.referrer == "facebook.com/page"
assert pageview.referrer_source == "Facebook"
end
- test "ignores event when referrer is a spammer", %{conn: conn, domain: domain} do
+ test "ignores event when referrer is a spammer", %{conn: conn, site: site} do
params = %{
- domain: domain,
+ domain: site.domain,
name: "pageview",
url: "http://gigride.live/",
referrer: "https://www.1-best-seo.com"
@@ -280,7 +296,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> post("/api/event", params)
assert response(conn, 202) == "ok"
- assert !get_event(domain)
+ assert !get_event(site)
end
test "blocks traffic from a domain when it's blocked", %{
@@ -300,15 +316,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> post("/api/event", params)
assert response(conn, 202) == "ok"
- refute get_event(site.domain)
+ refute get_event(site)
end
- test "ignores when referrer is internal", %{conn: conn, domain: domain} do
+ test "ignores when referrer is internal", %{conn: conn, site: site} do
params = %{
name: "pageview",
url: "http://gigride.live/",
referrer: "https://gigride.live",
- domain: domain
+ domain: site.domain
}
conn =
@@ -316,18 +332,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("user-agent", @user_agent)
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert response(conn, 202) == "ok"
assert pageview.referrer_source == ""
end
- test "ignores localhost referrer", %{conn: conn, domain: domain} do
+ test "ignores localhost referrer", %{conn: conn, site: site} do
params = %{
name: "pageview",
url: "http://gigride.live/",
referrer: "http://localhost:4000/",
- domain: domain
+ domain: site.domain
}
conn =
@@ -335,18 +351,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("user-agent", @user_agent)
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert response(conn, 202) == "ok"
assert pageview.referrer_source == ""
end
- test "parses subdomain referrer", %{conn: conn, domain: domain} do
+ test "parses subdomain referrer", %{conn: conn, site: site} do
params = %{
name: "pageview",
url: "http://gigride.live/",
referrer: "https://blog.gigride.live",
- domain: domain
+ domain: site.domain
}
conn =
@@ -354,68 +370,68 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("user-agent", @user_agent)
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert response(conn, 202) == "ok"
assert pageview.referrer_source == "blog.gigride.live"
end
- test "referrer is cleaned", %{conn: conn, domain: domain} do
+ test "referrer is cleaned", %{conn: conn, site: site} do
params = %{
name: "pageview",
url: "http://www.example.com/",
referrer: "https://www.indiehackers.com/page?query=param#hash",
- domain: domain
+ domain: site.domain
}
conn
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert pageview.referrer == "indiehackers.com/page"
end
- test "utm_source overrides referrer source", %{conn: conn, domain: domain} do
+ test "utm_source overrides referrer source", %{conn: conn, site: site} do
params = %{
name: "pageview",
url: "http://www.example.com/?utm_source=betalist",
referrer: "https://betalist.com/my-produxct",
- domain: domain
+ domain: site.domain
}
conn
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert pageview.referrer_source == "betalist"
end
- test "utm tags are stored", %{conn: conn, domain: domain} do
+ test "utm tags are stored", %{conn: conn, site: site} do
params = %{
name: "pageview",
url:
"http://www.example.com/?utm_medium=ads&utm_source=instagram&utm_campaign=video_story",
- domain: domain
+ domain: site.domain
}
conn
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert pageview.utm_medium == "ads"
assert pageview.utm_source == "instagram"
assert pageview.utm_campaign == "video_story"
end
- test "if it's an :unknown referrer, just the domain is used", %{conn: conn, domain: domain} do
+ test "if it's an :unknown referrer, just the domain is used", %{conn: conn, site: site} do
params = %{
name: "pageview",
url: "http://gigride.live/",
referrer: "https://www.indiehackers.com/landing-page-feedback",
- domain: domain
+ domain: site.domain
}
conn =
@@ -423,18 +439,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("user-agent", @user_agent)
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert response(conn, 202) == "ok"
assert pageview.referrer_source == "indiehackers.com"
end
- test "if the referrer is not http or https, it is ignored", %{conn: conn, domain: domain} do
+ test "if the referrer is not http or https, it is ignored", %{conn: conn, site: site} do
params = %{
name: "pageview",
url: "http://gigride.live/",
referrer: "android-app://com.google.android.gm",
- domain: domain
+ domain: site.domain
}
conn =
@@ -442,17 +458,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("user-agent", @user_agent)
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert response(conn, 202) == "ok"
assert pageview.referrer_source == ""
end
- test "screen size is calculated from user agent", %{conn: conn, domain: domain} do
+ test "screen size is calculated from user agent", %{conn: conn, site: site} do
params = %{
name: "pageview",
url: "http://gigride.live/",
- domain: domain
+ domain: site.domain
}
conn =
@@ -460,17 +476,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("user-agent", @user_agent_mobile)
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert response(conn, 202) == "ok"
assert pageview.screen_size == "Mobile"
end
- test "screen size is nil if user agent is unknown", %{conn: conn, domain: domain} do
+ test "screen size is nil if user agent is unknown", %{conn: conn, site: site} do
params = %{
name: "pageview",
url: "http://gigride.live/",
- domain: domain
+ domain: site.domain
}
conn =
@@ -478,17 +494,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("user-agent", "unknown UA")
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert response(conn, 202) == "ok"
assert pageview.screen_size == ""
end
- test "screen size is calculated from user_agent when is tablet", %{conn: conn, domain: domain} do
+ test "screen size is calculated from user_agent when is tablet", %{conn: conn, site: site} do
params = %{
name: "pageview",
url: "http://gigride.live/",
- domain: domain
+ domain: site.domain
}
conn =
@@ -496,7 +512,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("user-agent", @user_agent_tablet)
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert response(conn, 202) == "ok"
assert pageview.screen_size == "Tablet"
@@ -504,12 +520,12 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
test "screen size is calculated from user_agent when is desktop", %{
conn: conn,
- domain: domain
+ site: site
} do
params = %{
name: "pageview",
url: "http://gigride.live/",
- domain: domain
+ domain: site.domain
}
conn =
@@ -517,17 +533,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("user-agent", @user_agent)
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert response(conn, 202) == "ok"
assert pageview.screen_size == "Desktop"
end
- test "can trigger a custom event", %{conn: conn, domain: domain} do
+ test "can trigger a custom event", %{conn: conn, site: site} do
params = %{
name: "custom event",
url: "http://gigride.live/",
- domain: domain
+ domain: site.domain
}
conn =
@@ -535,17 +551,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("user-agent", @user_agent)
|> post("/api/event", params)
- event = get_event(domain)
+ event = get_event(site)
assert response(conn, 202) == "ok"
assert event.name == "custom event"
end
- test "casts custom props to string", %{conn: conn, domain: domain} do
+ test "casts custom props to string", %{conn: conn, site: site} do
params = %{
name: "Signup",
url: "http://gigride.live/",
- domain: domain,
+ domain: site.domain,
props: %{
bool_test: true,
number_test: 12
@@ -555,51 +571,51 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn
|> post("/api/event", params)
- event = get_event(domain)
+ event = get_event(site)
assert Map.get(event, :"meta.key") == ["bool_test", "number_test"]
assert Map.get(event, :"meta.value") == ["true", "12"]
end
- test "ignores malformed custom props", %{conn: conn, domain: domain} do
+ test "ignores malformed custom props", %{conn: conn, site: site} do
params = %{
name: "Signup",
url: "http://gigride.live/",
- domain: domain,
+ domain: site.domain,
props: "\"show-more:button\""
}
conn
|> post("/api/event", params)
- event = get_event(domain)
+ event = get_event(site)
assert Map.get(event, :"meta.key") == []
assert Map.get(event, :"meta.value") == []
end
- test "can send props stringified", %{conn: conn, domain: domain} do
+ test "can send props stringified", %{conn: conn, site: site} do
params = %{
name: "Signup",
url: "http://gigride.live/",
- domain: domain,
+ domain: site.domain,
props: Jason.encode!(%{number_test: 12})
}
conn
|> post("/api/event", params)
- event = get_event(domain)
+ event = get_event(site)
assert Map.get(event, :"meta.key") == ["number_test"]
assert Map.get(event, :"meta.value") == ["12"]
end
- test "ignores custom prop with array value", %{conn: conn, domain: domain} do
+ test "ignores custom prop with array value", %{conn: conn, site: site} do
params = %{
name: "Signup",
url: "http://gigride.live/",
- domain: domain,
+ domain: site.domain,
props: Jason.encode!(%{wat: ["some-thing"], other: "key"})
}
@@ -607,17 +623,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert conn.status == 202
- event = get_event(domain)
+ event = get_event(site)
assert Map.get(event, :"meta.key") == ["other"]
assert Map.get(event, :"meta.value") == ["key"]
end
- test "ignores custom prop with map value", %{conn: conn, domain: domain} do
+ test "ignores custom prop with map value", %{conn: conn, site: site} do
params = %{
name: "Signup",
url: "http://gigride.live/",
- domain: domain,
+ domain: site.domain,
props: Jason.encode!(%{foo: %{bar: "baz"}, other_key: 1})
}
@@ -625,17 +641,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert conn.status == 202
- event = get_event(domain)
+ event = get_event(site)
assert Map.get(event, :"meta.key") == ["other_key"]
assert Map.get(event, :"meta.value") == ["1"]
end
- test "ignores custom prop with empty string value", %{conn: conn, domain: domain} do
+ test "ignores custom prop with empty string value", %{conn: conn, site: site} do
params = %{
name: "Signup",
url: "http://gigride.live/",
- domain: domain,
+ domain: site.domain,
props: Jason.encode!(%{foo: "", other_key: true})
}
@@ -643,17 +659,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert conn.status == 202
- event = get_event(domain)
+ event = get_event(site)
assert Map.get(event, :"meta.key") == ["other_key"]
assert Map.get(event, :"meta.value") == ["true"]
end
- test "ignores custom prop with nil value", %{conn: conn, domain: domain} do
+ test "ignores custom prop with nil value", %{conn: conn, site: site} do
params = %{
name: "Signup",
url: "http://gigride.live/",
- domain: domain,
+ domain: site.domain,
props: Jason.encode!(%{foo: nil, other_key: true})
}
@@ -661,18 +677,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert conn.status == 202
- event = get_event(domain)
+ event = get_event(site)
assert Map.get(event, :"meta.key") == ["other_key"]
assert Map.get(event, :"meta.value") == ["true"]
end
- test "ignores a malformed referrer URL", %{conn: conn, domain: domain} do
+ test "ignores a malformed referrer URL", %{conn: conn, site: site} do
params = %{
name: "pageview",
url: "http://gigride.live/",
referrer: "https:://twitter.com",
- domain: domain
+ domain: site.domain
}
conn =
@@ -680,17 +696,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("user-agent", @user_agent)
|> post("/api/event", params)
- event = get_event(domain)
+ event = get_event(site)
assert response(conn, 202) == "ok"
assert event.referrer == ""
end
# Fake geo is loaded from test/priv/GeoLite2-City-Test.mmdb
- test "looks up location data from the ip address", %{conn: conn, domain: domain} do
+ test "looks up location data from the ip address", %{conn: conn, site: site} do
params = %{
name: "pageview",
- domain: domain,
+ domain: site.domain,
url: "http://gigride.live/"
}
@@ -698,7 +714,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("x-forwarded-for", "2.125.160.216")
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert pageview.country_code == "GB"
assert pageview.subdivision1_code == "GB-ENG"
@@ -706,10 +722,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert pageview.city_geoname_id == 2_655_045
end
- test "ignores unknown country code ZZ", %{conn: conn, domain: domain} do
+ test "ignores unknown country code ZZ", %{conn: conn, site: site} do
params = %{
name: "pageview",
- domain: domain,
+ domain: site.domain,
url: "http://gigride.live/"
}
@@ -717,7 +733,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("x-forwarded-for", "0.0.0.0")
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert pageview.country_code == <<0, 0>>
assert pageview.subdivision1_code == ""
@@ -725,10 +741,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert pageview.city_geoname_id == 0
end
- test "ignores disputed territory code XX", %{conn: conn, domain: domain} do
+ test "ignores disputed territory code XX", %{conn: conn, site: site} do
params = %{
name: "pageview",
- domain: domain,
+ domain: site.domain,
url: "http://gigride.live/"
}
@@ -736,7 +752,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("x-forwarded-for", "0.0.0.1")
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert pageview.country_code == <<0, 0>>
assert pageview.subdivision1_code == ""
@@ -744,10 +760,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert pageview.city_geoname_id == 0
end
- test "ignores TOR exit node country code T1", %{conn: conn, domain: domain} do
+ test "ignores TOR exit node country code T1", %{conn: conn, site: site} do
params = %{
name: "pageview",
- domain: domain,
+ domain: site.domain,
url: "http://gigride.live/"
}
@@ -755,7 +771,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("x-forwarded-for", "0.0.0.2")
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert pageview.country_code == <<0, 0>>
assert pageview.subdivision1_code == ""
@@ -763,10 +779,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert pageview.city_geoname_id == 0
end
- test "scrubs port from x-forwarded-for", %{conn: conn, domain: domain} do
+ test "scrubs port from x-forwarded-for", %{conn: conn, site: site} do
params = %{
name: "pageview",
- domain: domain,
+ domain: site.domain,
url: "http://gigride.live/"
}
@@ -774,15 +790,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("x-forwarded-for", "216.160.83.56:123")
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert pageview.country_code == "US"
end
- test "works with ipv6 without port in x-forwarded-for", %{conn: conn, domain: domain} do
+ test "works with ipv6 without port in x-forwarded-for", %{conn: conn, site: site} do
params = %{
name: "pageview",
- domain: domain,
+ domain: site.domain,
url: "http://gigride.live/"
}
@@ -790,15 +806,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("x-forwarded-for", "2001:218:1:1:1:1:1:1")
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert pageview.country_code == "JP"
end
- test "works with ipv6 with a port number in x-forwarded-for", %{conn: conn, domain: domain} do
+ test "works with ipv6 with a port number in x-forwarded-for", %{conn: conn, site: site} do
params = %{
name: "pageview",
- domain: domain,
+ domain: site.domain,
url: "http://gigride.live/"
}
@@ -806,18 +822,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("x-forwarded-for", "[2001:218:1:1:1:1:1:1]:123")
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert pageview.country_code == "JP"
end
test "uses cloudflare's special header for client IP address if present", %{
conn: conn,
- domain: domain
+ site: site
} do
params = %{
name: "pageview",
- domain: domain,
+ domain: site.domain,
url: "http://gigride.live/"
}
@@ -826,18 +842,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("cf-connecting-ip", "216.160.83.56")
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert pageview.country_code == "US"
end
test "uses BunnyCDN's custom header for client IP address if present", %{
conn: conn,
- domain: domain
+ site: site
} do
params = %{
name: "pageview",
- domain: domain,
+ domain: site.domain,
url: "http://gigride.live/"
}
@@ -846,18 +862,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("b-forwarded-for", "216.160.83.56,9.9.9.9")
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert pageview.country_code == "US"
end
test "Uses the Forwarded header when cf-connecting-ip and x-forwarded-for are missing", %{
conn: conn,
- domain: domain
+ site: site
} do
params = %{
name: "pageview",
- domain: domain,
+ domain: site.domain,
url: "http://gigride.live/"
}
@@ -865,15 +881,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("forwarded", "by=0.0.0.0;for=216.160.83.56;host=somehost.com;proto=https")
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert pageview.country_code == "US"
end
- test "Forwarded header can parse ipv6", %{conn: conn, domain: domain} do
+ test "Forwarded header can parse ipv6", %{conn: conn, site: site} do
params = %{
name: "pageview",
- domain: domain,
+ domain: site.domain,
url: "http://gigride.live/"
}
@@ -884,117 +900,117 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
)
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert pageview.country_code == "JP"
end
- test "URL is decoded", %{conn: conn, domain: domain} do
+ test "URL is decoded", %{conn: conn, site: site} do
params = %{
name: "pageview",
url:
"http://www.example.com/opportunity/category/%D8%AC%D9%88%D8%A7%D8%A6%D8%B2-%D9%88%D9%85%D8%B3%D8%A7%D8%A8%D9%82%D8%A7%D8%AA",
- domain: domain
+ domain: site.domain
}
conn
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert pageview.pathname == "/opportunity/category/جوائز-ومسابقات"
end
- test "accepts shorthand map keys", %{conn: conn, domain: domain} do
+ test "accepts shorthand map keys", %{conn: conn, site: site} do
params = %{
n: "pageview",
u: "http://www.example.com/opportunity",
- d: domain,
+ d: site.domain,
r: "https://facebook.com/page"
}
conn
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert pageview.pathname == "/opportunity"
assert pageview.referrer_source == "Facebook"
assert pageview.referrer == "facebook.com/page"
end
- test "records hash when in hash mode", %{conn: conn, domain: domain} do
+ test "records hash when in hash mode", %{conn: conn, site: site} do
params = %{
n: "pageview",
u: "http://www.example.com/#page-a",
- d: domain,
+ d: site.domain,
h: 1
}
conn
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert pageview.pathname == "/#page-a"
end
- test "does not record hash when hash mode is 0", %{conn: conn, domain: domain} do
+ test "does not record hash when hash mode is 0", %{conn: conn, site: site} do
params = %{
n: "pageview",
u: "http://www.example.com/#page-a",
- d: domain,
+ d: site.domain,
h: 0
}
conn
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert pageview.pathname == "/"
end
- test "decodes URL pathname, fragment and search", %{conn: conn, domain: domain} do
+ test "decodes URL pathname, fragment and search", %{conn: conn, site: site} do
params = %{
n: "pageview",
u:
"https://test.com/%EF%BA%9D%EF%BB%AD%EF%BA%8E%EF%BA%8B%EF%BA%AF-%EF%BB%AE%EF%BB%A4%EF%BA%B3%EF%BA%8E%EF%BA%92%EF%BB%97%EF%BA%8E%EF%BA%97?utm_source=%25balle%25",
- d: domain,
+ d: site.domain,
h: 1
}
conn
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert pageview.hostname == "test.com"
assert pageview.pathname == "/ﺝﻭﺎﺋﺯ-ﻮﻤﺳﺎﺒﻗﺎﺗ"
assert pageview.utm_source == "%balle%"
end
- test "can use double quotes in query params", %{conn: conn, domain: domain} do
+ test "can use double quotes in query params", %{conn: conn, site: site} do
q = URI.encode_query(%{"utm_source" => "Something \"quoted\""})
params = %{
n: "pageview",
u: "https://test.com/?" <> q,
- d: domain,
+ d: site.domain,
h: 1
}
conn
|> post("/api/event", params)
- pageview = get_event(domain)
+ pageview = get_event(site)
assert pageview.utm_source == "Something \"quoted\""
end
- test "responds 400 when required fields are missing", %{conn: conn, domain: domain} do
+ test "responds 400 when required fields are missing", %{conn: conn, site: site} do
params = %{
- domain: domain,
+ domain: site.domain,
name: "pageview"
}
@@ -1032,17 +1048,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
describe "user_id generation" do
setup do
- site1 = insert(:site)
- {:ok, domain: site1.domain}
+ site = insert(:site)
+ {:ok, site: site}
end
test "with same IP address and user agent, the same user ID is generated", %{
conn: conn,
- domain: domain
+ site: site
} do
params = %{
url: "https://user-id-test-domain.com/",
- domain: domain,
+ domain: site.domain,
name: "pageview"
}
@@ -1056,15 +1072,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("x-forwarded-for", "127.0.0.1")
|> post("/api/event", params)
- [one, two] = get_events(domain)
+ [one, two] = get_events(site)
assert one.user_id == two.user_id
end
- test "different IP address results in different user ID", %{conn: conn, domain: domain} do
+ test "different IP address results in different user ID", %{conn: conn, site: site} do
params = %{
url: "https://user-id-test-domain.com/",
- domain: domain,
+ domain: site.domain,
name: "pageview"
}
@@ -1078,15 +1094,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("x-forwarded-for", "82.32.12.1")
|> post("/api/event", params)
- [one, two] = get_events(domain)
+ [one, two] = get_events(site)
assert one.user_id != two.user_id
end
- test "different user agent results in different user ID", %{conn: conn, domain: domain} do
+ test "different user agent results in different user ID", %{conn: conn, site: site} do
params = %{
url: "https://user-id-test-domain.com/",
- domain: domain,
+ domain: site.domain,
name: "pageview"
}
@@ -1100,17 +1116,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("x-forwarded-for", "127.0.0.1")
|> post("/api/event", params)
- [one, two] = get_events(domain)
+ [one, two] = get_events(site)
assert one.user_id != two.user_id
end
- test "different domain value results in different user ID", %{conn: conn, domain: domain1} do
- domain2 = insert(:site).domain
+ test "different domain value results in different user ID", %{conn: conn, site: site1} do
+ site2 = insert(:site)
params = %{
url: "https://user-id-test-domain.com/",
- domain: domain1,
+ domain: site1.domain,
name: "pageview"
}
@@ -1122,18 +1138,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn
|> put_req_header("user-agent", @user_agent)
|> put_req_header("x-forwarded-for", "127.0.0.1")
- |> post("/api/event", Map.put(params, :domain, domain2))
+ |> post("/api/event", Map.put(params, :domain, site2.domain))
- one = get_event(domain1)
- two = get_event(domain2)
+ one = get_event(site1)
+ two = get_event(site2)
assert one.user_id != two.user_id
end
- test "different hostname results in different user ID", %{conn: conn, domain: domain} do
+ test "different hostname results in different user ID", %{conn: conn, site: site} do
params = %{
url: "https://user-id-test-domain.com/",
- domain: domain,
+ domain: site.domain,
name: "pageview"
}
@@ -1147,18 +1163,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("x-forwarded-for", "127.0.0.1")
|> post("/api/event", Map.put(params, :url, "https://other-domain.com/"))
- [one, two] = get_events(domain)
+ [one, two] = get_events(site)
assert one.user_id != two.user_id
end
test "different hostname results in the same user ID when the root domain in the same", %{
conn: conn,
- domain: domain
+ site: site
} do
params = %{
url: "https://user-id-test-domain.com/",
- domain: domain,
+ domain: site.domain,
name: "pageview"
}
@@ -1172,7 +1188,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("x-forwarded-for", "127.0.0.1")
|> post("/api/event", Map.put(params, :url, "https://app.user-id-test-domain.com/"))
- [one, two] = get_events(domain)
+ [one, two] = get_events(site)
assert one.user_id == two.user_id
end
@@ -1180,13 +1196,13 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
describe "remaining" do
setup do
- site1 = insert(:site)
- {:ok, domain: site1.domain}
+ site = insert(:site)
+ {:ok, site: site}
end
- test "defaults hostname to (none) when missing", %{conn: conn, domain: domain} do
+ test "defaults hostname to (none) when missing", %{conn: conn, site: site} do
params = %{
- domain: domain,
+ domain: site.domain,
name: "pageview",
url: "file:///android_asset/www/index.html"
}
@@ -1196,15 +1212,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("content-type", "text/plain")
|> post("/api/event", Jason.encode!(params))
- pageview = get_event(domain)
+ pageview = get_event(site)
assert response(conn, 202) == "ok"
assert pageview.hostname == "(none)"
end
- test "accepts chrome extension URLs", %{conn: conn, domain: domain} do
+ test "accepts chrome extension URLs", %{conn: conn, site: site} do
params = %{
- domain: domain,
+ domain: site.domain,
name: "pageview",
url: "chrome-extension://liipgellkffekalgefpjolodblggkmjg/popup.html"
}
@@ -1214,7 +1230,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("content-type", "text/plain")
|> post("/api/event", Jason.encode!(params))
- pageview = get_event(domain)
+ pageview = get_event(site)
assert response(conn, 202) == "ok"
assert pageview.hostname == "liipgellkffekalgefpjolodblggkmjg"
@@ -1232,23 +1248,43 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
end
end
- defp get_event(domain) do
+ defp get_event(site) do
Plausible.Event.WriteBuffer.flush()
- ClickhouseRepo.one(
- from e in Plausible.ClickhouseEvent,
- where: e.domain == ^domain,
- order_by: [desc: e.timestamp]
- )
+ if Plausible.v2?() do
+ ClickhouseRepo.one(
+ from(e in Plausible.ClickhouseEventV2,
+ where: e.site_id == ^site.id,
+ order_by: [desc: e.timestamp]
+ )
+ )
+ else
+ ClickhouseRepo.one(
+ from(e in Plausible.ClickhouseEvent,
+ where: e.domain == ^site.domain,
+ order_by: [desc: e.timestamp]
+ )
+ )
+ end
end
- defp get_events(domain) do
+ defp get_events(site) do
Plausible.Event.WriteBuffer.flush()
- ClickhouseRepo.all(
- from e in Plausible.ClickhouseEvent,
- where: e.domain == ^domain,
- order_by: [desc: e.timestamp]
- )
+ if Plausible.v2?() do
+ ClickhouseRepo.all(
+ from(e in Plausible.ClickhouseEventV2,
+ where: e.site_id == ^site.id,
+ order_by: [desc: e.timestamp]
+ )
+ )
+ else
+ ClickhouseRepo.all(
+ from(e in Plausible.ClickhouseEvent,
+ where: e.domain == ^site.domain,
+ order_by: [desc: e.timestamp]
+ )
+ )
+ end
end
end
diff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
index 15dbc469e9..4fa249163a 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
@@ -97,10 +97,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "aggregates a single metric", %{conn: conn, site: site} do
- populate_stats([
- build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ populate_stats(site, [
+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]),
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -120,12 +120,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
conn: conn,
site: site
} do
- populate_stats([
- build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
- build(:pageview, user_id: 456, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, user_id: 456, domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ populate_stats(site, [
+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]),
+ build(:pageview, user_id: 456, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, user_id: 456, timestamp: ~N[2021-01-01 00:25:00]),
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -145,10 +145,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
conn: conn,
site: site
} do
- populate_stats([
- build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ populate_stats(site, [
+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]),
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -171,19 +171,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
describe "comparisons" do
test "compare period=day with previous period", %{conn: conn, site: site} do
- populate_stats([
- build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]),
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -204,19 +202,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "compare period=6mo with previous period", %{conn: conn, site: site} do
- populate_stats([
- build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]),
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-02-01 00:25:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-03-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-03-01 00:00:00])
])
conn =
@@ -239,19 +235,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
describe "filters" do
test "can filter by source", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
referrer_source: "Google",
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -272,20 +266,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "can filter by no source/referrer", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
referrer_source: "Google",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@@ -308,19 +299,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "can filter by referrer", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
referrer: "https://facebook.com",
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -341,19 +330,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "can filter by utm_medium", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
utm_medium: "social",
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -374,19 +361,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "can filter by utm_source", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
utm_source: "Twitter",
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -407,19 +392,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "can filter by utm_campaign", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
utm_campaign: "profile",
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -440,19 +423,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "can filter by device type", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
screen_size: "Desktop",
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -473,19 +454,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "can filter by browser", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
browser: "Chrome",
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -506,20 +485,18 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "can filter by browser version", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
browser: "Chrome",
browser_version: "56",
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -540,19 +517,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "can filter by operating system", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
operating_system: "Mac",
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -573,19 +548,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "can filter by operating system version", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
operating_system_version: "10.5",
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -606,19 +579,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "can filter by country", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
country_code: "EE",
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -642,22 +613,19 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
conn: conn,
site: site
} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
pathname: "/blogpost",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview,
pathname: "/blogpost",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@@ -680,25 +648,22 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "filtering by event:name", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:event,
name: "Signup",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Signup",
- domain: site.domain,
user_id: @user_id,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
name: "Signup",
- domain: site.domain,
user_id: @user_id,
timestamp: ~N[2021-01-01 00:25:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -717,23 +682,20 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "combining filters", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
pathname: "/blogpost",
country_code: "EE",
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview,
pathname: "/blogpost",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@@ -830,15 +792,54 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
site: site
} do
create_sessions([
- %{domain: site.domain, session_id: 1000, country_code: "EE", sign: 1, events: 1},
- %{domain: site.domain, session_id: 1000, country_code: "EE", sign: -1, events: 1},
- %{domain: site.domain, session_id: 1000, country_code: "EE", sign: 1, events: 2}
+ %{
+ domain: site.domain,
+ site_id: site.id,
+ session_id: 1000,
+ country_code: "EE",
+ sign: 1,
+ events: 1
+ },
+ %{
+ domain: site.domain,
+ site_id: site.id,
+ session_id: 1000,
+ country_code: "EE",
+ sign: -1,
+ events: 1
+ },
+ %{
+ domain: site.domain,
+ site_id: site.id,
+ session_id: 1000,
+ country_code: "EE",
+ sign: 1,
+ events: 2
+ }
])
create_events([
- %{domain: site.domain, session_id: 1000, country_code: "EE", name: "pageview"},
- %{domain: site.domain, session_id: 1000, country_code: "EE", name: "pageview"},
- %{domain: site.domain, session_id: 1000, country_code: "EE", name: "pageview"}
+ %{
+ domain: site.domain,
+ site_id: site.id,
+ session_id: 1000,
+ country_code: "EE",
+ name: "pageview"
+ },
+ %{
+ domain: site.domain,
+ site_id: site.id,
+ session_id: 1000,
+ country_code: "EE",
+ name: "pageview"
+ },
+ %{
+ domain: site.domain,
+ site_id: site.id,
+ session_id: 1000,
+ country_code: "EE",
+ name: "pageview"
+ }
])
conn =
diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
index 9f930dfcfe..ccd5a67172 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
@@ -131,20 +131,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by visit:source", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
referrer_source: "Google",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
referrer_source: "Google",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
referrer_source: "",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@@ -166,10 +163,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by visit:country", %{conn: conn, site: site} do
- populate_stats([
- build(:pageview, country_code: "EE", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, country_code: "EE", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
- build(:pageview, country_code: "US", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ populate_stats(site, [
+ build(:pageview, country_code: "EE", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, country_code: "EE", timestamp: ~N[2021-01-01 00:25:00]),
+ build(:pageview, country_code: "US", timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -189,20 +186,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by visit:referrer", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
referrer: "https://ref.com",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
referrer: "https://ref.com",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
referrer: "",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@@ -224,20 +218,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by visit:utm_medium", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
utm_medium: "Search",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
utm_medium: "Search",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
utm_medium: "",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@@ -259,20 +250,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by visit:utm_source", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
utm_source: "Google",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
utm_source: "Google",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
utm_source: "",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@@ -390,20 +378,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by visit:device", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
screen_size: "Desktop",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
screen_size: "Desktop",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
screen_size: "Mobile",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@@ -425,20 +410,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by visit:os", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
operating_system: "Mac",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
operating_system: "Mac",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
operating_system: "Windows",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@@ -460,20 +442,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by visit:os_version", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
operating_system_version: "10.5",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
operating_system_version: "10.5",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
operating_system_version: "10.6",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@@ -495,10 +474,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by visit:browser", %{conn: conn, site: site} do
- populate_stats([
- build(:pageview, browser: "Firefox", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, browser: "Firefox", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
- build(:pageview, browser: "Safari", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ populate_stats(site, [
+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:25:00]),
+ build(:pageview, browser: "Safari", timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -518,20 +497,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by visit:browser_version", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
browser_version: "56",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
browser_version: "56",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
browser_version: "57",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@@ -553,12 +529,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by event:page", %{conn: conn, site: site} do
- populate_stats([
- build(:pageview, pathname: "/", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, pathname: "/", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
+ populate_stats(site, [
+ build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview,
pathname: "/plausible.io",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@@ -583,12 +558,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
conn: conn,
site: site
} do
- populate_stats([
- build(:pageview, pathname: "/", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, pathname: "/", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
+ populate_stats(site, [
+ build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview,
pathname: "/plausible.io",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@@ -609,19 +583,16 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
describe "custom events" do
test "can breakdown by event:name", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:event,
name: "Signup",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Signup",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
)
])
@@ -643,40 +614,34 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "can breakdown by event:name with visitors and events metrics", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
- domain: site.domain,
pathname: "/non-existing",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "404",
- domain: site.domain,
pathname: "/non-existing",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
- domain: site.domain,
pathname: "/non-existing",
timestamp: ~N[2021-01-01 00:00:01]
),
build(:pageview,
- domain: site.domain,
pathname: "/non-existing",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:00:02]
),
build(:event,
name: "404",
- domain: site.domain,
pathname: "/non-existing",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:00:02]
),
build(:pageview,
- domain: site.domain,
pathname: "/non-existing",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:00:03]
@@ -701,39 +666,34 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "can breakdown by event:name while filtering for something", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:event,
name: "Signup",
pathname: "/pageA",
browser: "Chrome",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Signup",
pathname: "/pageA",
browser: "Chrome",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Signup",
pathname: "/pageA",
browser: "Safari",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Signup",
pathname: "/pageB",
browser: "Chrome",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/pageA",
browser: "Chrome",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
)
])
@@ -759,21 +719,18 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
conn: conn,
site: site
} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
referrer_source: "Google",
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Signup",
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
- domain: site.domain,
referrer_source: "Twitter",
timestamp: ~N[2021-01-01 00:25:00]
)
@@ -796,26 +753,22 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "can breakdown by event:name when filtering by event:page", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
pathname: "/pageA",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/pageA",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
pathname: "/pageA",
name: "Signup",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/pageB",
- domain: site.domain,
referrer_source: "Twitter",
timestamp: ~N[2021-01-01 00:25:00]
)
@@ -839,28 +792,24 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "can breakdown by event:page when filtering by event:name", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:event,
name: "Signup",
pathname: "/pageA",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Signup",
pathname: "/pageA",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Signup",
pathname: "/pageB",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/pageB",
- domain: site.domain,
referrer_source: "Twitter",
timestamp: ~N[2021-01-01 00:25:00]
)
@@ -911,33 +860,29 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by custom event property", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:event,
name: "Purchase",
"meta.key": ["package"],
"meta.value": ["business"],
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Purchase",
"meta.key": ["package"],
"meta.value": ["personal"],
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Purchase",
"meta.key": ["package"],
"meta.value": ["business"],
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
name: "Some other event",
"meta.key": ["package"],
"meta.value": ["business"],
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
)
])
@@ -960,45 +905,39 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by custom event property, with (none)", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["16"],
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["16"],
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["16"],
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Purchase",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["14"],
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["14"],
- domain: site.domain,
timestamp: ~N[2021-01-01 00:26:00]
)
])
@@ -1022,33 +961,29 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by custom event property, limited", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["16"],
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["18"],
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["14"],
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["14"],
- domain: site.domain,
timestamp: ~N[2021-01-01 00:26:00]
)
])
@@ -1072,40 +1007,35 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by custom event property, paginated", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["16"],
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["16"],
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["18"],
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["14"],
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["14"],
- domain: site.domain,
timestamp: ~N[2021-01-01 00:26:00]
)
])
@@ -1135,14 +1065,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
insert(:goal, %{domain: site.domain, event_name: "Purchase"})
insert(:goal, %{domain: site.domain, page_path: "/test"})
- populate_stats([
+ populate_stats(site, [
build(:pageview,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00],
pathname: "/test"
),
build(:pageview,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:01],
pathname: "/test",
"meta.key": ["method"],
@@ -1150,14 +1078,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
),
build(:event,
name: "404",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:02],
"meta.key": ["method"],
"meta.value": ["HTTP"]
),
build(:event,
name: "Purchase",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:02],
"meta.key": ["method"],
"meta.value": ["HTTPS"]
@@ -1165,14 +1091,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
build(:event,
name: "404",
timestamp: ~N[2021-01-01 00:00:03],
- domain: site.domain,
"meta.key": ["OS", "method"],
"meta.value": ["Linux", "HTTP"]
),
build(:event,
name: "404",
timestamp: ~N[2021-01-01 00:00:04],
- domain: site.domain,
"meta.key": ["version"],
"meta.value": ["1"]
)
@@ -1213,29 +1137,25 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
describe "filtering" do
test "event:page filter for breakdown by session props", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
pathname: "/ignore",
browser: "Chrome",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/plausible.io",
browser: "Chrome",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/plausible.io",
browser: "Chrome",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
browser: "Safari",
pathname: "/plausible.io",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@@ -1409,25 +1329,21 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "IN filter for event:page", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
pathname: "/ignore",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/plausible.io",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/plausible.io",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/important-page",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@@ -1450,29 +1366,25 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "IN filter for visit:browser", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
pathname: "/ignore",
browser: "Firefox",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/plausible.io",
browser: "Chrome",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/plausible.io",
browser: "Safari",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/important-page",
browser: "Safari",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@@ -1495,25 +1407,21 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "IN filter for visit:entry_page", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
pathname: "/ignore",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/plausible.io",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/plausible.io",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/important-page",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@@ -1537,25 +1445,21 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "IN filter for event:name", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:event,
name: "Signup",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Signup",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Login",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Irrelevant",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@@ -1603,10 +1507,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
describe "pagination" do
test "can limit results", %{conn: conn, site: site} do
- populate_stats([
- build(:pageview, pathname: "/a", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, pathname: "/b", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
- build(:pageview, pathname: "/c", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ populate_stats(site, [
+ build(:pageview, pathname: "/a", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, pathname: "/b", timestamp: ~N[2021-01-01 00:25:00]),
+ build(:pageview, pathname: "/c", timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -1623,11 +1527,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "does not repeat results", %{conn: conn, site: site} do
- populate_stats([
- build(:pageview, %{domain: site.domain, "meta.key": ["item"], "meta.value": ["apple"]}),
- build(:pageview, %{domain: site.domain, "meta.key": ["item"], "meta.value": ["kiwi"]}),
- build(:pageview, %{domain: site.domain, "meta.key": ["item"], "meta.value": ["pineapple"]}),
- build(:pageview, %{domain: site.domain, "meta.key": ["item"], "meta.value": ["grapes"]})
+ populate_stats(site, [
+ build(:pageview, %{"meta.key": ["item"], "meta.value": ["apple"]}),
+ build(:pageview, %{"meta.key": ["item"], "meta.value": ["kiwi"]}),
+ build(:pageview, %{"meta.key": ["item"], "meta.value": ["pineapple"]}),
+ build(:pageview, %{"meta.key": ["item"], "meta.value": ["grapes"]})
])
params = %{
@@ -1693,10 +1597,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "can paginate results", %{conn: conn, site: site} do
- populate_stats([
- build(:pageview, pathname: "/a", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, pathname: "/b", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
- build(:pageview, pathname: "/c", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ populate_stats(site, [
+ build(:pageview, pathname: "/a", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, pathname: "/b", timestamp: ~N[2021-01-01 00:25:00]),
+ build(:pageview, pathname: "/c", timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -1820,13 +1724,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "filter by custom event property", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:event,
name: "Purchase",
"meta.key": ["package"],
"meta.value": ["business"],
browser: "Chrome",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
@@ -1834,7 +1737,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
"meta.key": ["package"],
"meta.value": ["business"],
browser: "Safari",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
@@ -1842,7 +1744,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
"meta.key": ["package"],
"meta.value": ["business"],
browser: "Safari",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
@@ -1850,7 +1751,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
"meta.key": ["package"],
"meta.value": ["personal"],
browser: "IE",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
)
])
@@ -1873,23 +1773,20 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "all metrics for breakdown by event prop", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
user_id: 1,
pathname: "/",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: 1,
pathname: "/plausible.io",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:10:00]
),
- build(:pageview, pathname: "/", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
+ build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview,
pathname: "/plausible.io",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
diff --git a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs
index b84c3cf1c2..b0e1f6109b 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs
@@ -261,9 +261,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "shows last 7 days of visitors", %{conn: conn, site: site} do
- populate_stats([
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-07 23:59:00])
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-07 23:59:00])
])
conn =
@@ -287,10 +287,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "shows last 6 months of visitors", %{conn: conn, site: site} do
- populate_stats([
- build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -313,11 +313,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "shows last 12 months of visitors", %{conn: conn, site: site} do
- populate_stats([
- build(:pageview, domain: site.domain, timestamp: ~N[2020-02-01 00:00:00]),
- build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2020-02-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -346,11 +346,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "shows last 12 months of visitors with interval daily", %{conn: conn, site: site} do
- populate_stats([
- build(:pageview, domain: site.domain, timestamp: ~N[2020-02-01 00:00:00]),
- build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2020-02-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -365,10 +365,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "shows a custom range with daily interval", %{conn: conn, site: site} do
- populate_stats([
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-02 00:00:00])
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-02 00:00:00])
])
conn =
@@ -387,11 +387,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "shows a custom range with monthly interval", %{conn: conn, site: site} do
- populate_stats([
- build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2020-12-01 00:00:00]),
- build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2020-12-01 00:05:00]),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-02 00:00:00])
+ populate_stats(site, [
+ build(:pageview, user_id: @user_id, timestamp: ~N[2020-12-01 00:00:00]),
+ build(:pageview, user_id: @user_id, timestamp: ~N[2020-12-01 00:05:00]),
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-02 00:00:00])
])
conn =
@@ -425,13 +425,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
describe "filters" do
test "can filter by source", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
referrer_source: "Google",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -447,11 +446,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "can filter by no source/referrer", %{conn: conn, site: site} do
- populate_stats([
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview,
referrer_source: "Google",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@@ -469,13 +467,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "can filter by referrer", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
referrer: "https://facebook.com",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -491,13 +488,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "can filter by utm_medium", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
utm_medium: "social",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -513,13 +509,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "can filter by utm_source", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
utm_source: "Twitter",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -535,13 +530,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "can filter by utm_campaign", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
utm_campaign: "profile",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -557,13 +551,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "can filter by device type", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
screen_size: "Desktop",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -579,20 +572,18 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "can filter by browser", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
browser: "Chrome",
browser_version: "56.1",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
browser: "Chrome",
browser_version: "55",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -608,26 +599,23 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "can filter by operating system", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
operating_system: "Mac",
operating_system_version: "10.5",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
operating_system: "Something else",
operating_system_version: "10.5",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
operating_system: "Mac",
operating_system_version: "10.4",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -643,22 +631,20 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "can filter by country", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
user_id: @user_id,
country_code: "EE",
operating_system_version: "10.5",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
country_code: "EE",
operating_system_version: "10.5",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:15:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -685,26 +671,22 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
conn: conn,
site: site
} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/hello",
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:05:00]
),
build(:pageview,
pathname: "/hello",
- domain: site.domain,
timestamp: ~N[2021-01-01 05:00:00]
),
build(:pageview,
pathname: "/goobye",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@@ -732,13 +714,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "can filter by event:name", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:event,
name: "Signup",
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@@ -754,33 +735,29 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "filter by custom event property", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:event,
name: "Purchase",
"meta.key": ["package"],
"meta.value": ["business"],
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Purchase",
"meta.key": ["package"],
"meta.value": ["business"],
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Purchase",
"meta.key": ["package"],
"meta.value": ["personal"],
- domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
name: "Purchase",
"meta.key": ["package"],
"meta.value": ["business"],
- domain: site.domain,
timestamp: ~N[2021-01-02 00:25:00]
)
])
@@ -801,19 +778,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
describe "metrics" do
test "shows pageviews,visits,views_per_visit for last 7d", %{conn: conn, site: site} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:05:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-07 23:59:00])
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-07 23:59:00])
])
conn =
@@ -876,30 +851,26 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
conn: conn,
site: site
} do
- populate_stats([
+ populate_stats(site, [
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-01 00:05:00]
),
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-03 00:00:00]
),
build(:pageview,
user_id: @user_id,
- domain: site.domain,
timestamp: ~N[2021-01-03 00:01:00]
),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-03 00:00:00]),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-03 00:00:00]),
- build(:pageview, domain: site.domain, timestamp: ~N[2021-01-07 23:59:00])
+ build(:pageview, timestamp: ~N[2021-01-03 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-03 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-07 23:59:00])
])
conn =
diff --git a/test/plausible_web/controllers/api/internal_controller_test.exs b/test/plausible_web/controllers/api/internal_controller_test.exs
index 65821f796b..0c8b35721d 100644
--- a/test/plausible_web/controllers/api/internal_controller_test.exs
+++ b/test/plausible_web/controllers/api/internal_controller_test.exs
@@ -14,7 +14,7 @@ defmodule PlausibleWeb.Api.InternalControllerTest do
test "is READY when site has at least 1 pageview", %{conn: conn, user: user} do
site = insert(:site, members: [user])
- Plausible.TestUtils.create_pageviews([%{domain: site.domain}])
+ Plausible.TestUtils.create_pageviews([%{site: site}])
conn = get(conn, "/api/#{site.domain}/status")
diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
index ffc38b7a12..ef2d482b36 100644
--- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
@@ -653,6 +653,41 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
insert(:goal, %{domain: site.domain, page_path: "/*"})
insert(:goal, %{domain: site.domain, page_path: "/**"})
+ populate_stats(site, [
+ build(:pageview,
+ pathname: "/hum",
+ timestamp: ~N[2019-07-01 23:00:00]
+ ),
+ build(:pageview,
+ pathname: "/register",
+ timestamp: ~N[2019-07-01 23:00:00]
+ ),
+ build(:pageview,
+ pathname: "/reg",
+ timestamp: ~N[2019-07-01 23:00:00]
+ ),
+ build(:pageview,
+ pathname: "/billing/success",
+ timestamp: ~N[2019-07-01 23:00:00]
+ ),
+ build(:pageview,
+ pathname: "/billing/upgrade/success",
+ timestamp: ~N[2019-07-01 23:00:00]
+ ),
+ build(:pageview,
+ pathname: "/signup/new",
+ timestamp: ~N[2019-07-01 23:00:00]
+ ),
+ build(:pageview,
+ pathname: "/signup/new/2",
+ timestamp: ~N[2019-07-01 23:00:00]
+ ),
+ build(:pageview,
+ pathname: "/signup/new/3",
+ timestamp: ~N[2019-07-01 23:00:00]
+ )
+ ])
+
conn =
get(
conn,
diff --git a/test/plausible_web/controllers/api/stats_controller/current_visitors_test.exs b/test/plausible_web/controllers/api/stats_controller/current_visitors_test.exs
index 23b1b26528..1bf1a7286a 100644
--- a/test/plausible_web/controllers/api/stats_controller/current_visitors_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/current_visitors_test.exs
@@ -5,6 +5,11 @@ defmodule PlausibleWeb.Api.StatsController.CurrentVisitorsTest do
setup [:create_user, :log_in, :create_site]
test "returns unique users in the last 5 minutes", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, user_id: 123),
+ build(:pageview, user_id: 456)
+ ])
+
conn = get(conn, "/api/stats/#{site.domain}/current-visitors")
assert json_response(conn, 200) == 2
diff --git a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
index 6742f42a73..eef27ce2cc 100644
--- a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
@@ -5,23 +5,37 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
setup [:create_user, :log_in, :create_site]
test "returns suggestions for pages without a query", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/"),
+ build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/register"),
+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/contact"),
+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/irrelevant")
+ ])
+
conn = get(conn, "/api/stats/#{site.domain}/suggestions/page?period=month&date=2019-01-01")
assert json_response(conn, 200) == [
%{"label" => "/", "value" => "/"},
- %{"label" => "/register", "value" => "/register"},
%{"label" => "/contact", "value" => "/contact"},
- %{"label" => "/irrelevant", "value" => "/irrelevant"}
+ %{"label" => "/irrelevant", "value" => "/irrelevant"},
+ %{"label" => "/register", "value" => "/register"}
]
end
test "returns suggestions for pages with a query", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/"),
+ build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/register"),
+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/contact"),
+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/irrelevant")
+ ])
+
conn =
get(conn, "/api/stats/#{site.domain}/suggestions/page?period=month&date=2019-01-01&q=re")
assert json_response(conn, 200) == [
- %{"label" => "/register", "value" => "/register"},
- %{"label" => "/irrelevant", "value" => "/irrelevant"}
+ %{"label" => "/irrelevant", "value" => "/irrelevant"},
+ %{"label" => "/register", "value" => "/register"}
]
end
@@ -42,16 +56,25 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
end
test "returns suggestions for sources", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2019-01-01 23:00:00], referrer_source: "Bing"),
+ build(:pageview, timestamp: ~N[2019-01-01 23:00:00], referrer_source: "10words")
+ ])
+
conn =
get(conn, "/api/stats/#{site.domain}/suggestions/source?period=month&date=2019-01-01")
assert json_response(conn, 200) == [
- %{"label" => "10words", "value" => "10words"},
- %{"label" => "Bing", "value" => "Bing"}
+ %{"label" => "Bing", "value" => "Bing"},
+ %{"label" => "10words", "value" => "10words"}
]
end
test "returns suggestions for countries", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/", country_code: "US")
+ ])
+
conn =
get(
conn,
@@ -106,6 +129,10 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
end
test "returns suggestions for screen sizes", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/", screen_size: "Desktop")
+ ])
+
conn =
get(conn, "/api/stats/#{site.domain}/suggestions/screen?period=month&date=2019-01-01")
@@ -113,6 +140,10 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
end
test "returns suggestions for browsers", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/", browser: "Chrome")
+ ])
+
conn =
get(conn, "/api/stats/#{site.domain}/suggestions/browser?period=month&date=2019-01-01")
@@ -122,6 +153,14 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
test "returns suggestions for browser versions", %{conn: conn, site: site} do
filters = Jason.encode!(%{browser: "Chrome"})
+ populate_stats(site, [
+ build(:pageview,
+ timestamp: ~N[2019-01-01 00:00:00],
+ browser: "Chrome",
+ browser_version: "78.0"
+ )
+ ])
+
conn =
get(
conn,
@@ -132,6 +171,10 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
end
test "returns suggestions for OS", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2019-01-01 00:00:00], operating_system: "Mac")
+ ])
+
conn = get(conn, "/api/stats/#{site.domain}/suggestions/os?period=month&date=2019-01-01")
assert json_response(conn, 200) == [%{"value" => "Mac", "label" => "Mac"}]
@@ -140,6 +183,14 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
test "returns suggestions for OS versions", %{conn: conn, site: site} do
filters = Jason.encode!(%{os: "Mac"})
+ populate_stats(site, [
+ build(:pageview,
+ timestamp: ~N[2019-01-01 00:00:00],
+ operating_system: "Mac",
+ operating_system_version: "10.15"
+ )
+ ])
+
conn =
get(
conn,
@@ -162,6 +213,14 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
end
test "returns suggestions for referrers", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview,
+ timestamp: ~N[2019-01-01 23:00:00],
+ pathname: "/",
+ referrer: "10words.com/page1"
+ )
+ ])
+
conn =
get(conn, "/api/stats/#{site.domain}/suggestions/referrer?period=month&date=2019-01-01")
diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exs
index eee7f47652..7ab6a20578 100644
--- a/test/plausible_web/controllers/site_controller_test.exs
+++ b/test/plausible_web/controllers/site_controller_test.exs
@@ -41,10 +41,12 @@ defmodule PlausibleWeb.SiteControllerTest do
end
test "lists all of your sites with last 24h visitors", %{conn: conn, user: user} do
- insert(:site, members: [user], domain: "test-site.com")
+ site = insert(:site, members: [user])
+
+ populate_stats(site, [build(:pageview), build(:pageview), build(:pageview)])
conn = get(conn, "/sites")
- assert html_response(conn, 200) =~ "test-site.com"
+ assert html_response(conn, 200) =~ site.domain
assert html_response(conn, 200) =~ "<b>3</b> visitors in last 24h"
end
diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exs
index 9ec925d1e7..0c7d32e83b 100644
--- a/test/plausible_web/controllers/stats_controller_test.exs
+++ b/test/plausible_web/controllers/stats_controller_test.exs
@@ -4,9 +4,10 @@ defmodule PlausibleWeb.StatsControllerTest do
describe "GET /:website - anonymous user" do
test "public site - shows site stats", %{conn: conn} do
- insert(:site, domain: "public-site.io", public: true)
+ site = insert(:site, public: true)
+ populate_stats(site, [build(:pageview)])
- conn = get(conn, "/public-site.io")
+ conn = get(conn, "/#{site.domain}")
assert html_response(conn, 200) =~ "stats-react-container"
end
@@ -27,6 +28,7 @@ defmodule PlausibleWeb.StatsControllerTest do
setup [:create_user, :log_in, :create_site]
test "can view stats of a website I've created", %{conn: conn, site: site} do
+ populate_stats(site, [build(:pageview)])
conn = get(conn, "/" <> site.domain)
assert html_response(conn, 200) =~ "stats-react-container"
end
diff --git a/test/support/clickhouse_setup.ex b/test/support/clickhouse_setup.ex
deleted file mode 100644
index ef4fa03e1b..0000000000
--- a/test/support/clickhouse_setup.ex
+++ /dev/null
@@ -1,337 +0,0 @@
-defmodule Plausible.Test.ClickhouseSetup do
- @conversion_1_session_id 123
- @conversion_2_session_id 234
-
- def run() do
- Plausible.TestUtils.create_events([
- %{
- name: "pageview",
- domain: "test-site.com",
- pathname: "/",
- country_code: "EE",
- browser: "Chrome",
- browser_version: "78.0",
- operating_system: "Mac",
- operating_system_version: "10.15",
- screen_size: "Desktop",
- referrer_source: "10words",
- referrer: "10words.com/page1",
- timestamp: ~N[2019-01-01 00:00:00],
- session_id: @conversion_1_session_id
- },
- %{
- name: "pageview",
- domain: "test-site.com",
- pathname: "/",
- country_code: "EE",
- browser: "Chrome",
- browser_version: "78.0",
- operating_system: "Mac",
- operating_system_version: "10.15",
- screen_size: "Desktop",
- referrer_source: "10words",
- referrer: "10words.com/page2",
- timestamp: ~N[2019-01-01 00:00:00]
- },
- %{
- name: "pageview",
- domain: "test-site.com",
- pathname: "/",
- country_code: "US",
- browser: "Chrome",
- browser_version: "79.0",
- operating_system: "Mac",
- operating_system_version: "10.16",
- screen_size: "Laptop",
- referrer_source: "",
- referrer: "",
- timestamp: ~N[2019-01-01 00:00:00]
- },
- %{
- name: "pageview",
- domain: "test-site.com",
- pathname: "/contact",
- country_code: "GB",
- browser: "Firefox",
- operating_system: "Android",
- screen_size: "Mobile",
- referrer_source: "Bing",
- timestamp: ~N[2019-01-01 00:00:00]
- },
- %{name: "pageview", domain: "test-site.com", timestamp: ~N[2019-01-31 00:00:00]},
- %{
- name: "Signup",
- domain: "test-site.com",
- session_id: @conversion_1_session_id,
- timestamp: ~N[2019-01-01 01:00:00],
- "meta.key": ["variant"],
- "meta.value": ["A"]
- },
- %{
- name: "Signup",
- domain: "test-site.com",
- session_id: @conversion_1_session_id,
- timestamp: ~N[2019-01-01 02:00:00],
- "meta.key": ["variant"],
- "meta.value": ["B"]
- },
- %{
- name: "Signup",
- domain: "test-site.com",
- session_id: @conversion_2_session_id,
- timestamp: ~N[2019-01-01 02:00:00],
- "meta.key": ["variant"],
- "meta.value": ["B"]
- },
- %{
- name: "pageview",
- pathname: "/register",
- domain: "test-site.com",
- session_id: @conversion_1_session_id,
- timestamp: ~N[2019-01-01 23:00:00]
- },
- %{
- name: "pageview",
- pathname: "/register",
- domain: "test-site.com",
- session_id: @conversion_2_session_id,
- timestamp: ~N[2019-01-01 23:00:00]
- },
- %{
- name: "pageview",
- pathname: "/irrelevant",
- domain: "test-site.com",
- session_id: @conversion_1_session_id,
- timestamp: ~N[2019-01-01 23:00:01]
- },
- %{
- name: "pageview",
- domain: "test-site.com",
- referrer_source: "Google",
- timestamp: ~N[2019-02-01 01:00:00]
- },
- %{
- name: "pageview",
- domain: "test-site.com",
- referrer_source: "Google",
- timestamp: ~N[2019-02-01 02:00:00]
- },
- %{
- name: "pageview",
- domain: "test-site.com",
- referrer: "t.co/some-link",
- referrer_source: "Twitter",
- timestamp: ~N[2019-03-01 01:00:00]
- },
- %{
- name: "pageview",
- domain: "test-site.com",
- referrer: "t.co/some-link",
- referrer_source: "Twitter",
- timestamp: ~N[2019-03-01 01:00:00]
- },
- %{
- name: "pageview",
- domain: "test-site.com",
- referrer: "t.co/nonexistent-link",
- referrer_source: "Twitter",
- timestamp: ~N[2019-03-01 02:00:00]
- },
- %{name: "pageview", domain: "test-site.com"},
- %{
- name: "pageview",
- domain: "test-site.com",
- timestamp: Timex.now() |> Timex.shift(minutes: -3)
- },
- %{
- name: "pageview",
- domain: "test-site.com",
- timestamp: Timex.now() |> Timex.shift(minutes: -6)
- },
- %{name: "pageview", domain: "tz-test.com", timestamp: ~N[2019-01-01 00:00:00]},
- %{name: "pageview", domain: "public-site.io"},
- %{
- name: "pageview",
- domain: "fetch-tweets-test.com",
- referrer: "t.co/a-link",
- referrer_source: "Twitter"
- },
- %{
- name: "pageview",
- domain: "fetch-tweets-test.com",
- referrer: "t.co/b-link",
- referrer_source: "Twitter",
- timestamp: Timex.now() |> Timex.shift(days: -5)
- },
- %{
- name: "pageview",
- pathname: "/register",
- domain: "test-site.com",
- timestamp: ~N[2019-07-01 23:00:00]
- },
- %{
- name: "pageview",
- pathname: "/signup/new",
- domain: "test-site.com",
- timestamp: ~N[2019-07-01 23:00:00]
- },
- %{
- name: "pageview",
- pathname: "/billing/success",
- domain: "test-site.com",
- timestamp: ~N[2019-07-01 23:00:00]
- },
- %{
- name: "pageview",
- pathname: "/",
- domain: "test-site.com",
- timestamp: ~N[2019-07-01 23:00:00]
- },
- %{
- name: "pageview",
- pathname: "/reg",
- domain: "test-site.com",
- timestamp: ~N[2019-07-01 23:00:00]
- },
- %{
- name: "pageview",
- pathname: "/signup/new/2",
- domain: "test-site.com",
- timestamp: ~N[2019-07-01 23:00:00]
- },
- %{
- name: "pageview",
- pathname: "/signup/new/3",
- domain: "test-site.com",
- timestamp: ~N[2019-07-01 23:00:00]
- },
- %{
- name: "pageview",
- pathname: "/billing/upgrade/success",
- domain: "test-site.com",
- timestamp: ~N[2019-07-01 23:00:00]
- }
- ])
-
- Plausible.TestUtils.create_sessions([
- %{
- domain: "test-site.com",
- entry_page: "/",
- exit_page: "/",
- referrer_source: "10words",
- referrer: "10words.com/page1",
- utm_medium: "listing",
- country_code: "US",
- screen_size: "Desktop",
- browser: "Chrome",
- browser_version: "78.0",
- operating_system: "Mac",
- operating_system_version: "10.15",
- session_id: @conversion_1_session_id,
- is_bounce: true,
- duration: 100,
- start: ~N[2019-01-01 02:00:00],
- timestamp: ~N[2019-01-01 02:00:00]
- },
- %{
- domain: "test-site.com",
- entry_page: "/",
- exit_page: "/",
- referrer_source: "10words",
- referrer: "10words.com/page1",
- utm_medium: "listing",
- session_id: @conversion_2_session_id,
- is_bounce: false,
- duration: 0,
- start: ~N[2019-01-01 02:00:00],
- timestamp: ~N[2019-01-01 02:00:00]
- },
- %{
- domain: "test-site.com",
- entry_page: "/",
- exit_page: "/",
- referrer_source: "Bing",
- referrer: "",
- utm_medium: "search",
- is_bounce: false,
- duration: 100,
- start: ~N[2019-01-01 03:00:00],
- timestamp: ~N[2019-01-01 03:00:00]
- },
- %{
- domain: "test-site.com",
- entry_page: "/",
- exit_page: "/",
- referrer_source: "Google",
- referrer: "",
- is_bounce: false,
- start: ~N[2019-02-01 01:00:00],
- timestamp: ~N[2019-02-01 01:00:00]
- },
- %{
- domain: "test-site.com",
- entry_page: "/",
- exit_page: "/",
- referrer_source: "Google",
- referrer: "",
- is_bounce: false,
- start: ~N[2019-02-01 02:00:00],
- timestamp: ~N[2019-02-01 02:00:00]
- },
- %{
- domain: "test-site.com",
- entry_page: "/",
- exit_page: "/",
- referrer: "t.co/some-link",
- referrer_source: "Twitter",
- start: ~N[2019-03-01 01:00:00],
- timestamp: ~N[2019-03-01 01:00:00]
- },
- %{
- domain: "test-site.com",
- entry_page: "/",
- exit_page: "/",
- referrer: "t.co/some-link",
- referrer_source: "Twitter",
- start: ~N[2019-03-01 01:00:00],
- timestamp: ~N[2019-03-01 01:00:00]
- },
- %{
- domain: "test-site.com",
- entry_page: "/",
- exit_page: "/",
- referrer: "t.co/nonexistent-link",
- referrer_source: "Twitter",
- start: ~N[2019-03-01 02:00:00],
- timestamp: ~N[2019-03-01 02:00:00]
- },
- %{
- domain: "test-site.com",
- entry_page: "/",
- exit_page: "/",
- referrer_source: "Bing",
- referrer: "bing.com",
- start: Timex.now() |> Timex.shift(minutes: -1),
- timestamp: Timex.now() |> Timex.shift(minutes: -1)
- },
- %{
- domain: "test-site.com",
- entry_page: "/",
- exit_page: "/exit",
- referrer_source: "10words",
- referrer: "10words.com",
- start: Timex.now() |> Timex.shift(minutes: -2),
- timestamp: Timex.now() |> Timex.shift(minutes: -2)
- },
- %{
- domain: "test-site.com",
- entry_page: "/",
- exit_page: "/exit",
- referrer_source: "10words",
- referrer: "10words.com",
- start: Timex.now() |> Timex.shift(minutes: -3),
- timestamp: Timex.now() |> Timex.shift(minutes: -3)
- }
- ])
- end
-end
diff --git a/test/support/factory.ex b/test/support/factory.ex
index 629cdbb075..c2e9ecf768 100644
--- a/test/support/factory.ex
+++ b/test/support/factory.ex
@@ -38,19 +38,35 @@ defmodule Plausible.Factory do
def ch_session_factory do
hostname = sequence(:domain, &"example-#{&1}.com")
- %Plausible.ClickhouseSession{
- sign: 1,
- session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
- user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
- hostname: hostname,
- domain: hostname,
- entry_page: "/",
- pageviews: 1,
- events: 1,
- start: Timex.now(),
- timestamp: Timex.now(),
- is_bounce: false
- }
+ if Plausible.v2?() do
+ %Plausible.ClickhouseSessionV2{
+ sign: 1,
+ session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
+ user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
+ hostname: hostname,
+ site_id: Enum.random(1000..10_000),
+ entry_page: "/",
+ pageviews: 1,
+ events: 1,
+ start: Timex.now(),
+ timestamp: Timex.now(),
+ is_bounce: false
+ }
+ else
+ %Plausible.ClickhouseSession{
+ sign: 1,
+ session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
+ user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
+ hostname: hostname,
+ domain: hostname,
+ entry_page: "/",
+ pageviews: 1,
+ events: 1,
+ start: Timex.now(),
+ timestamp: Timex.now(),
+ is_bounce: false
+ }
+ end
end
def pageview_factory do
@@ -65,14 +81,25 @@ defmodule Plausible.Factory do
def event_factory do
hostname = sequence(:domain, &"example-#{&1}.com")
- %Plausible.ClickhouseEvent{
- hostname: hostname,
- domain: hostname,
- pathname: "/",
- timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second),
- user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
- session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate())
- }
+ if Plausible.v2?() do
+ %Plausible.ClickhouseEventV2{
+ hostname: hostname,
+ site_id: Enum.random(1000..10_000),
+ pathname: "/",
+ timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second),
+ user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
+ session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate())
+ }
+ else
+ %Plausible.ClickhouseEvent{
+ hostname: hostname,
+ domain: hostname,
+ pathname: "/",
+ timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second),
+ user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
+ session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate())
+ }
+ end
end
def goal_factory do
diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex
index a129fd2ade..0cf49d40ef 100644
--- a/test/support/test_utils.ex
+++ b/test/support/test_utils.ex
@@ -37,7 +37,6 @@ defmodule Plausible.TestUtils do
def create_site(%{user: user}) do
site =
Factory.insert(:site,
- domain: "test-site.com",
members: [user]
)
@@ -73,30 +72,63 @@ defmodule Plausible.TestUtils do
def create_pageviews(pageviews) do
pageviews =
Enum.map(pageviews, fn pageview ->
+ pageview =
+ if Plausible.v2?() do
+ pageview
+ |> Map.delete(:site)
+ |> Map.put(:site_id, pageview.site.id)
+ else
+ pageview
+ |> Map.delete(:site)
+ |> Map.put(:domain, pageview.site.domain)
+ end
+
Factory.build(:pageview, pageview)
|> Map.from_struct()
|> Map.delete(:__meta__)
|> update_in([:timestamp], &to_naive_truncate/1)
end)
- Plausible.IngestRepo.insert_all(Plausible.ClickhouseEvent, pageviews)
+ if Plausible.v2?() do
+ Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, pageviews)
+ else
+ Plausible.IngestRepo.insert_all(Plausible.ClickhouseEvent, pageviews)
+ end
end
def create_events(events) do
events =
Enum.map(events, fn event ->
+ event =
+ if Plausible.v2?() do
+ Map.delete(event, :domain)
+ else
+ Map.delete(event, :site_id)
+ end
+
Factory.build(:event, event)
|> Map.from_struct()
|> Map.delete(:__meta__)
|> update_in([:timestamp], &to_naive_truncate/1)
end)
- Plausible.IngestRepo.insert_all(Plausible.ClickhouseEvent, events)
+ if Plausible.v2?() do
+ Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, events)
+ else
+ Plausible.IngestRepo.insert_all(Plausible.ClickhouseEvent, events)
+ end
end
def create_sessions(sessions) do
sessions =
Enum.map(sessions, fn session ->
+ session =
+ if Plausible.v2?() do
+ Map.delete(session, :domain)
+ else
+ Map.delete(session, :site_id)
+ end
+
Factory.build(:ch_session, session)
|> Map.from_struct()
|> Map.delete(:__meta__)
@@ -104,7 +136,11 @@ defmodule Plausible.TestUtils do
|> update_in([:start], &to_naive_truncate/1)
end)
- Plausible.IngestRepo.insert_all(Plausible.ClickhouseSession, sessions)
+ if Plausible.v2?() do
+ Plausible.IngestRepo.insert_all(Plausible.ClickhouseSessionV2, sessions)
+ else
+ Plausible.IngestRepo.insert_all(Plausible.ClickhouseSession, sessions)
+ end
end
def log_in(%{user: user, conn: conn}) do
@@ -134,6 +170,9 @@ defmodule Plausible.TestUtils do
def populate_stats(site, events) do
Enum.map(events, fn event ->
case event do
+ %Plausible.ClickhouseEventV2{} ->
+ Map.put(event, :site_id, site.id)
+
%Plausible.ClickhouseEvent{} ->
Map.put(event, :domain, site.domain)
@@ -158,6 +197,9 @@ defmodule Plausible.TestUtils do
end)
|> Enum.split_with(fn event ->
case event do
+ %Plausible.ClickhouseEventV2{} ->
+ true
+
%Plausible.ClickhouseEvent{} ->
true
@@ -174,11 +216,22 @@ defmodule Plausible.TestUtils do
sessions =
Enum.reduce(events, %{}, fn event, sessions ->
session_id = Plausible.Session.CacheStore.on_event(event, nil)
- Map.put(sessions, {event.domain, event.user_id}, session_id)
+
+ if Plausible.v2?() do
+ Map.put(sessions, {event.site_id, event.user_id}, session_id)
+ else
+ Map.put(sessions, {event.domain, event.user_id}, session_id)
+ end
end)
Enum.each(events, fn event ->
- event = Map.put(event, :session_id, sessions[{event.domain, event.user_id}])
+ event =
+ if Plausible.v2?() do
+ Map.put(event, :session_id, sessions[{event.site_id, event.user_id}])
+ else
+ Map.put(event, :session_id, sessions[{event.domain, event.user_id}])
+ end
+
Plausible.Event.WriteBuffer.insert(event)
end)
diff --git a/test/test_helper.exs b/test/test_helper.exs
index 62161c24ab..4d594919b2 100644
--- a/test/test_helper.exs
+++ b/test/test_helper.exs
@@ -1,7 +1,14 @@
{:ok, _} = Application.ensure_all_started(:ex_machina)
-Plausible.Test.ClickhouseSetup.run()
Mox.defmock(Plausible.HTTPClient.Mock, for: Plausible.HTTPClient.Interface)
FunWithFlags.enable(:visits_metric)
ExUnit.start(exclude: :slow)
Application.ensure_all_started(:double)
Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual)
+
+if Plausible.v2?() do
+ IO.puts("Running tests against v2 schema")
+else
+ IO.puts(
+ "Running tests against v1 schema. Use: `V2_MIGRATION_DONE=1 mix test` for secondary run."
+ )
+end
diff --git a/test/workers/send_check_stats_emails_test.exs b/test/workers/send_check_stats_emails_test.exs
index 8a021d5137..b247e119c5 100644
--- a/test/workers/send_check_stats_emails_test.exs
+++ b/test/workers/send_check_stats_emails_test.exs
@@ -25,6 +25,8 @@ defmodule Plausible.Workers.SendCheckStatsEmailsTest do
test "does not send an email if the user has configured a weekly report" do
user = insert(:user, inserted_at: days_ago(9), last_seen: days_ago(7))
site = insert(:site, domain: "test-site.com", members: [user])
+
+ populate_stats(site, [build(:pageview)])
insert(:weekly_report, site: site, recipients: ["[email protected]"])
perform_job(SendCheckStatsEmails, %{})
@@ -34,7 +36,8 @@ defmodule Plausible.Workers.SendCheckStatsEmailsTest do
test "sends an email after a week of signup if the user hasn't logged in" do
user = insert(:user, inserted_at: days_ago(8), last_seen: days_ago(8))
- insert(:site, domain: "test-site.com", members: [user])
+ site = insert(:site, domain: "test-site.com", members: [user])
+ populate_stats(site, [build(:pageview)])
perform_job(SendCheckStatsEmails, %{})
diff --git a/test/workers/send_email_report_test.exs b/test/workers/send_email_report_test.exs
index 0b22fd5577..ed3a77cb60 100644
--- a/test/workers/send_email_report_test.exs
+++ b/test/workers/send_email_report_test.exs
@@ -39,17 +39,17 @@ defmodule Plausible.Workers.SendEmailReportTest do
create_pageviews([
# Sunday before last, not counted
- %{domain: site.domain, timestamp: Timezone.convert(sunday_before_last, "UTC")},
+ %{site: site, timestamp: Timezone.convert(sunday_before_last, "UTC")},
# Sunday before last, not counted
- %{domain: site.domain, timestamp: Timezone.convert(sunday_before_last, "UTC")},
+ %{site: site, timestamp: Timezone.convert(sunday_before_last, "UTC")},
# Last monday, counted
- %{domain: site.domain, timestamp: Timezone.convert(last_monday, "UTC")},
+ %{site: site, timestamp: Timezone.convert(last_monday, "UTC")},
# Last sunday, counted
- %{domain: site.domain, timestamp: Timezone.convert(last_sunday, "UTC")},
+ %{site: site, timestamp: Timezone.convert(last_sunday, "UTC")},
# This monday, not counted
- %{domain: site.domain, timestamp: Timezone.convert(this_monday, "UTC")},
+ %{site: site, timestamp: Timezone.convert(this_monday, "UTC")},
# This monday, not counted
- %{domain: site.domain, timestamp: Timezone.convert(this_monday, "UTC")}
+ %{site: site, timestamp: Timezone.convert(this_monday, "UTC")}
])
perform_job(SendEmailReport, %{"site_id" => site.id, "interval" => "weekly"})
diff --git a/test/workers/send_site_setup_emails_test.exs b/test/workers/send_site_setup_emails_test.exs
index 3634f06609..64862d2ab6 100644
--- a/test/workers/send_site_setup_emails_test.exs
+++ b/test/workers/send_site_setup_emails_test.exs
@@ -41,7 +41,9 @@ defmodule Plausible.Workers.SendSiteSetupEmailsTest do
test "sends the setup completed email as soon as possible" do
user = insert(:user)
- insert(:site, members: [user], domain: "test-site.com")
+ site = insert(:site, members: [user])
+
+ populate_stats(site, [build(:pageview)])
perform_job(SendSiteSetupEmails, %{})
@@ -62,7 +64,7 @@ defmodule Plausible.Workers.SendSiteSetupEmailsTest do
subject: "Your Plausible setup: Waiting for the first page views"
)
- create_pageviews([%{domain: site.domain}])
+ create_pageviews([%{site: site}])
perform_job(SendSiteSetupEmails, %{})
assert_email_delivered_with(
diff --git a/test/workers/send_trial_notifications_test.exs b/test/workers/send_trial_notifications_test.exs
index c056f919f7..768c38a7bd 100644
--- a/test/workers/send_trial_notifications_test.exs
+++ b/test/workers/send_trial_notifications_test.exs
@@ -44,7 +44,7 @@ defmodule Plausible.Workers.SendTrialNotificationsTest do
]
)
- populate_stats(site, [build(:pageview, domain: site.domain)])
+ populate_stats(site, [build(:pageview)])
perform_job(SendTrialNotifications, %{})
@@ -55,7 +55,7 @@ defmodule Plausible.Workers.SendTrialNotificationsTest do
test "sends a reminder 7 days before trial ends (16 days after user signed up)" do
user = insert(:user, trial_expiry_date: Timex.now() |> Timex.shift(days: 7))
site = insert(:site, members: [user])
- populate_stats(site, [build(:pageview, domain: site.domain)])
+ populate_stats(site, [build(:pageview)])
perform_job(SendTrialNotifications, %{})
@@ -67,9 +67,9 @@ defmodule Plausible.Workers.SendTrialNotificationsTest do
site = insert(:site, members: [user])
populate_stats(site, [
- build(:pageview, domain: site.domain),
- build(:pageview, domain: site.domain),
- build(:pageview, domain: site.domain)
+ build(:pageview),
+ build(:pageview),
+ build(:pageview)
])
perform_job(SendTrialNotifications, %{})
@@ -82,9 +82,9 @@ defmodule Plausible.Workers.SendTrialNotificationsTest do
site = insert(:site, members: [user])
populate_stats(site, [
- build(:pageview, domain: site.domain),
- build(:pageview, domain: site.domain),
- build(:pageview, domain: site.domain)
+ build(:pageview),
+ build(:pageview),
+ build(:pageview)
])
perform_job(SendTrialNotifications, %{})
@@ -115,9 +115,9 @@ defmodule Plausible.Workers.SendTrialNotificationsTest do
site = insert(:site, members: [user])
populate_stats(site, [
- build(:pageview, domain: site.domain),
- build(:pageview, domain: site.domain),
- build(:pageview, domain: site.domain)
+ build(:pageview),
+ build(:pageview),
+ build(:pageview)
])
perform_job(SendTrialNotifications, %{})
@@ -130,9 +130,9 @@ defmodule Plausible.Workers.SendTrialNotificationsTest do
site = insert(:site, members: [user])
populate_stats(site, [
- build(:pageview, domain: site.domain),
- build(:pageview, domain: site.domain),
- build(:pageview, domain: site.domain)
+ build(:pageview),
+ build(:pageview),
+ build(:pageview)
])
insert(:subscription, user: user)