Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Switch to new clickhouse adapter (ch/chto) (#2733)
Adam6d79ca5093cd
diff --git a/config/runtime.exs b/config/runtime.exsindex 5cc3882baa..744b66e1cc 100644--- a/config/runtime.exs+++ b/config/runtime.exs@@ -296,7 +296,7 @@ config :plausible, Plausible.AsyncInsertRepo,queue_interval: 2000,url: ch_db_url,pool_size: 1,- clickhouse_settings: [+ settings: [async_insert: 1,wait_for_async_insert: 0]diff --git a/lib/mix/tasks/clean_clickhouse.ex b/lib/mix/tasks/clean_clickhouse.exindex 02c4c97c88..71b984b492 100644--- a/lib/mix/tasks/clean_clickhouse.ex+++ b/lib/mix/tasks/clean_clickhouse.ex@@ -1,10 +1,15 @@defmodule Mix.Tasks.CleanClickhouse douse Mix.Task+ alias Plausible.IngestRepo+def run(_) do- clean_events = "ALTER TABLE events DELETE WHERE 1"- clean_sessions = "ALTER TABLE sessions DELETE WHERE 1"- Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, clean_events)- Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, clean_sessions)+ %{rows: rows} = IngestRepo.query!("show tables")+ tables = Enum.map(rows, fn [table] -> table end)+ to_truncate = tables -- ["schema_migrations"]++ Enum.each(to_truncate, fn table ->+ IngestRepo.query!("truncate #{table}")+ end)endenddiff --git a/lib/plausible/async_insert_repo.ex b/lib/plausible/async_insert_repo.exindex 6fc2e931cc..d73bbe8ffd 100644--- a/lib/plausible/async_insert_repo.ex+++ b/lib/plausible/async_insert_repo.ex@@ -5,7 +5,7 @@ defmodule Plausible.AsyncInsertRepo douse Ecto.Repo,otp_app: :plausible,- adapter: ClickhouseEcto+ adapter: Ecto.Adapters.ClickHousedefmacro __using__(_) doquote dodiff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.exindex cc8f5f206e..f6f8631d97 100644--- a/lib/plausible/clickhouse_repo.ex+++ b/lib/plausible/clickhouse_repo.ex@@ -1,7 +1,7 @@defmodule Plausible.ClickhouseRepo douse Ecto.Repo,otp_app: :plausible,- adapter: ClickhouseEcto,+ adapter: Ecto.Adapters.ClickHouse,read_only: truedefmacro __using__(_) dodiff --git a/lib/plausible/event/clickhouse_schema.ex b/lib/plausible/event/clickhouse_schema.exindex cb957576a7..ae8ab148a0 100644--- a/lib/plausible/event/clickhouse_schema.ex+++ b/lib/plausible/event/clickhouse_schema.ex@@ -8,32 +8,32 @@ defmodule Plausible.ClickhouseEvent dofield :domain, :stringfield :hostname, :stringfield :pathname, :string- field :user_id, :integer- field :session_id, :integer+ field :user_id, Ch.Types.UInt64+ field :session_id, Ch.Types.UInt64field :timestamp, :naive_datetime- field :referrer, :string, default: ""- field :referrer_source, :string, default: ""- field :utm_medium, :string, default: ""- field :utm_source, :string, default: ""- field :utm_campaign, :string, default: ""- field :utm_content, :string, default: ""- field :utm_term, :string, default: ""+ field :referrer, :string+ field :referrer_source, :string+ field :utm_medium, :string+ field :utm_source, :string+ field :utm_campaign, :string+ field :utm_content, :string+ field :utm_term, :string- field :country_code, :string, default: ""- field :subdivision1_code, :string, default: ""- field :subdivision2_code, :string, default: ""- field :city_geoname_id, :integer, default: 0+ field :country_code, Ch.Types.FixedString, size: 2+ field :subdivision1_code, :string+ field :subdivision2_code, :string+ field :city_geoname_id, Ch.Types.UInt32- field :screen_size, :string, default: ""- field :operating_system, :string, default: ""- field :operating_system_version, :string, default: ""- field :browser, :string, default: ""- field :browser_version, :string, default: ""+ field :screen_size, :string+ field :operating_system, :string+ field :operating_system_version, :string+ field :browser, :string+ field :browser_version, :string- field :"meta.key", {:array, :string}, default: []- field :"meta.value", {:array, :string}, default: []- field :transferred_from, :string, default: ""+ field :"meta.key", {:array, :string}+ field :"meta.value", {:array, :string}+ field :transferred_from, :stringenddef new(attrs) dodiff --git a/lib/plausible/google/buffer.ex b/lib/plausible/google/buffer.exindex b43160dfad..a32679fecd 100644--- a/lib/plausible/google/buffer.ex+++ b/lib/plausible/google/buffer.ex@@ -91,6 +91,23 @@ defmodule Plausible.Google.Buffer doProcess.sleep(1000)Logger.info("Import: Flushing #{length(records)} from #{table_name} buffer")- Plausible.IngestRepo.insert_all(table_name, records)+ insert_all(table_name, records)end++ # used in tests `setup`+ @doc false+ def insert_all(table_name, records) do+ schema = table_schema(table_name)+ Plausible.IngestRepo.insert_all(schema, records)+ end++ defp table_schema("imported_visitors"), do: Plausible.Google.ImportedVisitor+ defp table_schema("imported_sources"), do: Plausible.Google.ImportedSource+ defp table_schema("imported_pages"), do: Plausible.Google.ImportedPage+ defp table_schema("imported_entry_pages"), do: Plausible.Google.ImportedEntryPage+ defp table_schema("imported_exit_pages"), do: Plausible.Google.ImportedExitPage+ defp table_schema("imported_locations"), do: Plausible.Google.ImportedLocation+ defp table_schema("imported_devices"), do: Plausible.Google.ImportedDevice+ defp table_schema("imported_browsers"), do: Plausible.Google.ImportedBrowser+ defp table_schema("imported_operating_systems"), do: Plausible.Google.ImportedOperatingSystemenddiff --git a/lib/plausible/google/schemas.ex b/lib/plausible/google/schemas.exnew file mode 100644index 0000000000..780b692cca--- /dev/null+++ b/lib/plausible/google/schemas.ex@@ -0,0 +1,148 @@+defmodule Plausible.Google.ImportedVisitor do+ @moduledoc false+ use Ecto.Schema++ @primary_key false+ schema "imported_visitors" do+ field :site_id, Ch.Types.UInt64+ field :date, :date+ field :visitors, Ch.Types.UInt64+ field :pageviews, Ch.Types.UInt64+ field :bounces, Ch.Types.UInt64+ field :visits, Ch.Types.UInt64+ field :visit_duration, Ch.Types.UInt64+ end+end++defmodule Plausible.Google.ImportedSource do+ @moduledoc false+ use Ecto.Schema++ @primary_key false+ schema "imported_sources" do+ field :site_id, Ch.Types.UInt64+ field :date, :date+ field :source, :string+ field :utm_medium, :string+ field :utm_campaign, :string+ field :utm_content, :string+ field :utm_term, :string+ field :visitors, Ch.Types.UInt64+ field :visits, Ch.Types.UInt64+ field :visit_duration, Ch.Types.UInt64+ field :bounces, Ch.Types.UInt32+ end+end++defmodule Plausible.Google.ImportedPage do+ @moduledoc false+ use Ecto.Schema++ @primary_key false+ schema "imported_pages" do+ field :site_id, Ch.Types.UInt64+ field :date, :date+ field :hostname, :string+ field :page, :string+ field :visitors, Ch.Types.UInt64+ field :pageviews, Ch.Types.UInt64+ field :exits, Ch.Types.UInt64+ field :time_on_page, Ch.Types.UInt64+ end+end++defmodule Plausible.Google.ImportedEntryPage do+ @moduledoc false+ use Ecto.Schema++ @primary_key false+ schema "imported_entry_pages" do+ field :site_id, Ch.Types.UInt64+ field :date, :date+ field :entry_page, :string+ field :visitors, Ch.Types.UInt64+ field :entrances, Ch.Types.UInt64+ field :visit_duration, Ch.Types.UInt64+ field :bounces, Ch.Types.UInt32+ end+end++defmodule Plausible.Google.ImportedExitPage do+ @moduledoc false+ use Ecto.Schema++ @primary_key false+ schema "imported_exit_pages" do+ field :site_id, Ch.Types.UInt64+ field :date, :date+ field :exit_page, :string+ field :visitors, Ch.Types.UInt64+ field :exits, Ch.Types.UInt64+ end+end++defmodule Plausible.Google.ImportedLocation do+ @moduledoc false+ use Ecto.Schema++ @primary_key false+ schema "imported_locations" do+ field :site_id, Ch.Types.UInt64+ field :date, :date+ field :country, :string+ field :region, :string+ field :city, Ch.Types.UInt64+ field :visitors, Ch.Types.UInt64+ field :visits, Ch.Types.UInt64+ field :visit_duration, Ch.Types.UInt64+ field :bounces, Ch.Types.UInt32+ end+end++defmodule Plausible.Google.ImportedDevice do+ @moduledoc false+ use Ecto.Schema++ @primary_key false+ schema "imported_devices" do+ field :site_id, Ch.Types.UInt64+ field :date, :date+ field :device, :string+ field :visitors, Ch.Types.UInt64+ field :visits, Ch.Types.UInt64+ field :visit_duration, Ch.Types.UInt64+ field :bounces, Ch.Types.UInt32+ end+end++defmodule Plausible.Google.ImportedBrowser do+ @moduledoc false+ use Ecto.Schema++ @primary_key false+ schema "imported_browsers" do+ field :site_id, Ch.Types.UInt64+ field :date, :date+ field :browser, :string+ field :visitors, Ch.Types.UInt64+ field :visits, Ch.Types.UInt64+ field :visit_duration, Ch.Types.UInt64+ field :bounces, Ch.Types.UInt32+ end+end++defmodule Plausible.Google.ImportedOperatingSystem do+ @moduledoc false+ use Ecto.Schema++ @primary_key false+ schema "imported_operating_systems" do+ field :site_id, Ch.Types.UInt64+ field :date, :date+ field :operating_system, :string+ field :visitors, Ch.Types.UInt64+ field :visits, Ch.Types.UInt64+ field :visit_duration, Ch.Types.UInt64+ field :bounces, Ch.Types.UInt32+ end+enddiff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.exindex 6b319df015..6e30daaef2 100644--- a/lib/plausible/imported/site.ex+++ b/lib/plausible/imported/site.ex@@ -31,7 +31,7 @@ defmodule Plausible.Imported dodefp parse_number(nr) do{float, ""} = Float.parse(nr)- float+ round(float)enddefp new_from_google_analytics(site_id, "imported_visitors", row) dodiff --git a/lib/plausible/ingest_repo.ex b/lib/plausible/ingest_repo.exindex 7928bd9014..3fe099f10f 100644--- a/lib/plausible/ingest_repo.ex+++ b/lib/plausible/ingest_repo.ex@@ -5,7 +5,7 @@ defmodule Plausible.IngestRepo douse Ecto.Repo,otp_app: :plausible,- adapter: ClickhouseEcto+ adapter: Ecto.Adapters.ClickHousedefmacro __using__(_) doquote dodiff --git a/lib/plausible/ingestion/counters.ex b/lib/plausible/ingestion/counters.exindex 8c4b70aaef..ad15327661 100644--- a/lib/plausible/ingestion/counters.ex+++ b/lib/plausible/ingestion/counters.ex@@ -14,7 +14,7 @@ defmodule Plausible.Ingestion.Counters doaggregate into a 1-minute resolution.Clickhouse connection is set to insert counters asynchronously every time- a pool checkout is made. Those properties are reverted once the insert is done+ a pool checkout is made. Those properties are reverted once the insert is done(or naturally, if the connection crashes)."""diff --git a/lib/plausible/ingestion/counters/record.ex b/lib/plausible/ingestion/counters/record.exindex 1e4c8fe1fb..941be5c9a0 100644--- a/lib/plausible/ingestion/counters/record.ex+++ b/lib/plausible/ingestion/counters/record.ex@@ -9,9 +9,9 @@ defmodule Plausible.Ingestion.Counters.Record do@primary_key falseschema "ingest_counters" dofield :event_timebucket, :utc_datetime- field :site_id, :integer+ field :site_id, Ch.Types.Nullable, type: Ch.Types.UInt64field :domain, :stringfield :metric, :string- field :value, :integer+ field :value, Ch.Types.UInt64endenddiff --git a/lib/plausible/purge.ex b/lib/plausible/purge.exindex a4be392136..a075108c4c 100644--- a/lib/plausible/purge.ex+++ b/lib/plausible/purge.ex@@ -16,7 +16,7 @@ defmodule Plausible.Purge do"""def delete_imported_stats!(site) doEnum.each(Plausible.Imported.tables(), fn table ->- sql = "ALTER TABLE #{table} DELETE WHERE site_id = ?"+ sql = "ALTER TABLE #{table} DELETE WHERE site_id = {$0:UInt64}"Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, sql, [site.id])end)diff --git a/lib/plausible/session/cache_store.ex b/lib/plausible/session/cache_store.exindex fe342a573b..b48e1bc229 100644--- a/lib/plausible/session/cache_store.ex+++ b/lib/plausible/session/cache_store.ex@@ -53,41 +53,16 @@ defmodule Plausible.Session.CacheStore doduration: Timex.diff(event.timestamp, session.start, :second) |> abs,pageviews:if(event.name == "pageview", do: session.pageviews + 1, else: session.pageviews),- country_code:- if(session.country_code == "", do: event.country_code, else: session.country_code),- subdivision1_code:- if(session.subdivision1_code == "",- do: event.subdivision1_code,- else: session.subdivision1_code- ),- subdivision2_code:- if(session.subdivision2_code == "",- do: event.subdivision2_code,- else: session.subdivision2_code- ),- city_geoname_id:- if(session.city_geoname_id == 0,- do: event.city_geoname_id,- else: session.city_geoname_id- ),- operating_system:- if(session.operating_system == "",- do: event.operating_system,- else: session.operating_system- ),+ country_code: session.country_code || event.country_code,+ subdivision1_code: session.subdivision1_code || event.subdivision1_code,+ subdivision2_code: session.subdivision2_code || event.subdivision2_code,+ city_geoname_id: session.city_geoname_id || event.city_geoname_id,+ operating_system: session.operating_system || event.operating_system,operating_system_version:- if(session.operating_system_version == "",- do: event.operating_system_version,- else: session.operating_system_version- ),- browser: if(session.browser == "", do: event.browser, else: session.browser),- browser_version:- if(session.browser_version == "",- do: event.browser_version,- else: session.browser_version- ),- screen_size:- if(session.screen_size == "", do: event.screen_size, else: session.screen_size),+ session.operating_system_version || event.operating_system_version,+ browser: session.browser || event.browser,+ browser_version: session.browser_version || event.browser_version,+ screen_size: session.screen_size || event.screen_size,events: session.events + 1}enddiff --git a/lib/plausible/session/clickhouse_schema.ex b/lib/plausible/session/clickhouse_schema.exindex 3645fd0f91..256fac3fe9 100644--- a/lib/plausible/session/clickhouse_schema.ex+++ b/lib/plausible/session/clickhouse_schema.ex@@ -6,20 +6,20 @@ defmodule Plausible.ClickhouseSession doschema "sessions" dofield :hostname, :stringfield :domain, :string- field :user_id, :integer- field :session_id, :integer+ field :user_id, Ch.Types.UInt64+ field :session_id, Ch.Types.UInt64field :start, :naive_datetime- field :duration, :integer+ field :duration, Ch.Types.UInt32field :is_bounce, :booleanfield :entry_page, :stringfield :exit_page, :string- field :pageviews, :integer- field :events, :integer- field :sign, :integer+ field :pageviews, Ch.Types.Int32+ field :events, Ch.Types.Int32+ field :sign, Ch.Types.Int8- field :"entry_meta.key", {:array, :string}, default: []- field :"entry_meta.value", {:array, :string}, default: []+ field :"entry_meta.key", {:array, :string}+ field :"entry_meta.value", {:array, :string}field :utm_medium, :stringfield :utm_source, :string@@ -29,10 +29,10 @@ defmodule Plausible.ClickhouseSession dofield :referrer, :stringfield :referrer_source, :string- field :country_code, :string, default: ""- field :subdivision1_code, :string, default: ""- field :subdivision2_code, :string, default: ""- field :city_geoname_id, :integer, default: 0+ field :country_code, Ch.Types.FixedString, size: 2+ field :subdivision1_code, :string+ field :subdivision2_code, :string+ field :city_geoname_id, Ch.Types.UInt32field :screen_size, :stringfield :operating_system, :string@@ -41,7 +41,7 @@ defmodule Plausible.ClickhouseSession dofield :browser_version, :stringfield :timestamp, :naive_datetime- field :transferred_from, :string, default: ""+ field :transferred_from, :stringenddef random_uint64() dodiff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.exindex 146d9c6446..4b913c4d30 100644--- a/lib/plausible/stats/aggregate.ex+++ b/lib/plausible/stats/aggregate.ex@@ -61,24 +61,28 @@ defmodule Plausible.Stats.Aggregate do){base_query_raw, base_query_raw_params} = ClickhouseRepo.to_sql(:all, q)+ where_param_idx = length(base_query_raw_params)+ where_param = "{$#{where_param_idx}:String}"{where_clause, where_arg} =case query.filters["event:page"] do{:is, page} ->- {"p = ?", page}+ {"p = #{where_param}", page}{:is_not, page} ->- {"p != ?", page}+ {"p != #{where_param}", page}{:matches, expr} ->regex = page_regex(expr)- {"match(p, ?)", regex}+ {"match(p, #{where_param})", regex}{:does_not_match, expr} ->regex = page_regex(expr)- {"not(match(p, ?))", regex}+ {"not(match(p, #{where_param}))", regex}end+ params = base_query_raw_params ++ [where_arg]+time_query = "SELECTavg(ifNotFinite(avgTime, null))@@ -102,7 +106,7 @@ defmodule Plausible.Stats.Aggregate doGROUP BY p,p2,s)GROUP BY p)"- {:ok, res} = ClickhouseRepo.query(time_query, base_query_raw_params ++ [where_arg])+ {:ok, res} = ClickhouseRepo.query(time_query, params)[[time_on_page]] = res.rows%{time_on_page: time_on_page}enddiff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.exindex 0a464c65e4..244f576401 100644--- a/lib/plausible/stats/base.ex+++ b/lib/plausible/stats/base.ex@@ -174,7 +174,7 @@ defmodule Plausible.Stats.Base do{:member, values} ->list = Enum.map(values, &db_prop_val(prop_name, &1))- from(s in sessions_q, where: fragment("? in tuple(?)", field(s, ^prop_name), ^list))+ from(s in sessions_q, where: field(s, ^prop_name) in ^list){:matches, expr} ->regex = page_regex(expr)@@ -399,8 +399,10 @@ defmodule Plausible.Stats.Base doNaiveDateTime.utc_now()|> Timex.shift(seconds: 5)|> beginning_of_time(site.native_stats_start_at)+ |> NaiveDateTime.truncate(:second)- first_datetime = NaiveDateTime.utc_now() |> Timex.shift(minutes: -5)+ first_datetime =+ NaiveDateTime.utc_now() |> Timex.shift(minutes: -5) |> NaiveDateTime.truncate(:second){first_datetime, last_datetime}end@@ -410,8 +412,10 @@ defmodule Plausible.Stats.Base doNaiveDateTime.utc_now()|> Timex.shift(seconds: 5)|> beginning_of_time(site.native_stats_start_at)+ |> NaiveDateTime.truncate(:second)- first_datetime = NaiveDateTime.utc_now() |> Timex.shift(minutes: -30)+ first_datetime =+ NaiveDateTime.utc_now() |> Timex.shift(minutes: -30) |> NaiveDateTime.truncate(:second){first_datetime, last_datetime}enddiff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex 0abe335529..66f0ea07b8 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -44,14 +44,14 @@ defmodule Plausible.Stats.Breakdown dooffset: ^offset,where:fragment(- "notEmpty(multiMatchAllIndices(?, array(?)) as indices)",+ "notEmpty(multiMatchAllIndices(?, ?) as indices)",e.pathname,^page_regexes),group_by: fragment("index"),select: %{index: fragment("arrayJoin(indices) as index"),- goal: fragment("concat('Visit ', array(?)[index])", ^page_exprs)+ goal: fragment("concat('Visit ', ?[index])", ^page_exprs)})|> select_event_metrics(metrics)@@ -242,6 +242,9 @@ defmodule Plausible.Stats.Breakdown do"round(sum(td)/count(case when p2 != p then 1 end))"end+ pages_idx = length(base_query_raw_params)+ params = base_query_raw_params ++ [pages]+time_query = "SELECTp,@@ -258,11 +261,11 @@ defmodule Plausible.Stats.Breakdown doneighbor(p, 1) as p2,neighbor(s, 1) as s2FROM (#{base_query_raw}))- WHERE s=s2 AND p IN tuple(?)+ WHERE s=s2 AND p IN {$#{pages_idx}:Array(String)}GROUP BY p,p2,s)GROUP BY p"- {:ok, res} = ClickhouseRepo.query(time_query, base_query_raw_params ++ [pages])+ {:ok, res} = ClickhouseRepo.query(time_query, params)if query.include_imported do# Imported page views have pre-calculated values@@ -367,7 +370,7 @@ defmodule Plausible.Stats.Breakdown dogroup_by: fragment("index"),select_merge: %{index: fragment("arrayJoin(indices) as index"),- page_match: fragment("array(?)[index]", ^match_exprs)+ page_match: fragment("?[index]", ^match_exprs)},order_by: {:asc, fragment("index")})diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.exindex 5ca3551c99..a23b48d65f 100644--- a/lib/plausible/stats/clickhouse.ex+++ b/lib/plausible/stats/clickhouse.ex@@ -176,7 +176,7 @@ defmodule Plausible.Stats.Clickhouse doClickhouseRepo.all(from e in "events",group_by: e.domain,- where: fragment("? IN tuple(?)", e.domain, ^domains),+ where: e.domain in ^domains,where: e.timestamp > fragment("now() - INTERVAL 24 HOUR"),select: {e.domain, fragment("uniq(user_id)")})@@ -421,7 +421,7 @@ defmodule Plausible.Stats.Clickhouse doelsefrom(e in q,- inner_lateral_join: meta in fragment("meta as m"),+ inner_lateral_join: meta in fragment("meta"),where: meta.key == ^key and meta.value == ^val)end@@ -435,19 +435,25 @@ defmodule Plausible.Stats.Clickhouse doenddefp utc_boundaries(%Query{period: "30m"}, site) do- last_datetime = NaiveDateTime.utc_now()+ last_datetime = NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second)first_datetime =- last_datetime |> Timex.shift(minutes: -30) |> beginning_of_time(site.native_stats_start_at)+ last_datetime+ |> Timex.shift(minutes: -30)+ |> beginning_of_time(site.native_stats_start_at)+ |> NaiveDateTime.truncate(:second){first_datetime, last_datetime}enddefp utc_boundaries(%Query{period: "realtime"}, site) do- last_datetime = NaiveDateTime.utc_now()+ last_datetime = NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second)first_datetime =- last_datetime |> Timex.shift(minutes: -5) |> beginning_of_time(site.native_stats_start_at)+ last_datetime+ |> Timex.shift(minutes: -5)+ |> beginning_of_time(site.native_stats_start_at)+ |> NaiveDateTime.truncate(:second){first_datetime, last_datetime}enddiff --git a/lib/plausible/stats/current_visitors.ex b/lib/plausible/stats/current_visitors.exindex 225c0d8a58..c254df889e 100644--- a/lib/plausible/stats/current_visitors.ex+++ b/lib/plausible/stats/current_visitors.ex@@ -6,6 +6,7 @@ defmodule Plausible.Stats.CurrentVisitors dofirst_datetime =NaiveDateTime.utc_now()|> Timex.shift(minutes: -5)+ |> NaiveDateTime.truncate(:second)ClickhouseRepo.one(from e in "events",diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.exindex 32458037d6..1243200533 100644--- a/lib/plausible/stats/imported.ex+++ b/lib/plausible/stats/imported.ex@@ -27,8 +27,8 @@ defmodule Plausible.Stats.Imported dofrom(s in Ecto.Query.subquery(native_q),full_join: i in subquery(imported_q),- on: field(s, :date) == field(i, :date),- select: %{date: field(s, :date)}+ on: s.date == i.date,+ select: %{date: fragment("greatest(?, ?)", s.date, i.date)})|> select_joined_metrics(metrics)enddiff --git a/lib/plausible/stats/props.ex b/lib/plausible/stats/props.exindex 3e457f61ce..7d4a264772 100644--- a/lib/plausible/stats/props.ex+++ b/lib/plausible/stats/props.ex@@ -51,7 +51,7 @@ defmodule Plausible.Stats.Props donil ->ClickhouseRepo.all(from e in base_event_query(site, query),- inner_lateral_join: meta in fragment("meta as m"),+ inner_lateral_join: meta in fragment("meta"),select: {e.name, meta.key},distinct: true)diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.exindex 76fcec917b..1125f39881 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/timeseries.ex@@ -24,8 +24,9 @@ defmodule Plausible.Stats.Timeseries doEnum.map(steps, fn step ->empty_row(step, metrics)- |> Map.merge(Enum.find(event_result, fn row -> row[:date] == step end) || %{})- |> Map.merge(Enum.find(session_result, fn row -> row[:date] == step end) || %{})+ |> Map.merge(Enum.find(event_result, fn row -> date_eq(row[:date], step) end) || %{})+ |> Map.merge(Enum.find(session_result, fn row -> date_eq(row[:date], step) end) || %{})+ |> Map.update!(:date, &date_format/1)end)end@@ -88,7 +89,6 @@ defmodule Plausible.Stats.Timeseries doquery.date_range.first|> Timex.to_datetime()|> Timex.shift(hours: step)- |> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{m}:{s}")end)end@@ -109,10 +109,29 @@ defmodule Plausible.Stats.Timeseries doquery.date_range.first|> Timex.to_datetime()|> Timex.shift(minutes: step)- |> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{m}:{s}")end)end+ defp date_eq(%DateTime{} = left, %DateTime{} = right) do+ NaiveDateTime.compare(left, right) == :eq+ end++ defp date_eq(%Date{} = left, %Date{} = right) do+ Date.compare(left, right) == :eq+ end++ defp date_eq(left, right) do+ left == right+ end++ defp date_format(%DateTime{} = date) do+ Timex.format!(date, "{YYYY}-{0M}-{0D} {h24}:{m}:{s}")+ end++ defp date_format(date) do+ date+ end+defp select_bucket(q, site, %Query{interval: "month"}) dofrom(e in q,diff --git a/mix.exs b/mix.exsindex 50be46c354..c69c3f47fe 100644--- a/mix.exs+++ b/mix.exs@@ -62,8 +62,7 @@ defmodule Plausible.MixProject do{:bcrypt_elixir, "~> 2.0"},{:bypass, "~> 2.1", only: [:dev, :test]},{:cachex, "~> 3.4"},- {:clickhouse_ecto, git: "https://github.com/plausible/clickhouse_ecto.git"},- {:clickhousex, github: "plausible/clickhousex", override: true},+ {:chto, github: "ruslandoga/chto"},{:combination, "~> 0.0.3"},{:connection, "~> 1.1", override: true},{:cors_plug, "~> 3.0"},diff --git a/mix.lock b/mix.lockindex 979dc5c4c6..ee8862debe 100644--- a/mix.lock+++ b/mix.lock@@ -8,11 +8,11 @@"bunt": {:hex, :bunt, "0.2.1", "e2d4792f7bc0ced7583ab54922808919518d0e57ee162901a16a1b6664ef3b14", [:mix], [], "hexpm", "a330bfb4245239787b15005e66ae6845c9cd524a288f0d141c148b02603777a5"},"bypass": {:hex, :bypass, "2.1.0", "909782781bf8e20ee86a9cabde36b259d44af8b9f38756173e8f5e2e1fabb9b1", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "d9b5df8fa5b7a6efa08384e9bbecfe4ce61c77d28a4282f79e02f1ef78d96b80"},"cachex": {:hex, :cachex, "3.4.0", "868b2959ea4aeb328c6b60ff66c8d5123c083466ad3c33d3d8b5f142e13101fb", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "370123b1ab4fba4d2965fb18f87fd758325709787c8c5fce35b3fe80645ccbe5"},- "castore": {:hex, :castore, "0.1.20", "62a0126cbb7cb3e259257827b9190f88316eb7aa3fdac01fd6f2dfd64e7f46e9", [:mix], [], "hexpm", "a020b7650529c986c454a4035b6b13a328e288466986307bea3aadb4c95ac98a"},+ "castore": {:hex, :castore, "0.1.22", "4127549e411bedd012ca3a308dede574f43819fe9394254ca55ab4895abfa1a2", [:mix], [], "hexpm", "c17576df47eb5aa1ee40cc4134316a99f5cad3e215d5c77b8dd3cfef12a22cac"},"certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"},- "chatterbox": {:hex, :ts_chatterbox, "0.13.0", "6f059d97bcaa758b8ea6fffe2b3b81362bd06b639d3ea2bb088335511d691ebf", [:rebar3], [{:hpack, "~>0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "b93d19104d86af0b3f2566c4cba2a57d2e06d103728246ba1ac6c3c0ff010aa7"},- "clickhouse_ecto": {:git, "https://github.com/plausible/clickhouse_ecto.git", "43126c020f07b097c55a81f68a906019fd061f29", []},- "clickhousex": {:git, "https://github.com/plausible/clickhousex.git", "cfeefaf21f0841bc4e5adccb3b1875a1e5eb3ee7", []},+ "ch": {:git, "https://github.com/ruslandoga/ch.git", "5894f81e246c049b9db5186497189cce226947aa", []},+ "chatterbox": {:hex, :ts_chatterbox, "0.13.0", "6f059d97bcaa758b8ea6fffe2b3b81362bd06b639d3ea2bb088335511d691ebf", [:rebar3], [{:hpack, "~> 0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "b93d19104d86af0b3f2566c4cba2a57d2e06d103728246ba1ac6c3c0ff010aa7"},+ "chto": {:git, "https://github.com/ruslandoga/chto.git", "b5913b36331d5bb0a349b08b592da42410643cd9", []},"combination": {:hex, :combination, "0.0.3", "746aedca63d833293ec6e835aa1f34974868829b1486b1e1cb0685f0b2ae1f41", [:mix], [], "hexpm", "72b099f463df42ef7dc6371d250c7070b57b6c5902853f69deb894f79eda18ca"},"combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},"comeonin": {:hex, :comeonin, "5.3.3", "2c564dac95a35650e9b6acfe6d2952083d8a08e4a89b93a481acb552b325892e", [:mix], [], "hexpm", "3e38c9c2cb080828116597ca8807bb482618a315bfafd98c90bc22a821cc84df"},@@ -32,7 +32,6 @@"ecto": {:hex, :ecto, "3.9.4", "3ee68e25dbe0c36f980f1ba5dd41ee0d3eb0873bccae8aeaf1a2647242bffa35", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "de5f988c142a3aa4ec18b85a4ec34a2390b65b24f02385c1144252ff6ff8ee75"},"ecto_sql": {:hex, :ecto_sql, "3.9.2", "34227501abe92dba10d9c3495ab6770e75e79b836d114c41108a4bf2ce200ad5", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.9.2", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "1eb5eeb4358fdbcd42eac11c1fbd87e3affd7904e639d77903c1358b2abd3f70"},"elixir_make": {:hex, :elixir_make, "0.6.3", "bc07d53221216838d79e03a8019d0839786703129599e9619f4ab74c8c096eac", [:mix], [], "hexpm", "f5cbd651c5678bcaabdbb7857658ee106b12509cd976c2c2fca99688e1daf716"},- "elixir_uuid": {:hex, :elixir_uuid, "1.2.1", "dce506597acb7e6b0daeaff52ff6a9043f5919a4c3315abb4143f0b00378c097", [:mix], [], "hexpm", "f7eba2ea6c3555cea09706492716b0d87397b88946e6380898c2889d68585752"},"envy": {:hex, :envy, "1.1.1", "0bc9bd654dec24fcdf203f7c5aa1b8f30620f12cfb28c589d5e9c38fe1b07475", [:mix], [], "hexpm", "7061eb1a47415fd757145d8dec10dc0b1e48344960265cb108f194c4252c3a89"},"erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"},"eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"},@@ -58,7 +57,6 @@"hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"},"html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"},"httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"},- "hut": {:hex, :hut, "1.3.0", "71f2f054e657c03f959cf1acc43f436ea87580696528ca2a55c8afb1b06c85e7", [:"erlang.mk", :rebar, :rebar3], [], "hexpm", "7e15d28555d8a1f2b5a3a931ec120af0753e4853a4c66053db354f35bf9ab563"},"idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},"jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"},"jsx": {:hex, :jsx, "2.8.3", "a05252d381885240744d955fbe3cf810504eb2567164824e19303ea59eef62cf", [:mix, :rebar3], [], "hexpm", "fc3499fed7a726995aa659143a248534adc754ebd16ccd437cd93b649a95091f"},diff --git a/priv/ingest_repo/migrations/.formatter.exs b/priv/ingest_repo/migrations/.formatter.exsnew file mode 100644index 0000000000..49f9151ed2--- /dev/null+++ b/priv/ingest_repo/migrations/.formatter.exs@@ -0,0 +1,4 @@+[+ import_deps: [:ecto_sql],+ inputs: ["*.exs"]+]diff --git a/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs b/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exsindex 55fcb39ac4..6f329cdfb9 100644--- a/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs+++ b/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs@@ -8,8 +8,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateEventsAndSessions dodefp create_events() docreate_if_not_exists table(:events,- engine:- "MergeTree() PARTITION BY toYYYYMM(timestamp) ORDER BY (domain, toDate(timestamp), user_id) SETTINGS index_granularity = 8192"+ primary_key: false,+ engine: "MergeTree",+ options:+ "PARTITION BY toYYYYMM(timestamp) ORDER BY (domain, toDate(timestamp), user_id) SETTINGS index_granularity = 8192") doadd(:name, :string)add(:domain, :string)@@ -30,15 +32,17 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateEventsAndSessions dodefp create_sessions() docreate_if_not_exists table(:sessions,- engine:- "CollapsingMergeTree(sign) PARTITION BY toYYYYMM(start) ORDER BY (domain, toDate(start), user_id, session_id) SETTINGS index_granularity = 8192"+ primary_key: false,+ engine: "CollapsingMergeTree(sign)",+ options:+ "PARTITION BY toYYYYMM(start) ORDER BY (domain, toDate(start), user_id, session_id) SETTINGS index_granularity = 8192") doadd(:session_id, :UInt64)add(:sign, :Int8)add(:domain, :string)add(:user_id, :UInt64)add(:hostname, :string)- add(:is_bounce, :boolean)+ add(:is_bounce, :UInt8)add(:entry_page, :string)add(:exit_page, :string)add(:pageviews, :integer)diff --git a/priv/ingest_repo/migrations/20201020083739_add_event_metadata.exs b/priv/ingest_repo/migrations/20201020083739_add_event_metadata.exsindex 32f2372dfd..4c1b6d44ad 100644--- a/priv/ingest_repo/migrations/20201020083739_add_event_metadata.exs+++ b/priv/ingest_repo/migrations/20201020083739_add_event_metadata.exs@@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.AddEventMetadata dodef change doalter table(:events) do- add :meta, {:nested, {{:key, :string}, {:value, :string}}}+ add :meta, :"Nested(key String, value String)"endendenddiff --git a/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs b/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exsindex 33bc8b23c4..5c7b72a875 100644--- a/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs+++ b/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs@@ -3,7 +3,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors dodef change docreate_if_not_exists table(:imported_visitors,- engine: "MergeTree() ORDER BY (site_id, date)"+ primary_key: false,+ engine: "MergeTree",+ options: "ORDER BY (site_id, date)") doadd(:site_id, :UInt64)add(:date, :date)@@ -15,7 +17,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors doendcreate_if_not_exists table(:imported_sources,- engine: "MergeTree() ORDER BY (site_id, date, source)"+ primary_key: false,+ engine: "MergeTree",+ options: "ORDER BY (site_id, date, source)") doadd(:site_id, :UInt64)add(:date, :date)@@ -31,7 +35,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors doendcreate_if_not_exists table(:imported_pages,- engine: "MergeTree() ORDER BY (site_id, date, hostname, page)"+ primary_key: false,+ engine: "MergeTree",+ options: "ORDER BY (site_id, date, hostname, page)") doadd(:site_id, :UInt64)add(:date, :date)@@ -44,7 +50,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors doendcreate_if_not_exists table(:imported_entry_pages,- engine: "MergeTree() ORDER BY (site_id, date, entry_page)"+ primary_key: false,+ engine: "MergeTree",+ options: "ORDER BY (site_id, date, entry_page)") doadd(:site_id, :UInt64)add(:date, :date)@@ -56,7 +64,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors doendcreate_if_not_exists table(:imported_exit_pages,- engine: "MergeTree() ORDER BY (site_id, date, exit_page)"+ primary_key: false,+ engine: "MergeTree",+ options: "ORDER BY (site_id, date, exit_page)") doadd(:site_id, :UInt64)add(:date, :date)@@ -66,7 +76,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors doendcreate_if_not_exists table(:imported_locations,- engine: "MergeTree() ORDER BY (site_id, date, country, region, city)"+ primary_key: false,+ engine: "MergeTree",+ options: "ORDER BY (site_id, date, country, region, city)") doadd(:site_id, :UInt64)add(:date, :date)@@ -80,7 +92,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors doendcreate_if_not_exists table(:imported_devices,- engine: "MergeTree() ORDER BY (site_id, date, device)"+ primary_key: false,+ engine: "MergeTree",+ options: "ORDER BY (site_id, date, device)") doadd(:site_id, :UInt64)add(:date, :date)@@ -92,7 +106,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors doendcreate_if_not_exists table(:imported_browsers,- engine: "MergeTree() ORDER BY (site_id, date, browser)"+ primary_key: false,+ engine: "MergeTree",+ options: "ORDER BY (site_id, date, browser)") doadd(:site_id, :UInt64)add(:date, :date)@@ -104,7 +120,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors doendcreate_if_not_exists table(:imported_operating_systems,- engine: "MergeTree() ORDER BY (site_id, date, operating_system)"+ primary_key: false,+ engine: "MergeTree",+ options: "ORDER BY (site_id, date, operating_system)") doadd(:site_id, :UInt64)add(:date, :date)diff --git a/priv/ingest_repo/migrations/20220404123000_add_entry_props_to_session.exs b/priv/ingest_repo/migrations/20220404123000_add_entry_props_to_session.exsindex aa951a2d6e..2c80ad8d68 100644--- a/priv/ingest_repo/migrations/20220404123000_add_entry_props_to_session.exs+++ b/priv/ingest_repo/migrations/20220404123000_add_entry_props_to_session.exs@@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.AddEntryPropsToSession dodef change doalter table(:sessions) do- add :"entry.meta", {:nested, {{:key, :string}, {:value, :string}}}+ add :"entry.meta", :"Nested(key String, value String)"endendenddiff --git a/priv/ingest_repo/migrations/20220422075510_add_entry_props.exs b/priv/ingest_repo/migrations/20220422075510_add_entry_props.exsindex a2907a70ab..ce45f6e3e6 100644--- a/priv/ingest_repo/migrations/20220422075510_add_entry_props.exs+++ b/priv/ingest_repo/migrations/20220422075510_add_entry_props.exs@@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.AddEntryProps dodef change doalter table(:sessions) do- add(:entry_meta, {:nested, {{:key, :string}, {:value, :string}}})+ add(:entry_meta, :"Nested(key String, value String)")endendenddiff --git a/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs b/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exsindex 231db40dc3..75596571c0 100644--- a/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs+++ b/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs@@ -3,8 +3,11 @@ defmodule Plausible.IngestRepo.Migrations.CreateIngestCountersTable dodef change docreate_if_not_exists table(:ingest_counters,- engine: "SummingMergeTree(value) ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket))") do-+ primary_key: false,+ engine: "SummingMergeTree(value)",+ options:+ "ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket))"+ ) doadd(:event_timebucket, :utc_datetime)add(:domain, :"LowCardinality(String)")add(:site_id, :"Nullable(UInt64)")diff --git a/priv/ingest_repo/structure.sql b/priv/ingest_repo/structure.sqlnew file mode 100644index 0000000000..550aaa7d51--- /dev/null+++ b/priv/ingest_repo/structure.sql@@ -0,0 +1,242 @@+CREATE TABLE plausible_events_db.sessions+(+ `session_id` UInt64,+ `sign` Int8,+ `domain` String,+ `user_id` UInt64,+ `hostname` String,+ `is_bounce` UInt8,+ `entry_page` String,+ `exit_page` String,+ `pageviews` Int32,+ `events` Int32,+ `duration` UInt32,+ `referrer` String,+ `referrer_source` String,+ `country_code` LowCardinality(FixedString(2)),+ `screen_size` LowCardinality(String),+ `operating_system` LowCardinality(String),+ `browser` LowCardinality(String),+ `start` DateTime,+ `timestamp` DateTime,+ `utm_medium` String,+ `utm_source` String,+ `utm_campaign` String,+ `browser_version` LowCardinality(String),+ `operating_system_version` LowCardinality(String),+ `subdivision1_code` LowCardinality(String),+ `subdivision2_code` LowCardinality(String),+ `city_geoname_id` UInt32,+ `utm_content` String,+ `utm_term` String,+ `transferred_from` String,+ `entry_meta.key` Array(String),+ `entry_meta.value` Array(String)+)+ENGINE = CollapsingMergeTree(sign)+PARTITION BY toYYYYMM(start)+ORDER BY (domain, toDate(start), user_id, session_id)+SAMPLE BY user_id+SETTINGS index_granularity = 8192;++CREATE TABLE plausible_events_db.ingest_counters+(+ `event_timebucket` DateTime,+ `domain` LowCardinality(String),+ `site_id` Nullable(UInt64),+ `metric` LowCardinality(String),+ `value` UInt64+)+ENGINE = SummingMergeTree(value)+ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket))+SETTINGS index_granularity = 8192;++CREATE TABLE plausible_events_db.imported_visitors+(+ `site_id` UInt64,+ `date` Date,+ `visitors` UInt64,+ `pageviews` UInt64,+ `bounces` UInt64,+ `visits` UInt64,+ `visit_duration` UInt64+)+ENGINE = MergeTree+ORDER BY (site_id, date)+SETTINGS index_granularity = 8192;++CREATE TABLE plausible_events_db.imported_sources+(+ `site_id` UInt64,+ `date` Date,+ `source` String,+ `utm_medium` String,+ `utm_campaign` String,+ `utm_content` String,+ `utm_term` String,+ `visitors` UInt64,+ `visits` UInt64,+ `visit_duration` UInt64,+ `bounces` UInt32+)+ENGINE = MergeTree+ORDER BY (site_id, date, source)+SETTINGS index_granularity = 8192;++CREATE TABLE plausible_events_db.imported_pages+(+ `site_id` UInt64,+ `date` Date,+ `hostname` String,+ `page` String,+ `visitors` UInt64,+ `pageviews` UInt64,+ `exits` UInt64,+ `time_on_page` UInt64+)+ENGINE = MergeTree+ORDER BY (site_id, date, hostname, page)+SETTINGS index_granularity = 8192;++CREATE TABLE plausible_events_db.imported_operating_systems+(+ `site_id` UInt64,+ `date` Date,+ `operating_system` String,+ `visitors` UInt64,+ `visits` UInt64,+ `visit_duration` UInt64,+ `bounces` UInt32+)+ENGINE = MergeTree+ORDER BY (site_id, date, operating_system)+SETTINGS index_granularity = 8192;++CREATE TABLE plausible_events_db.imported_locations+(+ `site_id` UInt64,+ `date` Date,+ `country` String,+ `region` String,+ `city` UInt64,+ `visitors` UInt64,+ `visits` UInt64,+ `visit_duration` UInt64,+ `bounces` UInt32+)+ENGINE = MergeTree+ORDER BY (site_id, date, country, region, city)+SETTINGS index_granularity = 8192;++CREATE TABLE plausible_events_db.imported_exit_pages+(+ `site_id` UInt64,+ `date` Date,+ `exit_page` String,+ `visitors` UInt64,+ `exits` UInt64+)+ENGINE = MergeTree+ORDER BY (site_id, date, exit_page)+SETTINGS index_granularity = 8192;++CREATE TABLE plausible_events_db.imported_entry_pages+(+ `site_id` UInt64,+ `date` Date,+ `entry_page` String,+ `visitors` UInt64,+ `entrances` UInt64,+ `visit_duration` UInt64,+ `bounces` UInt32+)+ENGINE = MergeTree+ORDER BY (site_id, date, entry_page)+SETTINGS index_granularity = 8192;++CREATE TABLE plausible_events_db.imported_devices+(+ `site_id` UInt64,+ `date` Date,+ `device` String,+ `visitors` UInt64,+ `visits` UInt64,+ `visit_duration` UInt64,+ `bounces` UInt32+)+ENGINE = MergeTree+ORDER BY (site_id, date, device)+SETTINGS index_granularity = 8192;++CREATE TABLE plausible_events_db.imported_browsers+(+ `site_id` UInt64,+ `date` Date,+ `browser` String,+ `visitors` UInt64,+ `visits` UInt64,+ `visit_duration` UInt64,+ `bounces` UInt32+)+ENGINE = MergeTree+ORDER BY (site_id, date, browser)+SETTINGS index_granularity = 8192;++CREATE TABLE plausible_events_db.events+(+ `name` String,+ `domain` String,+ `user_id` UInt64,+ `session_id` UInt64,+ `hostname` String,+ `pathname` String,+ `referrer` String,+ `referrer_source` String,+ `country_code` LowCardinality(FixedString(2)),+ `screen_size` LowCardinality(String),+ `operating_system` LowCardinality(String),+ `browser` LowCardinality(String),+ `timestamp` DateTime,+ `utm_medium` String,+ `utm_source` String,+ `utm_campaign` String,+ `meta.key` Array(String),+ `meta.value` Array(String),+ `browser_version` LowCardinality(String),+ `operating_system_version` LowCardinality(String),+ `subdivision1_code` LowCardinality(String),+ `subdivision2_code` LowCardinality(String),+ `city_geoname_id` UInt32,+ `utm_content` String,+ `utm_term` String,+ `transferred_from` String+)+ENGINE = MergeTree+PARTITION BY toYYYYMM(timestamp)+ORDER BY (domain, toDate(timestamp), user_id)+SAMPLE BY user_id+SETTINGS index_granularity = 8192;++CREATE TABLE plausible_events_db.schema_migrations+(+ `version` Int64,+ `inserted_at` DateTime+)+ENGINE = TinyLog;++INSERT INTO "plausible_events_db"."schema_migrations" (version, inserted_at) VALUES+(20200915070607,'2023-03-08 10:03:33'),+(20200918075025,'2023-03-08 10:03:33'),+(20201020083739,'2023-03-08 10:03:33'),+(20201106125234,'2023-03-08 10:03:33'),+(20210323130440,'2023-03-08 10:03:33'),+(20210712214034,'2023-03-08 10:03:33'),+(20211017093035,'2023-03-08 10:03:33'),+(20211112130238,'2023-03-08 10:03:33'),+(20220310104931,'2023-03-08 10:03:33'),+(20220404123000,'2023-03-08 10:03:33'),+(20220421161259,'2023-03-08 10:03:33'),+(20220422075510,'2023-03-08 10:03:33'),+(20230124140348,'2023-03-08 10:03:33'),+(20230210140348,'2023-03-08 10:03:33'),+(20230214114402,'2023-03-08 10:03:33');diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exsindex 065749e1f0..34fbf052c6 100644--- a/test/plausible/imported/imported_test.exs+++ b/test/plausible/imported/imported_test.exs@@ -7,7 +7,7 @@ defmodule Plausible.ImportedTest dodefp import_data(ga_data, site_id, table_name) doga_data|> Plausible.Imported.from_google_analytics(site_id, table_name)- |> then(&Plausible.IngestRepo.insert_all(table_name, &1))+ |> then(&Plausible.Google.Buffer.insert_all(table_name, &1))enddescribe "Parse and import third party data fetched from Google Analytics" dodiff --git a/test/plausible/ingestion/counters_test.exs b/test/plausible/ingestion/counters_test.exsindex bf7d14c3e6..19c73c9ce8 100644--- a/test/plausible/ingestion/counters_test.exs+++ b/test/plausible/ingestion/counters_test.exs@@ -37,8 +37,8 @@ defmodule Plausible.Ingestion.CountersTest doendtest "the database eventually sums the records within 1-minute buckets", %{test: test} do- # Testing if the database works is an unfunny way of integration testing,- # but on the upside it's quite straight-forward way of testing if the+ # Testing if the database works is an unfunny way of integration testing,+ # but on the upside it's quite straight-forward way of testing if the# 1-minute bucket rollups are applied when dumping the records that are# originally aggregated with 10s windows.on_exit(:detach, fn ->diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exsindex 73f60fbade..5a719282a4 100644--- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs@@ -39,6 +39,25 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doassert plot == [1] ++ zeroes ++ [1]end+ test "displays visitors for a day with imported data", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-31 00:00:00]),+ build(:imported_visitors, date: ~D[2021-01-01]),+ build(:imported_visitors, date: ~D[2021-01-31])+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&with_imported=true"+ )++ assert %{"plot" => plot, "imported_source" => "Google Analytics"} = json_response(conn, 200)++ assert plot == [2] ++ List.duplicate(0, 23)+ end+test "displays hourly stats in configured timezone", %{conn: conn, user: user} do# UTC+1site =@@ -60,7 +79,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doassert %{"plot" => plot} = json_response(conn, 200)- zeroes = Stream.repeatedly(fn -> 0 end) |> Stream.take(22) |> Enum.into([])+ zeroes = List.duplicate(0, 22)# Expecting pageview to show at 1am CETassert plot == [0, 1] ++ zeroes@@ -108,6 +127,26 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doassert Enum.sum(plot) == 4end+ test "displays visitors for a month with only imported data", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:imported_visitors, date: ~D[2021-01-01]),+ build(:imported_visitors, date: ~D[2021-01-31])+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true"+ )++ assert %{"plot" => plot, "imported_source" => "Google Analytics"} = json_response(conn, 200)++ assert Enum.count(plot) == 31+ assert List.first(plot) == 1+ assert List.last(plot) == 1+ assert Enum.sum(plot) == 2+ end+test "displays visitors for a month with imported data and filter", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, timestamp: ~N[2021-01-01 00:00:00], pathname: "/pageA"),@@ -154,6 +193,26 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doassert Enum.sum(plot) == 4end+ test "displays visitors for 6 months with only imported data", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:imported_visitors, date: ~D[2021-01-01]),+ build(:imported_visitors, date: ~D[2021-06-30])+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=6mo&date=2021-06-30&with_imported=true"+ )++ assert %{"plot" => plot} = json_response(conn, 200)++ assert Enum.count(plot) == 6+ assert List.first(plot) == 1+ assert List.last(plot) == 1+ assert Enum.sum(plot) == 2+ end+test "displays visitors for 12 months with imported data", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),@@ -176,6 +235,26 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doassert Enum.sum(plot) == 4end+ test "displays visitors for 12 months with only imported data", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:imported_visitors, date: ~D[2021-01-01]),+ build(:imported_visitors, date: ~D[2021-12-31])+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=12mo&date=2021-12-31&with_imported=true"+ )++ assert %{"plot" => plot} = json_response(conn, 200)++ assert Enum.count(plot) == 12+ assert List.first(plot) == 1+ assert List.last(plot) == 1+ assert Enum.sum(plot) == 2+ end+test "displays visitors for calendar year with imported data", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),@@ -198,6 +277,26 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doassert Enum.sum(plot) == 4end+ test "displays visitors for calendar year with only imported data", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:imported_visitors, date: ~D[2021-01-01]),+ build(:imported_visitors, date: ~D[2021-12-31])+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=year&date=2021-12-31&with_imported=true"+ )++ assert %{"plot" => plot} = json_response(conn, 200)++ assert Enum.count(plot) == 12+ assert List.first(plot) == 1+ assert List.last(plot) == 1+ assert Enum.sum(plot) == 2+ end+test "displays visitors for all time with just native data", %{conn: conn, site: site} douse Plausible.Repo@@ -292,6 +391,26 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doassert List.last(plot) == 2assert Enum.sum(plot) == 4end++ test "displays pageviews for a month with only imported data", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:imported_visitors, date: ~D[2021-01-01]),+ build(:imported_visitors, date: ~D[2021-01-31])+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=pageviews&with_imported=true"+ )++ assert %{"plot" => plot} = json_response(conn, 200)++ assert Enum.count(plot) == 31+ assert List.first(plot) == 1+ assert List.last(plot) == 1+ assert Enum.sum(plot) == 2+ endenddescribe "GET /api/stats/main-graph - bounce_rate plot" do@@ -337,6 +456,25 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doassert List.first(plot) == 50assert List.last(plot) == 100end++ test "displays bounce rate for a month with only imported data", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:imported_visitors, visits: 1, bounces: 0, date: ~D[2021-01-01]),+ build(:imported_visitors, visits: 1, bounces: 1, date: ~D[2021-01-31])+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=bounce_rate&with_imported=true"+ )++ assert %{"plot" => plot} = json_response(conn, 200)++ assert Enum.count(plot) == 31+ assert List.first(plot) == 0+ assert List.last(plot) == 100+ endenddescribe "GET /api/stats/main-graph - visit_duration plot" do@@ -387,6 +525,23 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doassert Enum.count(plot) == 31assert List.first(plot) == 200end++ test "displays visit_duration for a month with only imported data", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:imported_visitors, visits: 1, visit_duration: 100, date: ~D[2021-01-01])+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=visit_duration&with_imported=true"+ )++ assert %{"plot" => plot} = json_response(conn, 200)++ assert Enum.count(plot) == 31+ assert List.first(plot) == 100+ endenddescribe "GET /api/stats/main-graph - varying intervals" dodiff --git a/test/support/factory.ex b/test/support/factory.exindex 86f5d5719a..629cdbb075 100644--- a/test/support/factory.ex+++ b/test/support/factory.ex@@ -44,26 +44,12 @@ defmodule Plausible.Factory douser_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),hostname: hostname,domain: hostname,- referrer: "",- referrer_source: "",- utm_medium: "",- utm_source: "",- utm_campaign: "",- utm_content: "",- utm_term: "",entry_page: "/",pageviews: 1,events: 1,- duration: 0,start: Timex.now(),timestamp: Timex.now(),- is_bounce: false,- browser: "",- browser_version: "",- country_code: "",- screen_size: "",- operating_system: "",- operating_system_version: ""+ is_bounce: false}end@@ -85,22 +71,7 @@ defmodule Plausible.Factory dopathname: "/",timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second),user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),- session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),- referrer: "",- referrer_source: "",- utm_medium: "",- utm_source: "",- utm_campaign: "",- utm_content: "",- utm_term: "",- browser: "",- browser_version: "",- country_code: "",- screen_size: "",- operating_system: "",- operating_system_version: "",- "meta.key": [],- "meta.value": []+ session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate())}enddiff --git a/test/support/test_utils.ex b/test/support/test_utils.exindex bb506edc50..a129fd2ade 100644--- a/test/support/test_utils.ex+++ b/test/support/test_utils.ex@@ -73,28 +73,38 @@ defmodule Plausible.TestUtils dodef create_pageviews(pageviews) dopageviews =Enum.map(pageviews, fn pageview ->- Factory.build(:pageview, pageview) |> Map.from_struct() |> Map.delete(:__meta__)+ Factory.build(:pageview, pageview)+ |> Map.from_struct()+ |> Map.delete(:__meta__)+ |> update_in([:timestamp], &to_naive_truncate/1)end)- Plausible.IngestRepo.insert_all("events", pageviews)+ Plausible.IngestRepo.insert_all(Plausible.ClickhouseEvent, pageviews)enddef create_events(events) doevents =Enum.map(events, fn event ->- Factory.build(:event, event) |> Map.from_struct() |> Map.delete(:__meta__)+ Factory.build(:event, event)+ |> Map.from_struct()+ |> Map.delete(:__meta__)+ |> update_in([:timestamp], &to_naive_truncate/1)end)- Plausible.IngestRepo.insert_all("events", events)+ Plausible.IngestRepo.insert_all(Plausible.ClickhouseEvent, events)enddef create_sessions(sessions) dosessions =Enum.map(sessions, fn session ->- Factory.build(:ch_session, session) |> Map.from_struct() |> Map.delete(:__meta__)+ Factory.build(:ch_session, session)+ |> Map.from_struct()+ |> Map.delete(:__meta__)+ |> update_in([:timestamp], &to_naive_truncate/1)+ |> update_in([:start], &to_naive_truncate/1)end)- Plausible.IngestRepo.insert_all("sessions", sessions)+ Plausible.IngestRepo.insert_all(Plausible.ClickhouseSession, sessions)enddef log_in(%{user: user, conn: conn}) do@@ -136,7 +146,17 @@ defmodule Plausible.TestUtils dodef populate_stats(events) do{native, imported} =- Enum.split_with(events, fn event ->+ events+ |> Enum.map(fn event ->+ case event do+ %{timestamp: timestamp} ->+ %{event | timestamp: to_naive_truncate(timestamp)}++ _other ->+ event+ end+ end)+ |> Enum.split_with(fn event ->case event do%Plausible.ClickhouseEvent{} ->true@@ -168,7 +188,7 @@ defmodule Plausible.TestUtils dodefp populate_imported_stats(events) doEnum.group_by(events, &Map.fetch!(&1, :table), &Map.delete(&1, :table))- |> Enum.map(fn {table, events} -> Plausible.IngestRepo.insert_all(table, events) end)+ |> Enum.map(fn {table, events} -> Plausible.Google.Buffer.insert_all(table, events) end)enddef relative_time(shifts) do@@ -177,6 +197,14 @@ defmodule Plausible.TestUtils do|> NaiveDateTime.truncate(:second)end+ def to_naive_truncate(%DateTime{} = dt) do+ to_naive_truncate(DateTime.to_naive(dt))+ end++ def to_naive_truncate(%NaiveDateTime{} = naive) do+ NaiveDateTime.truncate(naive, :second)+ end+def eventually(expectation, wait_time_ms \\ 50, retries \\ 10) doEnum.reduce_while(1..retries, nil, fn attempt, _acc ->case expectation.() do