Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Switch to new clickhouse adapter (ch/chto) (#2733)

Adam6d79ca5093cd
diff --git a/config/runtime.exs b/config/runtime.exs
index 5cc3882baa..744b66e1cc 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -296,7 +296,7 @@ config :plausible, Plausible.AsyncInsertRepo,
queue_interval: 2000,
url: ch_db_url,
pool_size: 1,
- clickhouse_settings: [
+ settings: [
async_insert: 1,
wait_for_async_insert: 0
]
diff --git a/lib/mix/tasks/clean_clickhouse.ex b/lib/mix/tasks/clean_clickhouse.ex
index 02c4c97c88..71b984b492 100644
--- a/lib/mix/tasks/clean_clickhouse.ex
+++ b/lib/mix/tasks/clean_clickhouse.ex
@@ -1,10 +1,15 @@
defmodule Mix.Tasks.CleanClickhouse do
use Mix.Task
+ alias Plausible.IngestRepo
+
def run(_) do
- clean_events = "ALTER TABLE events DELETE WHERE 1"
- clean_sessions = "ALTER TABLE sessions DELETE WHERE 1"
- Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, clean_events)
- Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, clean_sessions)
+ %{rows: rows} = IngestRepo.query!("show tables")
+ tables = Enum.map(rows, fn [table] -> table end)
+ to_truncate = tables -- ["schema_migrations"]
+
+ Enum.each(to_truncate, fn table ->
+ IngestRepo.query!("truncate #{table}")
+ end)
end
end
diff --git a/lib/plausible/async_insert_repo.ex b/lib/plausible/async_insert_repo.ex
index 6fc2e931cc..d73bbe8ffd 100644
--- a/lib/plausible/async_insert_repo.ex
+++ b/lib/plausible/async_insert_repo.ex
@@ -5,7 +5,7 @@ defmodule Plausible.AsyncInsertRepo do
use Ecto.Repo,
otp_app: :plausible,
- adapter: ClickhouseEcto
+ adapter: Ecto.Adapters.ClickHouse
defmacro __using__(_) do
quote do
diff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.ex
index cc8f5f206e..f6f8631d97 100644
--- a/lib/plausible/clickhouse_repo.ex
+++ b/lib/plausible/clickhouse_repo.ex
@@ -1,7 +1,7 @@
defmodule Plausible.ClickhouseRepo do
use Ecto.Repo,
otp_app: :plausible,
- adapter: ClickhouseEcto,
+ adapter: Ecto.Adapters.ClickHouse,
read_only: true
defmacro __using__(_) do
diff --git a/lib/plausible/event/clickhouse_schema.ex b/lib/plausible/event/clickhouse_schema.ex
index cb957576a7..ae8ab148a0 100644
--- a/lib/plausible/event/clickhouse_schema.ex
+++ b/lib/plausible/event/clickhouse_schema.ex
@@ -8,32 +8,32 @@ defmodule Plausible.ClickhouseEvent do
field :domain, :string
field :hostname, :string
field :pathname, :string
- field :user_id, :integer
- field :session_id, :integer
+ field :user_id, Ch.Types.UInt64
+ field :session_id, Ch.Types.UInt64
field :timestamp, :naive_datetime
- field :referrer, :string, default: ""
- field :referrer_source, :string, default: ""
- field :utm_medium, :string, default: ""
- field :utm_source, :string, default: ""
- field :utm_campaign, :string, default: ""
- field :utm_content, :string, default: ""
- field :utm_term, :string, default: ""
+ field :referrer, :string
+ field :referrer_source, :string
+ field :utm_medium, :string
+ field :utm_source, :string
+ field :utm_campaign, :string
+ field :utm_content, :string
+ field :utm_term, :string
- field :country_code, :string, default: ""
- field :subdivision1_code, :string, default: ""
- field :subdivision2_code, :string, default: ""
- field :city_geoname_id, :integer, default: 0
+ field :country_code, Ch.Types.FixedString, size: 2
+ field :subdivision1_code, :string
+ field :subdivision2_code, :string
+ field :city_geoname_id, Ch.Types.UInt32
- field :screen_size, :string, default: ""
- field :operating_system, :string, default: ""
- field :operating_system_version, :string, default: ""
- field :browser, :string, default: ""
- field :browser_version, :string, default: ""
+ field :screen_size, :string
+ field :operating_system, :string
+ field :operating_system_version, :string
+ field :browser, :string
+ field :browser_version, :string
- field :"meta.key", {:array, :string}, default: []
- field :"meta.value", {:array, :string}, default: []
- field :transferred_from, :string, default: ""
+ field :"meta.key", {:array, :string}
+ field :"meta.value", {:array, :string}
+ field :transferred_from, :string
end
def new(attrs) do
diff --git a/lib/plausible/google/buffer.ex b/lib/plausible/google/buffer.ex
index b43160dfad..a32679fecd 100644
--- a/lib/plausible/google/buffer.ex
+++ b/lib/plausible/google/buffer.ex
@@ -91,6 +91,23 @@ defmodule Plausible.Google.Buffer do
Process.sleep(1000)
Logger.info("Import: Flushing #{length(records)} from #{table_name} buffer")
- Plausible.IngestRepo.insert_all(table_name, records)
+ insert_all(table_name, records)
end
+
+ # used in tests `setup`
+ @doc false
+ def insert_all(table_name, records) do
+ schema = table_schema(table_name)
+ Plausible.IngestRepo.insert_all(schema, records)
+ end
+
+ defp table_schema("imported_visitors"), do: Plausible.Google.ImportedVisitor
+ defp table_schema("imported_sources"), do: Plausible.Google.ImportedSource
+ defp table_schema("imported_pages"), do: Plausible.Google.ImportedPage
+ defp table_schema("imported_entry_pages"), do: Plausible.Google.ImportedEntryPage
+ defp table_schema("imported_exit_pages"), do: Plausible.Google.ImportedExitPage
+ defp table_schema("imported_locations"), do: Plausible.Google.ImportedLocation
+ defp table_schema("imported_devices"), do: Plausible.Google.ImportedDevice
+ defp table_schema("imported_browsers"), do: Plausible.Google.ImportedBrowser
+ defp table_schema("imported_operating_systems"), do: Plausible.Google.ImportedOperatingSystem
end
diff --git a/lib/plausible/google/schemas.ex b/lib/plausible/google/schemas.ex
new file mode 100644
index 0000000000..780b692cca
--- /dev/null
+++ b/lib/plausible/google/schemas.ex
@@ -0,0 +1,148 @@
+defmodule Plausible.Google.ImportedVisitor do
+ @moduledoc false
+ use Ecto.Schema
+
+ @primary_key false
+ schema "imported_visitors" do
+ field :site_id, Ch.Types.UInt64
+ field :date, :date
+ field :visitors, Ch.Types.UInt64
+ field :pageviews, Ch.Types.UInt64
+ field :bounces, Ch.Types.UInt64
+ field :visits, Ch.Types.UInt64
+ field :visit_duration, Ch.Types.UInt64
+ end
+end
+
+defmodule Plausible.Google.ImportedSource do
+ @moduledoc false
+ use Ecto.Schema
+
+ @primary_key false
+ schema "imported_sources" do
+ field :site_id, Ch.Types.UInt64
+ field :date, :date
+ field :source, :string
+ field :utm_medium, :string
+ field :utm_campaign, :string
+ field :utm_content, :string
+ field :utm_term, :string
+ field :visitors, Ch.Types.UInt64
+ field :visits, Ch.Types.UInt64
+ field :visit_duration, Ch.Types.UInt64
+ field :bounces, Ch.Types.UInt32
+ end
+end
+
+defmodule Plausible.Google.ImportedPage do
+ @moduledoc false
+ use Ecto.Schema
+
+ @primary_key false
+ schema "imported_pages" do
+ field :site_id, Ch.Types.UInt64
+ field :date, :date
+ field :hostname, :string
+ field :page, :string
+ field :visitors, Ch.Types.UInt64
+ field :pageviews, Ch.Types.UInt64
+ field :exits, Ch.Types.UInt64
+ field :time_on_page, Ch.Types.UInt64
+ end
+end
+
+defmodule Plausible.Google.ImportedEntryPage do
+ @moduledoc false
+ use Ecto.Schema
+
+ @primary_key false
+ schema "imported_entry_pages" do
+ field :site_id, Ch.Types.UInt64
+ field :date, :date
+ field :entry_page, :string
+ field :visitors, Ch.Types.UInt64
+ field :entrances, Ch.Types.UInt64
+ field :visit_duration, Ch.Types.UInt64
+ field :bounces, Ch.Types.UInt32
+ end
+end
+
+defmodule Plausible.Google.ImportedExitPage do
+ @moduledoc false
+ use Ecto.Schema
+
+ @primary_key false
+ schema "imported_exit_pages" do
+ field :site_id, Ch.Types.UInt64
+ field :date, :date
+ field :exit_page, :string
+ field :visitors, Ch.Types.UInt64
+ field :exits, Ch.Types.UInt64
+ end
+end
+
+defmodule Plausible.Google.ImportedLocation do
+ @moduledoc false
+ use Ecto.Schema
+
+ @primary_key false
+ schema "imported_locations" do
+ field :site_id, Ch.Types.UInt64
+ field :date, :date
+ field :country, :string
+ field :region, :string
+ field :city, Ch.Types.UInt64
+ field :visitors, Ch.Types.UInt64
+ field :visits, Ch.Types.UInt64
+ field :visit_duration, Ch.Types.UInt64
+ field :bounces, Ch.Types.UInt32
+ end
+end
+
+defmodule Plausible.Google.ImportedDevice do
+ @moduledoc false
+ use Ecto.Schema
+
+ @primary_key false
+ schema "imported_devices" do
+ field :site_id, Ch.Types.UInt64
+ field :date, :date
+ field :device, :string
+ field :visitors, Ch.Types.UInt64
+ field :visits, Ch.Types.UInt64
+ field :visit_duration, Ch.Types.UInt64
+ field :bounces, Ch.Types.UInt32
+ end
+end
+
+defmodule Plausible.Google.ImportedBrowser do
+ @moduledoc false
+ use Ecto.Schema
+
+ @primary_key false
+ schema "imported_browsers" do
+ field :site_id, Ch.Types.UInt64
+ field :date, :date
+ field :browser, :string
+ field :visitors, Ch.Types.UInt64
+ field :visits, Ch.Types.UInt64
+ field :visit_duration, Ch.Types.UInt64
+ field :bounces, Ch.Types.UInt32
+ end
+end
+
+defmodule Plausible.Google.ImportedOperatingSystem do
+ @moduledoc false
+ use Ecto.Schema
+
+ @primary_key false
+ schema "imported_operating_systems" do
+ field :site_id, Ch.Types.UInt64
+ field :date, :date
+ field :operating_system, :string
+ field :visitors, Ch.Types.UInt64
+ field :visits, Ch.Types.UInt64
+ field :visit_duration, Ch.Types.UInt64
+ field :bounces, Ch.Types.UInt32
+ end
+end
diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex
index 6b319df015..6e30daaef2 100644
--- a/lib/plausible/imported/site.ex
+++ b/lib/plausible/imported/site.ex
@@ -31,7 +31,7 @@ defmodule Plausible.Imported do
defp parse_number(nr) do
{float, ""} = Float.parse(nr)
- float
+ round(float)
end
defp new_from_google_analytics(site_id, "imported_visitors", row) do
diff --git a/lib/plausible/ingest_repo.ex b/lib/plausible/ingest_repo.ex
index 7928bd9014..3fe099f10f 100644
--- a/lib/plausible/ingest_repo.ex
+++ b/lib/plausible/ingest_repo.ex
@@ -5,7 +5,7 @@ defmodule Plausible.IngestRepo do
use Ecto.Repo,
otp_app: :plausible,
- adapter: ClickhouseEcto
+ adapter: Ecto.Adapters.ClickHouse
defmacro __using__(_) do
quote do
diff --git a/lib/plausible/ingestion/counters.ex b/lib/plausible/ingestion/counters.ex
index 8c4b70aaef..ad15327661 100644
--- a/lib/plausible/ingestion/counters.ex
+++ b/lib/plausible/ingestion/counters.ex
@@ -14,7 +14,7 @@ defmodule Plausible.Ingestion.Counters do
aggregate into a 1-minute resolution.
Clickhouse connection is set to insert counters asynchronously every time
- a pool checkout is made. Those properties are reverted once the insert is done
+ a pool checkout is made. Those properties are reverted once the insert is done
(or naturally, if the connection crashes).
"""
diff --git a/lib/plausible/ingestion/counters/record.ex b/lib/plausible/ingestion/counters/record.ex
index 1e4c8fe1fb..941be5c9a0 100644
--- a/lib/plausible/ingestion/counters/record.ex
+++ b/lib/plausible/ingestion/counters/record.ex
@@ -9,9 +9,9 @@ defmodule Plausible.Ingestion.Counters.Record do
@primary_key false
schema "ingest_counters" do
field :event_timebucket, :utc_datetime
- field :site_id, :integer
+ field :site_id, Ch.Types.Nullable, type: Ch.Types.UInt64
field :domain, :string
field :metric, :string
- field :value, :integer
+ field :value, Ch.Types.UInt64
end
end
diff --git a/lib/plausible/purge.ex b/lib/plausible/purge.ex
index a4be392136..a075108c4c 100644
--- a/lib/plausible/purge.ex
+++ b/lib/plausible/purge.ex
@@ -16,7 +16,7 @@ defmodule Plausible.Purge do
"""
def delete_imported_stats!(site) do
Enum.each(Plausible.Imported.tables(), fn table ->
- sql = "ALTER TABLE #{table} DELETE WHERE site_id = ?"
+ sql = "ALTER TABLE #{table} DELETE WHERE site_id = {$0:UInt64}"
Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, sql, [site.id])
end)
diff --git a/lib/plausible/session/cache_store.ex b/lib/plausible/session/cache_store.ex
index fe342a573b..b48e1bc229 100644
--- a/lib/plausible/session/cache_store.ex
+++ b/lib/plausible/session/cache_store.ex
@@ -53,41 +53,16 @@ defmodule Plausible.Session.CacheStore do
duration: Timex.diff(event.timestamp, session.start, :second) |> abs,
pageviews:
if(event.name == "pageview", do: session.pageviews + 1, else: session.pageviews),
- country_code:
- if(session.country_code == "", do: event.country_code, else: session.country_code),
- subdivision1_code:
- if(session.subdivision1_code == "",
- do: event.subdivision1_code,
- else: session.subdivision1_code
- ),
- subdivision2_code:
- if(session.subdivision2_code == "",
- do: event.subdivision2_code,
- else: session.subdivision2_code
- ),
- city_geoname_id:
- if(session.city_geoname_id == 0,
- do: event.city_geoname_id,
- else: session.city_geoname_id
- ),
- operating_system:
- if(session.operating_system == "",
- do: event.operating_system,
- else: session.operating_system
- ),
+ country_code: session.country_code || event.country_code,
+ subdivision1_code: session.subdivision1_code || event.subdivision1_code,
+ subdivision2_code: session.subdivision2_code || event.subdivision2_code,
+ city_geoname_id: session.city_geoname_id || event.city_geoname_id,
+ operating_system: session.operating_system || event.operating_system,
operating_system_version:
- if(session.operating_system_version == "",
- do: event.operating_system_version,
- else: session.operating_system_version
- ),
- browser: if(session.browser == "", do: event.browser, else: session.browser),
- browser_version:
- if(session.browser_version == "",
- do: event.browser_version,
- else: session.browser_version
- ),
- screen_size:
- if(session.screen_size == "", do: event.screen_size, else: session.screen_size),
+ session.operating_system_version || event.operating_system_version,
+ browser: session.browser || event.browser,
+ browser_version: session.browser_version || event.browser_version,
+ screen_size: session.screen_size || event.screen_size,
events: session.events + 1
}
end
diff --git a/lib/plausible/session/clickhouse_schema.ex b/lib/plausible/session/clickhouse_schema.ex
index 3645fd0f91..256fac3fe9 100644
--- a/lib/plausible/session/clickhouse_schema.ex
+++ b/lib/plausible/session/clickhouse_schema.ex
@@ -6,20 +6,20 @@ defmodule Plausible.ClickhouseSession do
schema "sessions" do
field :hostname, :string
field :domain, :string
- field :user_id, :integer
- field :session_id, :integer
+ field :user_id, Ch.Types.UInt64
+ field :session_id, Ch.Types.UInt64
field :start, :naive_datetime
- field :duration, :integer
+ field :duration, Ch.Types.UInt32
field :is_bounce, :boolean
field :entry_page, :string
field :exit_page, :string
- field :pageviews, :integer
- field :events, :integer
- field :sign, :integer
+ field :pageviews, Ch.Types.Int32
+ field :events, Ch.Types.Int32
+ field :sign, Ch.Types.Int8
- field :"entry_meta.key", {:array, :string}, default: []
- field :"entry_meta.value", {:array, :string}, default: []
+ field :"entry_meta.key", {:array, :string}
+ field :"entry_meta.value", {:array, :string}
field :utm_medium, :string
field :utm_source, :string
@@ -29,10 +29,10 @@ defmodule Plausible.ClickhouseSession do
field :referrer, :string
field :referrer_source, :string
- field :country_code, :string, default: ""
- field :subdivision1_code, :string, default: ""
- field :subdivision2_code, :string, default: ""
- field :city_geoname_id, :integer, default: 0
+ field :country_code, Ch.Types.FixedString, size: 2
+ field :subdivision1_code, :string
+ field :subdivision2_code, :string
+ field :city_geoname_id, Ch.Types.UInt32
field :screen_size, :string
field :operating_system, :string
@@ -41,7 +41,7 @@ defmodule Plausible.ClickhouseSession do
field :browser_version, :string
field :timestamp, :naive_datetime
- field :transferred_from, :string, default: ""
+ field :transferred_from, :string
end
def random_uint64() do
diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex
index 146d9c6446..4b913c4d30 100644
--- a/lib/plausible/stats/aggregate.ex
+++ b/lib/plausible/stats/aggregate.ex
@@ -61,24 +61,28 @@ defmodule Plausible.Stats.Aggregate do
)
{base_query_raw, base_query_raw_params} = ClickhouseRepo.to_sql(:all, q)
+ where_param_idx = length(base_query_raw_params)
+ where_param = "{$#{where_param_idx}:String}"
{where_clause, where_arg} =
case query.filters["event:page"] do
{:is, page} ->
- {"p = ?", page}
+ {"p = #{where_param}", page}
{:is_not, page} ->
- {"p != ?", page}
+ {"p != #{where_param}", page}
{:matches, expr} ->
regex = page_regex(expr)
- {"match(p, ?)", regex}
+ {"match(p, #{where_param})", regex}
{:does_not_match, expr} ->
regex = page_regex(expr)
- {"not(match(p, ?))", regex}
+ {"not(match(p, #{where_param}))", regex}
end
+ params = base_query_raw_params ++ [where_arg]
+
time_query = "
SELECT
avg(ifNotFinite(avgTime, null))
@@ -102,7 +106,7 @@ defmodule Plausible.Stats.Aggregate do
GROUP BY p,p2,s)
GROUP BY p)"
- {:ok, res} = ClickhouseRepo.query(time_query, base_query_raw_params ++ [where_arg])
+ {:ok, res} = ClickhouseRepo.query(time_query, params)
[[time_on_page]] = res.rows
%{time_on_page: time_on_page}
end
diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex
index 0a464c65e4..244f576401 100644
--- a/lib/plausible/stats/base.ex
+++ b/lib/plausible/stats/base.ex
@@ -174,7 +174,7 @@ defmodule Plausible.Stats.Base do
{:member, values} ->
list = Enum.map(values, &db_prop_val(prop_name, &1))
- from(s in sessions_q, where: fragment("? in tuple(?)", field(s, ^prop_name), ^list))
+ from(s in sessions_q, where: field(s, ^prop_name) in ^list)
{:matches, expr} ->
regex = page_regex(expr)
@@ -399,8 +399,10 @@ defmodule Plausible.Stats.Base do
NaiveDateTime.utc_now()
|> Timex.shift(seconds: 5)
|> beginning_of_time(site.native_stats_start_at)
+ |> NaiveDateTime.truncate(:second)
- first_datetime = NaiveDateTime.utc_now() |> Timex.shift(minutes: -5)
+ first_datetime =
+ NaiveDateTime.utc_now() |> Timex.shift(minutes: -5) |> NaiveDateTime.truncate(:second)
{first_datetime, last_datetime}
end
@@ -410,8 +412,10 @@ defmodule Plausible.Stats.Base do
NaiveDateTime.utc_now()
|> Timex.shift(seconds: 5)
|> beginning_of_time(site.native_stats_start_at)
+ |> NaiveDateTime.truncate(:second)
- first_datetime = NaiveDateTime.utc_now() |> Timex.shift(minutes: -30)
+ first_datetime =
+ NaiveDateTime.utc_now() |> Timex.shift(minutes: -30) |> NaiveDateTime.truncate(:second)
{first_datetime, last_datetime}
end
diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex
index 0abe335529..66f0ea07b8 100644
--- a/lib/plausible/stats/breakdown.ex
+++ b/lib/plausible/stats/breakdown.ex
@@ -44,14 +44,14 @@ defmodule Plausible.Stats.Breakdown do
offset: ^offset,
where:
fragment(
- "notEmpty(multiMatchAllIndices(?, array(?)) as indices)",
+ "notEmpty(multiMatchAllIndices(?, ?) as indices)",
e.pathname,
^page_regexes
),
group_by: fragment("index"),
select: %{
index: fragment("arrayJoin(indices) as index"),
- goal: fragment("concat('Visit ', array(?)[index])", ^page_exprs)
+ goal: fragment("concat('Visit ', ?[index])", ^page_exprs)
}
)
|> select_event_metrics(metrics)
@@ -242,6 +242,9 @@ defmodule Plausible.Stats.Breakdown do
"round(sum(td)/count(case when p2 != p then 1 end))"
end
+ pages_idx = length(base_query_raw_params)
+ params = base_query_raw_params ++ [pages]
+
time_query = "
SELECT
p,
@@ -258,11 +261,11 @@ defmodule Plausible.Stats.Breakdown do
neighbor(p, 1) as p2,
neighbor(s, 1) as s2
FROM (#{base_query_raw}))
- WHERE s=s2 AND p IN tuple(?)
+ WHERE s=s2 AND p IN {$#{pages_idx}:Array(String)}
GROUP BY p,p2,s)
GROUP BY p"
- {:ok, res} = ClickhouseRepo.query(time_query, base_query_raw_params ++ [pages])
+ {:ok, res} = ClickhouseRepo.query(time_query, params)
if query.include_imported do
# Imported page views have pre-calculated values
@@ -367,7 +370,7 @@ defmodule Plausible.Stats.Breakdown do
group_by: fragment("index"),
select_merge: %{
index: fragment("arrayJoin(indices) as index"),
- page_match: fragment("array(?)[index]", ^match_exprs)
+ page_match: fragment("?[index]", ^match_exprs)
},
order_by: {:asc, fragment("index")}
)
diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex
index 5ca3551c99..a23b48d65f 100644
--- a/lib/plausible/stats/clickhouse.ex
+++ b/lib/plausible/stats/clickhouse.ex
@@ -176,7 +176,7 @@ defmodule Plausible.Stats.Clickhouse do
ClickhouseRepo.all(
from e in "events",
group_by: e.domain,
- where: fragment("? IN tuple(?)", e.domain, ^domains),
+ where: e.domain in ^domains,
where: e.timestamp > fragment("now() - INTERVAL 24 HOUR"),
select: {e.domain, fragment("uniq(user_id)")}
)
@@ -421,7 +421,7 @@ defmodule Plausible.Stats.Clickhouse do
else
from(
e in q,
- inner_lateral_join: meta in fragment("meta as m"),
+ inner_lateral_join: meta in fragment("meta"),
where: meta.key == ^key and meta.value == ^val
)
end
@@ -435,19 +435,25 @@ defmodule Plausible.Stats.Clickhouse do
end
defp utc_boundaries(%Query{period: "30m"}, site) do
- last_datetime = NaiveDateTime.utc_now()
+ last_datetime = NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second)
first_datetime =
- last_datetime |> Timex.shift(minutes: -30) |> beginning_of_time(site.native_stats_start_at)
+ last_datetime
+ |> Timex.shift(minutes: -30)
+ |> beginning_of_time(site.native_stats_start_at)
+ |> NaiveDateTime.truncate(:second)
{first_datetime, last_datetime}
end
defp utc_boundaries(%Query{period: "realtime"}, site) do
- last_datetime = NaiveDateTime.utc_now()
+ last_datetime = NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second)
first_datetime =
- last_datetime |> Timex.shift(minutes: -5) |> beginning_of_time(site.native_stats_start_at)
+ last_datetime
+ |> Timex.shift(minutes: -5)
+ |> beginning_of_time(site.native_stats_start_at)
+ |> NaiveDateTime.truncate(:second)
{first_datetime, last_datetime}
end
diff --git a/lib/plausible/stats/current_visitors.ex b/lib/plausible/stats/current_visitors.ex
index 225c0d8a58..c254df889e 100644
--- a/lib/plausible/stats/current_visitors.ex
+++ b/lib/plausible/stats/current_visitors.ex
@@ -6,6 +6,7 @@ defmodule Plausible.Stats.CurrentVisitors do
first_datetime =
NaiveDateTime.utc_now()
|> Timex.shift(minutes: -5)
+ |> NaiveDateTime.truncate(:second)
ClickhouseRepo.one(
from e in "events",
diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex
index 32458037d6..1243200533 100644
--- a/lib/plausible/stats/imported.ex
+++ b/lib/plausible/stats/imported.ex
@@ -27,8 +27,8 @@ defmodule Plausible.Stats.Imported do
from(s in Ecto.Query.subquery(native_q),
full_join: i in subquery(imported_q),
- on: field(s, :date) == field(i, :date),
- select: %{date: field(s, :date)}
+ on: s.date == i.date,
+ select: %{date: fragment("greatest(?, ?)", s.date, i.date)}
)
|> select_joined_metrics(metrics)
end
diff --git a/lib/plausible/stats/props.ex b/lib/plausible/stats/props.ex
index 3e457f61ce..7d4a264772 100644
--- a/lib/plausible/stats/props.ex
+++ b/lib/plausible/stats/props.ex
@@ -51,7 +51,7 @@ defmodule Plausible.Stats.Props do
nil ->
ClickhouseRepo.all(
from e in base_event_query(site, query),
- inner_lateral_join: meta in fragment("meta as m"),
+ inner_lateral_join: meta in fragment("meta"),
select: {e.name, meta.key},
distinct: true
)
diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex
index 76fcec917b..1125f39881 100644
--- a/lib/plausible/stats/timeseries.ex
+++ b/lib/plausible/stats/timeseries.ex
@@ -24,8 +24,9 @@ defmodule Plausible.Stats.Timeseries do
Enum.map(steps, fn step ->
empty_row(step, metrics)
- |> Map.merge(Enum.find(event_result, fn row -> row[:date] == step end) || %{})
- |> Map.merge(Enum.find(session_result, fn row -> row[:date] == step end) || %{})
+ |> Map.merge(Enum.find(event_result, fn row -> date_eq(row[:date], step) end) || %{})
+ |> Map.merge(Enum.find(session_result, fn row -> date_eq(row[:date], step) end) || %{})
+ |> Map.update!(:date, &date_format/1)
end)
end
@@ -88,7 +89,6 @@ defmodule Plausible.Stats.Timeseries do
query.date_range.first
|> Timex.to_datetime()
|> Timex.shift(hours: step)
- |> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{m}:{s}")
end)
end
@@ -109,10 +109,29 @@ defmodule Plausible.Stats.Timeseries do
query.date_range.first
|> Timex.to_datetime()
|> Timex.shift(minutes: step)
- |> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{m}:{s}")
end)
end
+ defp date_eq(%DateTime{} = left, %DateTime{} = right) do
+ NaiveDateTime.compare(left, right) == :eq
+ end
+
+ defp date_eq(%Date{} = left, %Date{} = right) do
+ Date.compare(left, right) == :eq
+ end
+
+ defp date_eq(left, right) do
+ left == right
+ end
+
+ defp date_format(%DateTime{} = date) do
+ Timex.format!(date, "{YYYY}-{0M}-{0D} {h24}:{m}:{s}")
+ end
+
+ defp date_format(date) do
+ date
+ end
+
defp select_bucket(q, site, %Query{interval: "month"}) do
from(
e in q,
diff --git a/mix.exs b/mix.exs
index 50be46c354..c69c3f47fe 100644
--- a/mix.exs
+++ b/mix.exs
@@ -62,8 +62,7 @@ defmodule Plausible.MixProject do
{:bcrypt_elixir, "~> 2.0"},
{:bypass, "~> 2.1", only: [:dev, :test]},
{:cachex, "~> 3.4"},
- {:clickhouse_ecto, git: "https://github.com/plausible/clickhouse_ecto.git"},
- {:clickhousex, github: "plausible/clickhousex", override: true},
+ {:chto, github: "ruslandoga/chto"},
{:combination, "~> 0.0.3"},
{:connection, "~> 1.1", override: true},
{:cors_plug, "~> 3.0"},
diff --git a/mix.lock b/mix.lock
index 979dc5c4c6..ee8862debe 100644
--- a/mix.lock
+++ b/mix.lock
@@ -8,11 +8,11 @@
"bunt": {:hex, :bunt, "0.2.1", "e2d4792f7bc0ced7583ab54922808919518d0e57ee162901a16a1b6664ef3b14", [:mix], [], "hexpm", "a330bfb4245239787b15005e66ae6845c9cd524a288f0d141c148b02603777a5"},
"bypass": {:hex, :bypass, "2.1.0", "909782781bf8e20ee86a9cabde36b259d44af8b9f38756173e8f5e2e1fabb9b1", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "d9b5df8fa5b7a6efa08384e9bbecfe4ce61c77d28a4282f79e02f1ef78d96b80"},
"cachex": {:hex, :cachex, "3.4.0", "868b2959ea4aeb328c6b60ff66c8d5123c083466ad3c33d3d8b5f142e13101fb", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "370123b1ab4fba4d2965fb18f87fd758325709787c8c5fce35b3fe80645ccbe5"},
- "castore": {:hex, :castore, "0.1.20", "62a0126cbb7cb3e259257827b9190f88316eb7aa3fdac01fd6f2dfd64e7f46e9", [:mix], [], "hexpm", "a020b7650529c986c454a4035b6b13a328e288466986307bea3aadb4c95ac98a"},
+ "castore": {:hex, :castore, "0.1.22", "4127549e411bedd012ca3a308dede574f43819fe9394254ca55ab4895abfa1a2", [:mix], [], "hexpm", "c17576df47eb5aa1ee40cc4134316a99f5cad3e215d5c77b8dd3cfef12a22cac"},
"certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"},
- "chatterbox": {:hex, :ts_chatterbox, "0.13.0", "6f059d97bcaa758b8ea6fffe2b3b81362bd06b639d3ea2bb088335511d691ebf", [:rebar3], [{:hpack, "~>0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "b93d19104d86af0b3f2566c4cba2a57d2e06d103728246ba1ac6c3c0ff010aa7"},
- "clickhouse_ecto": {:git, "https://github.com/plausible/clickhouse_ecto.git", "43126c020f07b097c55a81f68a906019fd061f29", []},
- "clickhousex": {:git, "https://github.com/plausible/clickhousex.git", "cfeefaf21f0841bc4e5adccb3b1875a1e5eb3ee7", []},
+ "ch": {:git, "https://github.com/ruslandoga/ch.git", "5894f81e246c049b9db5186497189cce226947aa", []},
+ "chatterbox": {:hex, :ts_chatterbox, "0.13.0", "6f059d97bcaa758b8ea6fffe2b3b81362bd06b639d3ea2bb088335511d691ebf", [:rebar3], [{:hpack, "~> 0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "b93d19104d86af0b3f2566c4cba2a57d2e06d103728246ba1ac6c3c0ff010aa7"},
+ "chto": {:git, "https://github.com/ruslandoga/chto.git", "b5913b36331d5bb0a349b08b592da42410643cd9", []},
"combination": {:hex, :combination, "0.0.3", "746aedca63d833293ec6e835aa1f34974868829b1486b1e1cb0685f0b2ae1f41", [:mix], [], "hexpm", "72b099f463df42ef7dc6371d250c7070b57b6c5902853f69deb894f79eda18ca"},
"combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
"comeonin": {:hex, :comeonin, "5.3.3", "2c564dac95a35650e9b6acfe6d2952083d8a08e4a89b93a481acb552b325892e", [:mix], [], "hexpm", "3e38c9c2cb080828116597ca8807bb482618a315bfafd98c90bc22a821cc84df"},
@@ -32,7 +32,6 @@
"ecto": {:hex, :ecto, "3.9.4", "3ee68e25dbe0c36f980f1ba5dd41ee0d3eb0873bccae8aeaf1a2647242bffa35", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "de5f988c142a3aa4ec18b85a4ec34a2390b65b24f02385c1144252ff6ff8ee75"},
"ecto_sql": {:hex, :ecto_sql, "3.9.2", "34227501abe92dba10d9c3495ab6770e75e79b836d114c41108a4bf2ce200ad5", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.9.2", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "1eb5eeb4358fdbcd42eac11c1fbd87e3affd7904e639d77903c1358b2abd3f70"},
"elixir_make": {:hex, :elixir_make, "0.6.3", "bc07d53221216838d79e03a8019d0839786703129599e9619f4ab74c8c096eac", [:mix], [], "hexpm", "f5cbd651c5678bcaabdbb7857658ee106b12509cd976c2c2fca99688e1daf716"},
- "elixir_uuid": {:hex, :elixir_uuid, "1.2.1", "dce506597acb7e6b0daeaff52ff6a9043f5919a4c3315abb4143f0b00378c097", [:mix], [], "hexpm", "f7eba2ea6c3555cea09706492716b0d87397b88946e6380898c2889d68585752"},
"envy": {:hex, :envy, "1.1.1", "0bc9bd654dec24fcdf203f7c5aa1b8f30620f12cfb28c589d5e9c38fe1b07475", [:mix], [], "hexpm", "7061eb1a47415fd757145d8dec10dc0b1e48344960265cb108f194c4252c3a89"},
"erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"},
"eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"},
@@ -58,7 +57,6 @@
"hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"},
"html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"},
"httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"},
- "hut": {:hex, :hut, "1.3.0", "71f2f054e657c03f959cf1acc43f436ea87580696528ca2a55c8afb1b06c85e7", [:"erlang.mk", :rebar, :rebar3], [], "hexpm", "7e15d28555d8a1f2b5a3a931ec120af0753e4853a4c66053db354f35bf9ab563"},
"idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
"jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"},
"jsx": {:hex, :jsx, "2.8.3", "a05252d381885240744d955fbe3cf810504eb2567164824e19303ea59eef62cf", [:mix, :rebar3], [], "hexpm", "fc3499fed7a726995aa659143a248534adc754ebd16ccd437cd93b649a95091f"},
diff --git a/priv/ingest_repo/migrations/.formatter.exs b/priv/ingest_repo/migrations/.formatter.exs
new file mode 100644
index 0000000000..49f9151ed2
--- /dev/null
+++ b/priv/ingest_repo/migrations/.formatter.exs
@@ -0,0 +1,4 @@
+[
+ import_deps: [:ecto_sql],
+ inputs: ["*.exs"]
+]
diff --git a/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs b/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs
index 55fcb39ac4..6f329cdfb9 100644
--- a/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs
+++ b/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs
@@ -8,8 +8,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateEventsAndSessions do
defp create_events() do
create_if_not_exists table(:events,
- engine:
- "MergeTree() PARTITION BY toYYYYMM(timestamp) ORDER BY (domain, toDate(timestamp), user_id) SETTINGS index_granularity = 8192"
+ primary_key: false,
+ engine: "MergeTree",
+ options:
+ "PARTITION BY toYYYYMM(timestamp) ORDER BY (domain, toDate(timestamp), user_id) SETTINGS index_granularity = 8192"
) do
add(:name, :string)
add(:domain, :string)
@@ -30,15 +32,17 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateEventsAndSessions do
defp create_sessions() do
create_if_not_exists table(:sessions,
- engine:
- "CollapsingMergeTree(sign) PARTITION BY toYYYYMM(start) ORDER BY (domain, toDate(start), user_id, session_id) SETTINGS index_granularity = 8192"
+ primary_key: false,
+ engine: "CollapsingMergeTree(sign)",
+ options:
+ "PARTITION BY toYYYYMM(start) ORDER BY (domain, toDate(start), user_id, session_id) SETTINGS index_granularity = 8192"
) do
add(:session_id, :UInt64)
add(:sign, :Int8)
add(:domain, :string)
add(:user_id, :UInt64)
add(:hostname, :string)
- add(:is_bounce, :boolean)
+ add(:is_bounce, :UInt8)
add(:entry_page, :string)
add(:exit_page, :string)
add(:pageviews, :integer)
diff --git a/priv/ingest_repo/migrations/20201020083739_add_event_metadata.exs b/priv/ingest_repo/migrations/20201020083739_add_event_metadata.exs
index 32f2372dfd..4c1b6d44ad 100644
--- a/priv/ingest_repo/migrations/20201020083739_add_event_metadata.exs
+++ b/priv/ingest_repo/migrations/20201020083739_add_event_metadata.exs
@@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.AddEventMetadata do
def change do
alter table(:events) do
- add :meta, {:nested, {{:key, :string}, {:value, :string}}}
+ add :meta, :"Nested(key String, value String)"
end
end
end
diff --git a/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs b/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs
index 33bc8b23c4..5c7b72a875 100644
--- a/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs
+++ b/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs
@@ -3,7 +3,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
def change do
create_if_not_exists table(:imported_visitors,
- engine: "MergeTree() ORDER BY (site_id, date)"
+ primary_key: false,
+ engine: "MergeTree",
+ options: "ORDER BY (site_id, date)"
) do
add(:site_id, :UInt64)
add(:date, :date)
@@ -15,7 +17,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
end
create_if_not_exists table(:imported_sources,
- engine: "MergeTree() ORDER BY (site_id, date, source)"
+ primary_key: false,
+ engine: "MergeTree",
+ options: "ORDER BY (site_id, date, source)"
) do
add(:site_id, :UInt64)
add(:date, :date)
@@ -31,7 +35,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
end
create_if_not_exists table(:imported_pages,
- engine: "MergeTree() ORDER BY (site_id, date, hostname, page)"
+ primary_key: false,
+ engine: "MergeTree",
+ options: "ORDER BY (site_id, date, hostname, page)"
) do
add(:site_id, :UInt64)
add(:date, :date)
@@ -44,7 +50,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
end
create_if_not_exists table(:imported_entry_pages,
- engine: "MergeTree() ORDER BY (site_id, date, entry_page)"
+ primary_key: false,
+ engine: "MergeTree",
+ options: "ORDER BY (site_id, date, entry_page)"
) do
add(:site_id, :UInt64)
add(:date, :date)
@@ -56,7 +64,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
end
create_if_not_exists table(:imported_exit_pages,
- engine: "MergeTree() ORDER BY (site_id, date, exit_page)"
+ primary_key: false,
+ engine: "MergeTree",
+ options: "ORDER BY (site_id, date, exit_page)"
) do
add(:site_id, :UInt64)
add(:date, :date)
@@ -66,7 +76,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
end
create_if_not_exists table(:imported_locations,
- engine: "MergeTree() ORDER BY (site_id, date, country, region, city)"
+ primary_key: false,
+ engine: "MergeTree",
+ options: "ORDER BY (site_id, date, country, region, city)"
) do
add(:site_id, :UInt64)
add(:date, :date)
@@ -80,7 +92,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
end
create_if_not_exists table(:imported_devices,
- engine: "MergeTree() ORDER BY (site_id, date, device)"
+ primary_key: false,
+ engine: "MergeTree",
+ options: "ORDER BY (site_id, date, device)"
) do
add(:site_id, :UInt64)
add(:date, :date)
@@ -92,7 +106,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
end
create_if_not_exists table(:imported_browsers,
- engine: "MergeTree() ORDER BY (site_id, date, browser)"
+ primary_key: false,
+ engine: "MergeTree",
+ options: "ORDER BY (site_id, date, browser)"
) do
add(:site_id, :UInt64)
add(:date, :date)
@@ -104,7 +120,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
end
create_if_not_exists table(:imported_operating_systems,
- engine: "MergeTree() ORDER BY (site_id, date, operating_system)"
+ primary_key: false,
+ engine: "MergeTree",
+ options: "ORDER BY (site_id, date, operating_system)"
) do
add(:site_id, :UInt64)
add(:date, :date)
diff --git a/priv/ingest_repo/migrations/20220404123000_add_entry_props_to_session.exs b/priv/ingest_repo/migrations/20220404123000_add_entry_props_to_session.exs
index aa951a2d6e..2c80ad8d68 100644
--- a/priv/ingest_repo/migrations/20220404123000_add_entry_props_to_session.exs
+++ b/priv/ingest_repo/migrations/20220404123000_add_entry_props_to_session.exs
@@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.AddEntryPropsToSession do
def change do
alter table(:sessions) do
- add :"entry.meta", {:nested, {{:key, :string}, {:value, :string}}}
+ add :"entry.meta", :"Nested(key String, value String)"
end
end
end
diff --git a/priv/ingest_repo/migrations/20220422075510_add_entry_props.exs b/priv/ingest_repo/migrations/20220422075510_add_entry_props.exs
index a2907a70ab..ce45f6e3e6 100644
--- a/priv/ingest_repo/migrations/20220422075510_add_entry_props.exs
+++ b/priv/ingest_repo/migrations/20220422075510_add_entry_props.exs
@@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.AddEntryProps do
def change do
alter table(:sessions) do
- add(:entry_meta, {:nested, {{:key, :string}, {:value, :string}}})
+ add(:entry_meta, :"Nested(key String, value String)")
end
end
end
diff --git a/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs b/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs
index 231db40dc3..75596571c0 100644
--- a/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs
+++ b/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs
@@ -3,8 +3,11 @@ defmodule Plausible.IngestRepo.Migrations.CreateIngestCountersTable do
def change do
create_if_not_exists table(:ingest_counters,
- engine: "SummingMergeTree(value) ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket))") do
-
+ primary_key: false,
+ engine: "SummingMergeTree(value)",
+ options:
+ "ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket))"
+ ) do
add(:event_timebucket, :utc_datetime)
add(:domain, :"LowCardinality(String)")
add(:site_id, :"Nullable(UInt64)")
diff --git a/priv/ingest_repo/structure.sql b/priv/ingest_repo/structure.sql
new file mode 100644
index 0000000000..550aaa7d51
--- /dev/null
+++ b/priv/ingest_repo/structure.sql
@@ -0,0 +1,242 @@
+CREATE TABLE plausible_events_db.sessions
+(
+ `session_id` UInt64,
+ `sign` Int8,
+ `domain` String,
+ `user_id` UInt64,
+ `hostname` String,
+ `is_bounce` UInt8,
+ `entry_page` String,
+ `exit_page` String,
+ `pageviews` Int32,
+ `events` Int32,
+ `duration` UInt32,
+ `referrer` String,
+ `referrer_source` String,
+ `country_code` LowCardinality(FixedString(2)),
+ `screen_size` LowCardinality(String),
+ `operating_system` LowCardinality(String),
+ `browser` LowCardinality(String),
+ `start` DateTime,
+ `timestamp` DateTime,
+ `utm_medium` String,
+ `utm_source` String,
+ `utm_campaign` String,
+ `browser_version` LowCardinality(String),
+ `operating_system_version` LowCardinality(String),
+ `subdivision1_code` LowCardinality(String),
+ `subdivision2_code` LowCardinality(String),
+ `city_geoname_id` UInt32,
+ `utm_content` String,
+ `utm_term` String,
+ `transferred_from` String,
+ `entry_meta.key` Array(String),
+ `entry_meta.value` Array(String)
+)
+ENGINE = CollapsingMergeTree(sign)
+PARTITION BY toYYYYMM(start)
+ORDER BY (domain, toDate(start), user_id, session_id)
+SAMPLE BY user_id
+SETTINGS index_granularity = 8192;
+
+CREATE TABLE plausible_events_db.ingest_counters
+(
+ `event_timebucket` DateTime,
+ `domain` LowCardinality(String),
+ `site_id` Nullable(UInt64),
+ `metric` LowCardinality(String),
+ `value` UInt64
+)
+ENGINE = SummingMergeTree(value)
+ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket))
+SETTINGS index_granularity = 8192;
+
+CREATE TABLE plausible_events_db.imported_visitors
+(
+ `site_id` UInt64,
+ `date` Date,
+ `visitors` UInt64,
+ `pageviews` UInt64,
+ `bounces` UInt64,
+ `visits` UInt64,
+ `visit_duration` UInt64
+)
+ENGINE = MergeTree
+ORDER BY (site_id, date)
+SETTINGS index_granularity = 8192;
+
+CREATE TABLE plausible_events_db.imported_sources
+(
+ `site_id` UInt64,
+ `date` Date,
+ `source` String,
+ `utm_medium` String,
+ `utm_campaign` String,
+ `utm_content` String,
+ `utm_term` String,
+ `visitors` UInt64,
+ `visits` UInt64,
+ `visit_duration` UInt64,
+ `bounces` UInt32
+)
+ENGINE = MergeTree
+ORDER BY (site_id, date, source)
+SETTINGS index_granularity = 8192;
+
+CREATE TABLE plausible_events_db.imported_pages
+(
+ `site_id` UInt64,
+ `date` Date,
+ `hostname` String,
+ `page` String,
+ `visitors` UInt64,
+ `pageviews` UInt64,
+ `exits` UInt64,
+ `time_on_page` UInt64
+)
+ENGINE = MergeTree
+ORDER BY (site_id, date, hostname, page)
+SETTINGS index_granularity = 8192;
+
+CREATE TABLE plausible_events_db.imported_operating_systems
+(
+ `site_id` UInt64,
+ `date` Date,
+ `operating_system` String,
+ `visitors` UInt64,
+ `visits` UInt64,
+ `visit_duration` UInt64,
+ `bounces` UInt32
+)
+ENGINE = MergeTree
+ORDER BY (site_id, date, operating_system)
+SETTINGS index_granularity = 8192;
+
+CREATE TABLE plausible_events_db.imported_locations
+(
+ `site_id` UInt64,
+ `date` Date,
+ `country` String,
+ `region` String,
+ `city` UInt64,
+ `visitors` UInt64,
+ `visits` UInt64,
+ `visit_duration` UInt64,
+ `bounces` UInt32
+)
+ENGINE = MergeTree
+ORDER BY (site_id, date, country, region, city)
+SETTINGS index_granularity = 8192;
+
+CREATE TABLE plausible_events_db.imported_exit_pages
+(
+ `site_id` UInt64,
+ `date` Date,
+ `exit_page` String,
+ `visitors` UInt64,
+ `exits` UInt64
+)
+ENGINE = MergeTree
+ORDER BY (site_id, date, exit_page)
+SETTINGS index_granularity = 8192;
+
+CREATE TABLE plausible_events_db.imported_entry_pages
+(
+ `site_id` UInt64,
+ `date` Date,
+ `entry_page` String,
+ `visitors` UInt64,
+ `entrances` UInt64,
+ `visit_duration` UInt64,
+ `bounces` UInt32
+)
+ENGINE = MergeTree
+ORDER BY (site_id, date, entry_page)
+SETTINGS index_granularity = 8192;
+
+CREATE TABLE plausible_events_db.imported_devices
+(
+ `site_id` UInt64,
+ `date` Date,
+ `device` String,
+ `visitors` UInt64,
+ `visits` UInt64,
+ `visit_duration` UInt64,
+ `bounces` UInt32
+)
+ENGINE = MergeTree
+ORDER BY (site_id, date, device)
+SETTINGS index_granularity = 8192;
+
+CREATE TABLE plausible_events_db.imported_browsers
+(
+ `site_id` UInt64,
+ `date` Date,
+ `browser` String,
+ `visitors` UInt64,
+ `visits` UInt64,
+ `visit_duration` UInt64,
+ `bounces` UInt32
+)
+ENGINE = MergeTree
+ORDER BY (site_id, date, browser)
+SETTINGS index_granularity = 8192;
+
+CREATE TABLE plausible_events_db.events
+(
+ `name` String,
+ `domain` String,
+ `user_id` UInt64,
+ `session_id` UInt64,
+ `hostname` String,
+ `pathname` String,
+ `referrer` String,
+ `referrer_source` String,
+ `country_code` LowCardinality(FixedString(2)),
+ `screen_size` LowCardinality(String),
+ `operating_system` LowCardinality(String),
+ `browser` LowCardinality(String),
+ `timestamp` DateTime,
+ `utm_medium` String,
+ `utm_source` String,
+ `utm_campaign` String,
+ `meta.key` Array(String),
+ `meta.value` Array(String),
+ `browser_version` LowCardinality(String),
+ `operating_system_version` LowCardinality(String),
+ `subdivision1_code` LowCardinality(String),
+ `subdivision2_code` LowCardinality(String),
+ `city_geoname_id` UInt32,
+ `utm_content` String,
+ `utm_term` String,
+ `transferred_from` String
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMM(timestamp)
+ORDER BY (domain, toDate(timestamp), user_id)
+SAMPLE BY user_id
+SETTINGS index_granularity = 8192;
+
+CREATE TABLE plausible_events_db.schema_migrations
+(
+ `version` Int64,
+ `inserted_at` DateTime
+)
+ENGINE = TinyLog;
+
+INSERT INTO "plausible_events_db"."schema_migrations" (version, inserted_at) VALUES
+(20200915070607,'2023-03-08 10:03:33'),
+(20200918075025,'2023-03-08 10:03:33'),
+(20201020083739,'2023-03-08 10:03:33'),
+(20201106125234,'2023-03-08 10:03:33'),
+(20210323130440,'2023-03-08 10:03:33'),
+(20210712214034,'2023-03-08 10:03:33'),
+(20211017093035,'2023-03-08 10:03:33'),
+(20211112130238,'2023-03-08 10:03:33'),
+(20220310104931,'2023-03-08 10:03:33'),
+(20220404123000,'2023-03-08 10:03:33'),
+(20220421161259,'2023-03-08 10:03:33'),
+(20220422075510,'2023-03-08 10:03:33'),
+(20230124140348,'2023-03-08 10:03:33'),
+(20230210140348,'2023-03-08 10:03:33'),
+(20230214114402,'2023-03-08 10:03:33');
diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs
index 065749e1f0..34fbf052c6 100644
--- a/test/plausible/imported/imported_test.exs
+++ b/test/plausible/imported/imported_test.exs
@@ -7,7 +7,7 @@ defmodule Plausible.ImportedTest do
defp import_data(ga_data, site_id, table_name) do
ga_data
|> Plausible.Imported.from_google_analytics(site_id, table_name)
- |> then(&Plausible.IngestRepo.insert_all(table_name, &1))
+ |> then(&Plausible.Google.Buffer.insert_all(table_name, &1))
end
describe "Parse and import third party data fetched from Google Analytics" do
diff --git a/test/plausible/ingestion/counters_test.exs b/test/plausible/ingestion/counters_test.exs
index bf7d14c3e6..19c73c9ce8 100644
--- a/test/plausible/ingestion/counters_test.exs
+++ b/test/plausible/ingestion/counters_test.exs
@@ -37,8 +37,8 @@ defmodule Plausible.Ingestion.CountersTest do
end
test "the database eventually sums the records within 1-minute buckets", %{test: test} do
- # Testing if the database works is an unfunny way of integration testing,
- # but on the upside it's quite straight-forward way of testing if the
+ # Testing if the database works is an unfunny way of integration testing,
+ # but on the upside it's quite straight-forward way of testing if the
# 1-minute bucket rollups are applied when dumping the records that are
# originally aggregated with 10s windows.
on_exit(:detach, fn ->
diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
index 73f60fbade..5a719282a4 100644
--- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
@@ -39,6 +39,25 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert plot == [1] ++ zeroes ++ [1]
end
+ test "displays visitors for a day with imported data", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-31 00:00:00]),
+ build(:imported_visitors, date: ~D[2021-01-01]),
+ build(:imported_visitors, date: ~D[2021-01-31])
+ ])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&with_imported=true"
+ )
+
+ assert %{"plot" => plot, "imported_source" => "Google Analytics"} = json_response(conn, 200)
+
+ assert plot == [2] ++ List.duplicate(0, 23)
+ end
+
test "displays hourly stats in configured timezone", %{conn: conn, user: user} do
# UTC+1
site =
@@ -60,7 +79,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert %{"plot" => plot} = json_response(conn, 200)
- zeroes = Stream.repeatedly(fn -> 0 end) |> Stream.take(22) |> Enum.into([])
+ zeroes = List.duplicate(0, 22)
# Expecting pageview to show at 1am CET
assert plot == [0, 1] ++ zeroes
@@ -108,6 +127,26 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert Enum.sum(plot) == 4
end
+ test "displays visitors for a month with only imported data", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:imported_visitors, date: ~D[2021-01-01]),
+ build(:imported_visitors, date: ~D[2021-01-31])
+ ])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true"
+ )
+
+ assert %{"plot" => plot, "imported_source" => "Google Analytics"} = json_response(conn, 200)
+
+ assert Enum.count(plot) == 31
+ assert List.first(plot) == 1
+ assert List.last(plot) == 1
+ assert Enum.sum(plot) == 2
+ end
+
test "displays visitors for a month with imported data and filter", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, timestamp: ~N[2021-01-01 00:00:00], pathname: "/pageA"),
@@ -154,6 +193,26 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert Enum.sum(plot) == 4
end
+ test "displays visitors for 6 months with only imported data", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:imported_visitors, date: ~D[2021-01-01]),
+ build(:imported_visitors, date: ~D[2021-06-30])
+ ])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=6mo&date=2021-06-30&with_imported=true"
+ )
+
+ assert %{"plot" => plot} = json_response(conn, 200)
+
+ assert Enum.count(plot) == 6
+ assert List.first(plot) == 1
+ assert List.last(plot) == 1
+ assert Enum.sum(plot) == 2
+ end
+
test "displays visitors for 12 months with imported data", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
@@ -176,6 +235,26 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert Enum.sum(plot) == 4
end
+ test "displays visitors for 12 months with only imported data", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:imported_visitors, date: ~D[2021-01-01]),
+ build(:imported_visitors, date: ~D[2021-12-31])
+ ])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=12mo&date=2021-12-31&with_imported=true"
+ )
+
+ assert %{"plot" => plot} = json_response(conn, 200)
+
+ assert Enum.count(plot) == 12
+ assert List.first(plot) == 1
+ assert List.last(plot) == 1
+ assert Enum.sum(plot) == 2
+ end
+
test "displays visitors for calendar year with imported data", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
@@ -198,6 +277,26 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert Enum.sum(plot) == 4
end
+ test "displays visitors for calendar year with only imported data", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:imported_visitors, date: ~D[2021-01-01]),
+ build(:imported_visitors, date: ~D[2021-12-31])
+ ])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=year&date=2021-12-31&with_imported=true"
+ )
+
+ assert %{"plot" => plot} = json_response(conn, 200)
+
+ assert Enum.count(plot) == 12
+ assert List.first(plot) == 1
+ assert List.last(plot) == 1
+ assert Enum.sum(plot) == 2
+ end
+
test "displays visitors for all time with just native data", %{conn: conn, site: site} do
use Plausible.Repo
@@ -292,6 +391,26 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert List.last(plot) == 2
assert Enum.sum(plot) == 4
end
+
+ test "displays pageviews for a month with only imported data", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:imported_visitors, date: ~D[2021-01-01]),
+ build(:imported_visitors, date: ~D[2021-01-31])
+ ])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=pageviews&with_imported=true"
+ )
+
+ assert %{"plot" => plot} = json_response(conn, 200)
+
+ assert Enum.count(plot) == 31
+ assert List.first(plot) == 1
+ assert List.last(plot) == 1
+ assert Enum.sum(plot) == 2
+ end
end
describe "GET /api/stats/main-graph - bounce_rate plot" do
@@ -337,6 +456,25 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert List.first(plot) == 50
assert List.last(plot) == 100
end
+
+ test "displays bounce rate for a month with only imported data", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:imported_visitors, visits: 1, bounces: 0, date: ~D[2021-01-01]),
+ build(:imported_visitors, visits: 1, bounces: 1, date: ~D[2021-01-31])
+ ])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=bounce_rate&with_imported=true"
+ )
+
+ assert %{"plot" => plot} = json_response(conn, 200)
+
+ assert Enum.count(plot) == 31
+ assert List.first(plot) == 0
+ assert List.last(plot) == 100
+ end
end
describe "GET /api/stats/main-graph - visit_duration plot" do
@@ -387,6 +525,23 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert Enum.count(plot) == 31
assert List.first(plot) == 200
end
+
+ test "displays visit_duration for a month with only imported data", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:imported_visitors, visits: 1, visit_duration: 100, date: ~D[2021-01-01])
+ ])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=visit_duration&with_imported=true"
+ )
+
+ assert %{"plot" => plot} = json_response(conn, 200)
+
+ assert Enum.count(plot) == 31
+ assert List.first(plot) == 100
+ end
end
describe "GET /api/stats/main-graph - varying intervals" do
diff --git a/test/support/factory.ex b/test/support/factory.ex
index 86f5d5719a..629cdbb075 100644
--- a/test/support/factory.ex
+++ b/test/support/factory.ex
@@ -44,26 +44,12 @@ defmodule Plausible.Factory do
user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
hostname: hostname,
domain: hostname,
- referrer: "",
- referrer_source: "",
- utm_medium: "",
- utm_source: "",
- utm_campaign: "",
- utm_content: "",
- utm_term: "",
entry_page: "/",
pageviews: 1,
events: 1,
- duration: 0,
start: Timex.now(),
timestamp: Timex.now(),
- is_bounce: false,
- browser: "",
- browser_version: "",
- country_code: "",
- screen_size: "",
- operating_system: "",
- operating_system_version: ""
+ is_bounce: false
}
end
@@ -85,22 +71,7 @@ defmodule Plausible.Factory do
pathname: "/",
timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second),
user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
- session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
- referrer: "",
- referrer_source: "",
- utm_medium: "",
- utm_source: "",
- utm_campaign: "",
- utm_content: "",
- utm_term: "",
- browser: "",
- browser_version: "",
- country_code: "",
- screen_size: "",
- operating_system: "",
- operating_system_version: "",
- "meta.key": [],
- "meta.value": []
+ session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate())
}
end
diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex
index bb506edc50..a129fd2ade 100644
--- a/test/support/test_utils.ex
+++ b/test/support/test_utils.ex
@@ -73,28 +73,38 @@ defmodule Plausible.TestUtils do
def create_pageviews(pageviews) do
pageviews =
Enum.map(pageviews, fn pageview ->
- Factory.build(:pageview, pageview) |> Map.from_struct() |> Map.delete(:__meta__)
+ Factory.build(:pageview, pageview)
+ |> Map.from_struct()
+ |> Map.delete(:__meta__)
+ |> update_in([:timestamp], &to_naive_truncate/1)
end)
- Plausible.IngestRepo.insert_all("events", pageviews)
+ Plausible.IngestRepo.insert_all(Plausible.ClickhouseEvent, pageviews)
end
def create_events(events) do
events =
Enum.map(events, fn event ->
- Factory.build(:event, event) |> Map.from_struct() |> Map.delete(:__meta__)
+ Factory.build(:event, event)
+ |> Map.from_struct()
+ |> Map.delete(:__meta__)
+ |> update_in([:timestamp], &to_naive_truncate/1)
end)
- Plausible.IngestRepo.insert_all("events", events)
+ Plausible.IngestRepo.insert_all(Plausible.ClickhouseEvent, events)
end
def create_sessions(sessions) do
sessions =
Enum.map(sessions, fn session ->
- Factory.build(:ch_session, session) |> Map.from_struct() |> Map.delete(:__meta__)
+ Factory.build(:ch_session, session)
+ |> Map.from_struct()
+ |> Map.delete(:__meta__)
+ |> update_in([:timestamp], &to_naive_truncate/1)
+ |> update_in([:start], &to_naive_truncate/1)
end)
- Plausible.IngestRepo.insert_all("sessions", sessions)
+ Plausible.IngestRepo.insert_all(Plausible.ClickhouseSession, sessions)
end
def log_in(%{user: user, conn: conn}) do
@@ -136,7 +146,17 @@ defmodule Plausible.TestUtils do
def populate_stats(events) do
{native, imported} =
- Enum.split_with(events, fn event ->
+ events
+ |> Enum.map(fn event ->
+ case event do
+ %{timestamp: timestamp} ->
+ %{event | timestamp: to_naive_truncate(timestamp)}
+
+ _other ->
+ event
+ end
+ end)
+ |> Enum.split_with(fn event ->
case event do
%Plausible.ClickhouseEvent{} ->
true
@@ -168,7 +188,7 @@ defmodule Plausible.TestUtils do
defp populate_imported_stats(events) do
Enum.group_by(events, &Map.fetch!(&1, :table), &Map.delete(&1, :table))
- |> Enum.map(fn {table, events} -> Plausible.IngestRepo.insert_all(table, events) end)
+ |> Enum.map(fn {table, events} -> Plausible.Google.Buffer.insert_all(table, events) end)
end
def relative_time(shifts) do
@@ -177,6 +197,14 @@ defmodule Plausible.TestUtils do
|> NaiveDateTime.truncate(:second)
end
+ def to_naive_truncate(%DateTime{} = dt) do
+ to_naive_truncate(DateTime.to_naive(dt))
+ end
+
+ def to_naive_truncate(%NaiveDateTime{} = naive) do
+ NaiveDateTime.truncate(naive, :second)
+ end
+
def eventually(expectation, wait_time_ms \\ 50, retries \\ 10) do
Enum.reduce_while(1..retries, nil, fn attempt, _acc ->
case expectation.() do