Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Channels: Speed up clickhouse calculations (#4789)
Karl-Aksel Puulmannd620432227b0
diff --git a/lib/mix/tasks/clean_clickhouse.ex b/lib/mix/tasks/clean_clickhouse.exindex 16731c9764..2430272e49 100644--- a/lib/mix/tasks/clean_clickhouse.ex+++ b/lib/mix/tasks/clean_clickhouse.ex@@ -6,7 +6,18 @@ defmodule Mix.Tasks.CleanClickhouse dodef run(_) do%{rows: rows} = IngestRepo.query!("show tables")tables = Enum.map(rows, fn [table] -> table end)- to_truncate = tables -- ["schema_migrations", "location_data", "location_data_dict"]++ to_truncate =+ tables --+ [+ "schema_migrations",+ "location_data",+ "location_data_dict",+ "acquisition_channel_source_category",+ "acquisition_channel_source_category_dict",+ "acquisition_channel_paid_sources",+ "acquisition_channel_paid_sources_dict"+ ]Enum.each(to_truncate, fn table ->IngestRepo.query!("truncate #{table}")diff --git a/lib/plausible/data_migration.ex b/lib/plausible/data_migration.exindex d99e2e6527..48f847261e 100644--- a/lib/plausible/data_migration.ex+++ b/lib/plausible/data_migration.ex@@ -84,7 +84,7 @@ defmodule Plausible.DataMigration do|> String.split(";", trim: true)|> Enum.with_index(1)|> Enum.reduce_while(:ok, fn {query, index}, _ ->- case do_run("name-#{index}", query, options) do+ case do_run("#{name}-#{index}", query, options) do{:ok, _} -> {:cont, :ok}error -> {:halt, error}enddiff --git a/lib/plausible/data_migration/acquisition_channel.ex b/lib/plausible/data_migration/acquisition_channel.exindex efd6e91ec8..48992140ca 100644--- a/lib/plausible/data_migration/acquisition_channel.ex+++ b/lib/plausible/data_migration/acquisition_channel.ex@@ -1,40 +1,26 @@defmodule Plausible.DataMigration.AcquisitionChannel do@moduledoc """- Creates functions to calculate acquisition channel in ClickHouse+ Creates dictionaries and functions to calculate acquisition channel in ClickHouseSQL files available at: priv/data_migrations/AcquisitionChannel/sql"""use Plausible.DataMigration, dir: "AcquisitionChannel", repo: Plausible.IngestRepodef run(opts \\ []) do- source_categories =- Plausible.Ingestion.Acquisition.source_categories()- |> invert_map()-on_cluster_statement = Plausible.MigrationUtils.on_cluster_statement("sessions_v2")run_sql_multi("acquisition_channel_functions",[- on_cluster_statement: on_cluster_statement+ on_cluster_statement: on_cluster_statement,+ dictionary_connection_params: Plausible.MigrationUtils.dictionary_connection_params()],params: %{- "source_category_shopping" => source_categories["SOURCE_CATEGORY_SHOPPING"],- "source_category_social" => source_categories["SOURCE_CATEGORY_SOCIAL"],- "source_category_video" => source_categories["SOURCE_CATEGORY_VIDEO"],- "source_category_search" => source_categories["SOURCE_CATEGORY_SEARCH"],- "source_category_email" => source_categories["SOURCE_CATEGORY_EMAIL"],+ "source_categories" =>+ Plausible.Ingestion.Acquisition.source_categories() |> Map.to_list(),"paid_sources" => Plausible.Ingestion.Source.paid_sources()},quiet: Keyword.get(opts, :quiet, false))end-- defp invert_map(source_categories) do- source_categories- |> Enum.group_by(- fn {_source, category} -> category end,- fn {source, _category} -> source end- )- endenddiff --git a/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_functions.sql.eex b/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_functions.sql.eexindex 439b5cf097..48250998b6 100644--- a/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_functions.sql.eex+++ b/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_functions.sql.eex@@ -1,26 +1,64 @@+CREATE OR REPLACE TABLE acquisition_channel_source_category(referrer_source String, category LowCardinality(String))+<%= @on_cluster_statement %>+Engine = MergeTree()+ORDER BY referrer_source+AS+SELECT t.1 AS referrer_source, t.2 AS category+FROM (+ SELECT arrayJoin({source_categories:Array(Tuple(String, String))}) AS t+);++CREATE OR REPLACE DICTIONARY acquisition_channel_source_category_dict+<%= @on_cluster_statement %>+(+ `referrer_source` String,+ `category` String+)+PRIMARY KEY referrer_source+SOURCE(CLICKHOUSE(TABLE acquisition_channel_source_category <%= @dictionary_connection_params %>))+LIFETIME(0)+LAYOUT(hashed());++CREATE OR REPLACE TABLE acquisition_channel_paid_sources(referrer_source String)+<%= @on_cluster_statement %>+Engine = MergeTree()+ORDER BY referrer_source+AS+SELECT arrayJoin({paid_sources:Array(String)}) AS referrer_source;++CREATE OR REPLACE DICTIONARY acquisition_channel_paid_sources_dict+<%= @on_cluster_statement %>+(+ `referrer_source` String+)+PRIMARY KEY referrer_source+SOURCE(CLICKHOUSE(TABLE acquisition_channel_paid_sources <%= @dictionary_connection_params %>))+LIFETIME(0)+LAYOUT(hashed());+CREATE OR REPLACE FUNCTION acquisition_channel_has_category_shopping <%= @on_cluster_statement %> AS(referrer_source) ->- has({source_category_shopping:Array(String)}, referrer_source);+ dictGet('acquisition_channel_source_category_dict', 'category', referrer_source) = 'SOURCE_CATEGORY_SHOPPING';CREATE OR REPLACE FUNCTION acquisition_channel_has_category_social <%= @on_cluster_statement %> AS(referrer_source) ->- has({source_category_social:Array(String)}, referrer_source);+ dictGet('acquisition_channel_source_category_dict', 'category', referrer_source) = 'SOURCE_CATEGORY_SOCIAL';CREATE OR REPLACE FUNCTION acquisition_channel_has_category_video <%= @on_cluster_statement %> AS(referrer_source) ->- has({source_category_video:Array(String)}, referrer_source);+ dictGet('acquisition_channel_source_category_dict', 'category', referrer_source) = 'SOURCE_CATEGORY_VIDEO';CREATE OR REPLACE FUNCTION acquisition_channel_has_category_search <%= @on_cluster_statement %> AS(referrer_source) ->- has({source_category_search:Array(String)}, referrer_source);+ dictGet('acquisition_channel_source_category_dict', 'category', referrer_source) = 'SOURCE_CATEGORY_SEARCH';CREATE OR REPLACE FUNCTION acquisition_channel_has_category_email <%= @on_cluster_statement %> AS(referrer_source) ->- has({source_category_email:Array(String)}, referrer_source);+ dictGet('acquisition_channel_source_category_dict', 'category', referrer_source) = 'SOURCE_CATEGORY_EMAIL';CREATE OR REPLACE FUNCTION acquisition_channel_paid_utm_source <%= @on_cluster_statement %> AS(referrer_source) ->- has({paid_sources:Array(String)}, referrer_source);+ dictHas('acquisition_channel_paid_sources_dict', referrer_source);CREATE OR REPLACE FUNCTION acquisition_channel_cross_network <%= @on_cluster_statement %> AS(utm_campaign) ->