Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Split Clickhouse pools into Read-Only and Read/Write (dedicated to writes) (#2661)
Adam Rutkowski8f85b110aa43
diff --git a/config/config.exs b/config/config.exsindex a514f79ada..a07f5c9a15 100644--- a/config/config.exs+++ b/config/config.exs@@ -1,7 +1,7 @@import Configconfig :plausible,- ecto_repos: [Plausible.Repo, Plausible.ClickhouseRepo]+ ecto_repos: [Plausible.Repo, Plausible.IngestRepo]config :plausible, PlausibleWeb.Endpoint,pubsub_server: Plausible.PubSub,diff --git a/config/runtime.exs b/config/runtime.exsindex 8e6df2e449..94db325cfc 100644--- a/config/runtime.exs+++ b/config/runtime.exs@@ -81,6 +81,14 @@ ch_db_url ="http://plausible_events_db:8123/plausible_events_db")+{ingest_pool_size, ""} =+ get_var_from_path_or_env(+ config_dir,+ "CLICKHOUSE_INGEST_POOL_SIZE",+ "5"+ )+ |> Integer.parse()+{ch_flush_interval_ms, ""} =config_dir|> get_var_from_path_or_env("CLICKHOUSE_FLUSH_INTERVAL_MS", "5000")@@ -268,12 +276,19 @@ config :plausible, :google,max_buffer_size: get_int_from_path_or_env(config_dir, "GOOGLE_MAX_BUFFER_SIZE", 10_000)config :plausible, Plausible.ClickhouseRepo,+ loggers: [Ecto.LogEntry],+ queue_target: 500,+ queue_interval: 2000,+ url: ch_db_url++config :plausible, Plausible.IngestRepo,loggers: [Ecto.LogEntry],queue_target: 500,queue_interval: 2000,url: ch_db_url,flush_interval_ms: ch_flush_interval_ms,- max_buffer_size: ch_max_buffer_size+ max_buffer_size: ch_max_buffer_size,+ pool_size: ingest_pool_sizecase mailer_adapter do"Bamboo.PostmarkAdapter" ->diff --git a/lib/plausible/application.ex b/lib/plausible/application.exindex 24e85432c4..fa9660d93e 100644--- a/lib/plausible/application.ex+++ b/lib/plausible/application.ex@@ -9,6 +9,7 @@ defmodule Plausible.Application dochildren = [Plausible.Repo,Plausible.ClickhouseRepo,+ Plausible.IngestRepo,{Finch, name: Plausible.Finch, pools: finch_pool_config()},{Phoenix.PubSub, name: Plausible.PubSub},Plausible.Session.Salts,diff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.exindex 67cfa61bf2..cc8f5f206e 100644--- a/lib/plausible/clickhouse_repo.ex+++ b/lib/plausible/clickhouse_repo.ex@@ -1,7 +1,8 @@defmodule Plausible.ClickhouseRepo douse Ecto.Repo,otp_app: :plausible,- adapter: ClickhouseEcto+ adapter: ClickhouseEcto,+ read_only: truedefmacro __using__(_) doquote dodiff --git a/lib/plausible/event/write_buffer.ex b/lib/plausible/event/write_buffer.exindex 6dc3e7ed87..ccb3fcb25f 100644--- a/lib/plausible/event/write_buffer.ex+++ b/lib/plausible/event/write_buffer.ex@@ -2,6 +2,8 @@ defmodule Plausible.Event.WriteBuffer douse GenServerrequire Logger+ alias Plausible.IngestRepo+def start_link(_opts) doGenServer.start_link(__MODULE__, [], name: __MODULE__)end@@ -62,15 +64,15 @@ defmodule Plausible.Event.WriteBuffer doevents ->Logger.info("Flushing #{length(events)} events")events = Enum.map(events, &(Map.from_struct(&1) |> Map.delete(:__meta__)))- Plausible.ClickhouseRepo.insert_all(Plausible.ClickhouseEvent, events)+ IngestRepo.insert_all(Plausible.ClickhouseEvent, events)endenddefp flush_interval_ms() do- Keyword.fetch!(Application.get_env(:plausible, Plausible.ClickhouseRepo), :flush_interval_ms)+ Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :flush_interval_ms)enddefp max_buffer_size() do- Keyword.fetch!(Application.get_env(:plausible, Plausible.ClickhouseRepo), :max_buffer_size)+ Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :max_buffer_size)endenddiff --git a/lib/plausible/google/buffer.ex b/lib/plausible/google/buffer.exindex 9000c4909d..b43160dfad 100644--- a/lib/plausible/google/buffer.ex+++ b/lib/plausible/google/buffer.ex@@ -91,6 +91,6 @@ defmodule Plausible.Google.Buffer doProcess.sleep(1000)Logger.info("Import: Flushing #{length(records)} from #{table_name} buffer")- Plausible.ClickhouseRepo.insert_all(table_name, records)+ Plausible.IngestRepo.insert_all(table_name, records)endenddiff --git a/lib/plausible/ingest_repo.ex b/lib/plausible/ingest_repo.exnew file mode 100644index 0000000000..7928bd9014--- /dev/null+++ b/lib/plausible/ingest_repo.ex@@ -0,0 +1,17 @@+defmodule Plausible.IngestRepo do+ @moduledoc """+ Write-centric Clickhouse access interface+ """++ use Ecto.Repo,+ otp_app: :plausible,+ adapter: ClickhouseEcto++ defmacro __using__(_) do+ quote do+ alias Plausible.IngestRepo+ import Ecto+ import Ecto.Query, only: [from: 1, from: 2]+ end+ end+enddiff --git a/lib/plausible/prom_ex.ex b/lib/plausible/prom_ex.exindex 550b5c05ae..cf903eb013 100644--- a/lib/plausible/prom_ex.ex+++ b/lib/plausible/prom_ex.ex@@ -9,7 +9,7 @@ defmodule Plausible.PromEx doPlugins.Application,Plugins.Beam,{Plugins.Phoenix, router: PlausibleWeb.Router, endpoint: PlausibleWeb.Endpoint},- Plugins.Ecto,+ {Plugins.Ecto, repos: [Plausible.Repo, Plausible.ClickhouseRepo, Plausible.IngestRepo]},Plugins.Oban,Plausible.PromEx.Plugins.PlausibleMetrics]diff --git a/lib/plausible/session/write_buffer.ex b/lib/plausible/session/write_buffer.exindex 563b6d7a82..de23a6802c 100644--- a/lib/plausible/session/write_buffer.ex+++ b/lib/plausible/session/write_buffer.ex@@ -2,6 +2,8 @@ defmodule Plausible.Session.WriteBuffer douse GenServerrequire Logger+ alias Plausible.IngestRepo+def start_link(_opts) doGenServer.start_link(__MODULE__, [], name: __MODULE__)end@@ -67,15 +69,15 @@ defmodule Plausible.Session.WriteBuffer do|> Enum.map(&(Map.from_struct(&1) |> Map.delete(:__meta__)))|> Enum.reverse()- Plausible.ClickhouseRepo.insert_all(Plausible.ClickhouseSession, sessions)+ IngestRepo.insert_all(Plausible.ClickhouseSession, sessions)endenddefp flush_interval_ms() do- Keyword.fetch!(Application.get_env(:plausible, Plausible.ClickhouseRepo), :flush_interval_ms)+ Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :flush_interval_ms)enddefp max_buffer_size() do- Keyword.fetch!(Application.get_env(:plausible, Plausible.ClickhouseRepo), :max_buffer_size)+ Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :max_buffer_size)endenddiff --git a/priv/clickhouse_repo/migrations/20200915070607_create_events_and_sessions.exs b/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exssimilarity index 100%rename from priv/clickhouse_repo/migrations/20200915070607_create_events_and_sessions.exsrename to priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exsdiff --git a/priv/clickhouse_repo/migrations/20200918075025_add_utm_tags.exs b/priv/ingest_repo/migrations/20200918075025_add_utm_tags.exssimilarity index 100%rename from priv/clickhouse_repo/migrations/20200918075025_add_utm_tags.exsrename to priv/ingest_repo/migrations/20200918075025_add_utm_tags.exsdiff --git a/priv/clickhouse_repo/migrations/20201020083739_add_event_metadata.exs b/priv/ingest_repo/migrations/20201020083739_add_event_metadata.exssimilarity index 100%rename from priv/clickhouse_repo/migrations/20201020083739_add_event_metadata.exsrename to priv/ingest_repo/migrations/20201020083739_add_event_metadata.exsdiff --git a/priv/clickhouse_repo/migrations/20201106125234_add_browser_version_and_os_version.exs b/priv/ingest_repo/migrations/20201106125234_add_browser_version_and_os_version.exssimilarity index 100%rename from priv/clickhouse_repo/migrations/20201106125234_add_browser_version_and_os_version.exsrename to priv/ingest_repo/migrations/20201106125234_add_browser_version_and_os_version.exsdiff --git a/priv/clickhouse_repo/migrations/20210323130440_add_sample_by.exs b/priv/ingest_repo/migrations/20210323130440_add_sample_by.exssimilarity index 100%rename from priv/clickhouse_repo/migrations/20210323130440_add_sample_by.exsrename to priv/ingest_repo/migrations/20210323130440_add_sample_by.exsdiff --git a/priv/clickhouse_repo/migrations/20210712214034_add_more_location_details.exs b/priv/ingest_repo/migrations/20210712214034_add_more_location_details.exssimilarity index 100%rename from priv/clickhouse_repo/migrations/20210712214034_add_more_location_details.exsrename to priv/ingest_repo/migrations/20210712214034_add_more_location_details.exsdiff --git a/priv/clickhouse_repo/migrations/20211017093035_add_utm_content_and_term.exs b/priv/ingest_repo/migrations/20211017093035_add_utm_content_and_term.exssimilarity index 100%rename from priv/clickhouse_repo/migrations/20211017093035_add_utm_content_and_term.exsrename to priv/ingest_repo/migrations/20211017093035_add_utm_content_and_term.exsdiff --git a/priv/clickhouse_repo/migrations/20211112130238_create_imported_tables.exs b/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exssimilarity index 100%rename from priv/clickhouse_repo/migrations/20211112130238_create_imported_tables.exsrename to priv/ingest_repo/migrations/20211112130238_create_imported_tables.exsdiff --git a/priv/clickhouse_repo/migrations/20220310104931_add_transferred_from.exs b/priv/ingest_repo/migrations/20220310104931_add_transferred_from.exssimilarity index 100%rename from priv/clickhouse_repo/migrations/20220310104931_add_transferred_from.exsrename to priv/ingest_repo/migrations/20220310104931_add_transferred_from.exsdiff --git a/priv/clickhouse_repo/migrations/20220404123000_add_entry_props_to_session.exs b/priv/ingest_repo/migrations/20220404123000_add_entry_props_to_session.exssimilarity index 100%rename from priv/clickhouse_repo/migrations/20220404123000_add_entry_props_to_session.exsrename to priv/ingest_repo/migrations/20220404123000_add_entry_props_to_session.exsdiff --git a/priv/clickhouse_repo/migrations/20220421161259_remove_entry_props.exs b/priv/ingest_repo/migrations/20220421161259_remove_entry_props.exssimilarity index 100%rename from priv/clickhouse_repo/migrations/20220421161259_remove_entry_props.exsrename to priv/ingest_repo/migrations/20220421161259_remove_entry_props.exsdiff --git a/priv/clickhouse_repo/migrations/20220422075510_add_entry_props.exs b/priv/ingest_repo/migrations/20220422075510_add_entry_props.exssimilarity index 100%rename from priv/clickhouse_repo/migrations/20220422075510_add_entry_props.exsrename to priv/ingest_repo/migrations/20220422075510_add_entry_props.exsdiff --git a/priv/clickhouse_repo/migrations/20230124140348_add_city_name_to_imported_locations.exs b/priv/ingest_repo/migrations/20230124140348_add_city_name_to_imported_locations.exssimilarity index 100%rename from priv/clickhouse_repo/migrations/20230124140348_add_city_name_to_imported_locations.exsrename to priv/ingest_repo/migrations/20230124140348_add_city_name_to_imported_locations.exsdiff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exsindex 04cd02e7f7..e8cefb3303 100644--- a/test/plausible/imported/imported_test.exs+++ b/test/plausible/imported/imported_test.exs@@ -7,7 +7,7 @@ defmodule Plausible.ImportedTest dodefp import_data(ga_data, site_id, table_name) doga_data|> Plausible.Imported.from_google_analytics(site_id, table_name)- |> then(&Plausible.ClickhouseRepo.insert_all(table_name, &1))+ |> then(&Plausible.IngestRepo.insert_all(table_name, &1))enddescribe "Parse and import third party data fetched from Google Analytics" dodiff --git a/test/support/test_utils.ex b/test/support/test_utils.exindex 698c4e11bd..92b9cc18c9 100644--- a/test/support/test_utils.ex+++ b/test/support/test_utils.ex@@ -71,7 +71,7 @@ defmodule Plausible.TestUtils doFactory.build(:pageview, pageview) |> Map.from_struct() |> Map.delete(:__meta__)end)- Plausible.ClickhouseRepo.insert_all("events", pageviews)+ Plausible.IngestRepo.insert_all("events", pageviews)enddef create_events(events) do@@ -80,7 +80,7 @@ defmodule Plausible.TestUtils doFactory.build(:event, event) |> Map.from_struct() |> Map.delete(:__meta__)end)- Plausible.ClickhouseRepo.insert_all("events", events)+ Plausible.IngestRepo.insert_all("events", events)enddef create_sessions(sessions) do@@ -89,7 +89,7 @@ defmodule Plausible.TestUtils doFactory.build(:ch_session, session) |> Map.from_struct() |> Map.delete(:__meta__)end)- Plausible.ClickhouseRepo.insert_all("sessions", sessions)+ Plausible.IngestRepo.insert_all("sessions", sessions)enddef log_in(%{user: user, conn: conn}) do@@ -163,7 +163,7 @@ defmodule Plausible.TestUtils dodefp populate_imported_stats(events) doEnum.group_by(events, &Map.fetch!(&1, :table), &Map.delete(&1, :table))- |> Enum.map(fn {table, events} -> Plausible.ClickhouseRepo.insert_all(table, events) end)+ |> Enum.map(fn {table, events} -> Plausible.IngestRepo.insert_all(table, events) end)enddef relative_time(shifts) do