Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Split clickhouse repos - making the main one read only (#2826)
hq1154ce3a44cbd
diff --git a/config/runtime.exs b/config/runtime.exsindex 3151029a43..50804dd256 100644--- a/config/runtime.exs+++ b/config/runtime.exs@@ -289,7 +289,10 @@ config :plausible, Plausible.ClickhouseRepo,queue_target: 500,queue_interval: 2000,url: ch_db_url,- transport_opts: ch_transport_opts+ transport_opts: ch_transport_opts,+ settings: [+ readonly: 1+ ]config :plausible, Plausible.IngestRepo,loggers: [Ecto.LogEntry],@@ -313,6 +316,14 @@ config :plausible, Plausible.AsyncInsertRepo,wait_for_async_insert: 0]+config :plausible, Plausible.ImportDeletionRepo,+ loggers: [Ecto.LogEntry],+ queue_target: 500,+ queue_interval: 2000,+ url: ch_db_url,+ transport_opts: ch_transport_opts,+ pool_size: 1+case mailer_adapter do"Bamboo.PostmarkAdapter" ->config :plausible, Plausible.Mailer,diff --git a/lib/plausible/application.ex b/lib/plausible/application.exindex 27c1d697f5..b2f75f8f6f 100644--- a/lib/plausible/application.ex+++ b/lib/plausible/application.ex@@ -11,6 +11,7 @@ defmodule Plausible.Application doPlausible.ClickhouseRepo,Plausible.IngestRepo,Plausible.AsyncInsertRepo,+ Plausible.ImportDeletionRepo,Plausible.Ingestion.Counters,{Finch, name: Plausible.Finch, pools: finch_pool_config()},{Phoenix.PubSub, name: Plausible.PubSub},diff --git a/lib/plausible/import_deletion_repo.ex b/lib/plausible/import_deletion_repo.exnew file mode 100644index 0000000000..3c8a012d8a--- /dev/null+++ b/lib/plausible/import_deletion_repo.ex@@ -0,0 +1,17 @@+defmodule Plausible.ImportDeletionRepo do+ @moduledoc """+ A dedicated repo for import related mutations+ """++ use Ecto.Repo,+ otp_app: :plausible,+ adapter: Ecto.Adapters.ClickHouse++ defmacro __using__(_) do+ quote do+ alias Plausible.ImportDeletionRepo+ import Ecto+ import Ecto.Query, only: [from: 1, from: 2]+ end+ end+enddiff --git a/lib/plausible/purge.ex b/lib/plausible/purge.exindex a075108c4c..4288f74b53 100644--- a/lib/plausible/purge.ex+++ b/lib/plausible/purge.ex@@ -17,7 +17,7 @@ defmodule Plausible.Purge dodef delete_imported_stats!(site) doEnum.each(Plausible.Imported.tables(), fn table ->sql = "ALTER TABLE #{table} DELETE WHERE site_id = {$0:UInt64}"- Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, sql, [site.id])+ Ecto.Adapters.SQL.query!(Plausible.ImportDeletionRepo, sql, [site.id])end)clear_stats_start_date!(site)diff --git a/lib/plausible/site/admin.ex b/lib/plausible/site/admin.exindex 032f196624..6333b1d7db 100644--- a/lib/plausible/site/admin.ex+++ b/lib/plausible/site/admin.ex@@ -53,18 +53,6 @@ defmodule Plausible.SiteAdmin do]end- def list_actions(_conn) do- [- transfer_data: %{- name: "Transfer data",- inputs: [- %{name: "domain", title: "to domain", default: nil}- ],- action: fn _conn, sites, params -> transfer_data(sites, params) end- }- ]- end-defp format_date(date) doTimex.format!(date, "{Mshort} {D}, {YYYY}")end@@ -83,58 +71,6 @@ defmodule Plausible.SiteAdmin do|> Enum.join(", ")end- def transfer_data([from_site], params) do- if Plausible.v2?() do- {:error, "Transfers temporarily unspported with v2"}- else- to_site = Repo.get_by(Plausible.Site, domain: params["domain"])-- if to_site do- event_q = event_transfer_query(from_site.domain, to_site.domain)- {:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, event_q, [], timeout: 30_000)-- session_q = session_transfer_query(from_site.domain, to_site.domain)-- {:ok, _} =- Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, session_q, [], timeout: 30_000)-- start_date = Plausible.Stats.Clickhouse.pageview_start_date_local(from_site)-- {:ok, _} =- to_site- |> Plausible.Site.set_stats_start_date(start_date)- |> Plausible.Site.set_native_stats_start_at(from_site.native_stats_start_at)- |> Repo.update()-- :ok- else- {:error, "Cannot transfer to non-existing domain"}- end- end- end-- def transfer_data(_, _), do: {:error, "Please select exactly one site for this action"}-- def session_transfer_query(from_domain, to_domain) do- fields = get_struct_fields(Plausible.ClickhouseSession)-- "INSERT INTO sessions (" <>- stringify_fields(fields) <>- ") SELECT " <>- stringify_fields(fields, to_domain, from_domain) <>- " FROM (SELECT * FROM sessions WHERE domain='#{from_domain}')"- end-- def event_transfer_query(from_domain, to_domain) do- fields = get_struct_fields(Plausible.ClickhouseEvent)-- "INSERT INTO events (" <>- stringify_fields(fields) <>- ") SELECT " <>- stringify_fields(fields, to_domain, from_domain) <>- " FROM (SELECT * FROM events WHERE domain='#{from_domain}')"- end-def get_struct_fields(module) domodule.__struct__()|> Map.drop([:__meta__, :__struct__])@@ -143,19 +79,6 @@ defmodule Plausible.SiteAdmin do|> Enum.sort()end- defp stringify_fields(fields), do: Enum.join(fields, ", ")-- defp stringify_fields(fields, domain_value, transferred_from_value) do- Enum.map(fields, fn field ->- case field do- "domain" -> "'#{domain_value}' as domain"- "transferred_from" -> "'#{transferred_from_value}' as transferred_from"- _ -> field- end- end)- |> stringify_fields()- end-def create_changeset(schema, attrs), do: Plausible.Site.crm_changeset(schema, attrs)def update_changeset(schema, attrs), do: Plausible.Site.crm_changeset(schema, attrs)enddiff --git a/test/plausible/site/admin_test.exs b/test/plausible/site/admin_test.exsdeleted file mode 100644index 99d5429c67..0000000000--- a/test/plausible/site/admin_test.exs+++ /dev/null@@ -1,130 +0,0 @@-defmodule Plausible.SiteAdminTest do- use Plausible.DataCase, async: true-- alias Plausible.{- SiteAdmin,- ClickhouseRepo,- ClickhouseEvent,- ClickhouseSession- }-- describe "transfer_data" do- test "event and session structs remain the same after transfer" do- if Plausible.v2?() do- :ok- else- from_site = insert(:site)- to_site = insert(:site)-- populate_stats(from_site, [build(:pageview)])-- event_before = get_event_by_domain(from_site.domain)- session_before = get_session_by_domain(from_site.domain)-- SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})-- event_after = get_event_by_domain(to_site.domain)- session_after = get_session_by_domain(to_site.domain)-- assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}- assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}- assert session_before == %ClickhouseSession{session_after | transferred_from: ""}- assert event_after.transferred_from == from_site.domain- assert session_after.transferred_from == from_site.domain- end- end-- test "transfers all events and sessions" do- if Plausible.v2?() do- :ok- else- from_site = insert(:site)- to_site = insert(:site)-- populate_stats(from_site, [- build(:pageview, user_id: 123),- build(:event, name: "Signup", user_id: 123),- build(:pageview, user_id: 456),- build(:event, name: "Signup", user_id: 789)- ])-- SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})-- transferred_events =- ClickhouseRepo.all(- from e in Plausible.ClickhouseEvent, where: e.domain == ^to_site.domain- )-- transferred_sessions =- ClickhouseRepo.all(- from e in Plausible.ClickhouseSession, where: e.domain == ^to_site.domain- )-- assert length(transferred_events) == 4- assert length(transferred_sessions) == 3- end- end-- test "updates stats_start_date on site record" do- if Plausible.v2?() do- :ok- else- from_site = insert(:site)- to_site = insert(:site)-- populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])])-- SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})-- assert Repo.reload(to_site).stats_start_date == ~D[2022-01-01]- end- end-- test "updates native_stats_start_date on based on the from_site record" do- if Plausible.v2?() do- :ok- else- from_site = insert(:site, native_stats_start_at: ~N[2022-01-01 01:00:00])- to_site = insert(:site)-- populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])])-- SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})-- assert Repo.reload(to_site).native_stats_start_at == ~N[2022-01-01 01:00:00]- end- end-- test "session_transfer_query" do- actual = SiteAdmin.session_transfer_query("from.com", "to.com")-- expected =- "INSERT INTO sessions (browser, browser_version, city_geoname_id, country_code, domain, duration, entry_meta.key, entry_meta.value, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, duration, entry_meta.key, entry_meta.value, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM sessions WHERE domain='from.com')"-- assert actual == expected- end-- test "event_transfer_query" do- actual = SiteAdmin.event_transfer_query("from.com", "to.com")-- expected =- "INSERT INTO events (browser, browser_version, city_geoname_id, country_code, domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM events WHERE domain='from.com')"-- assert actual == expected- end- end-- defp get_event_by_domain(domain) do- q = from e in Plausible.ClickhouseEvent, where: e.domain == ^domain-- Plausible.ClickhouseRepo.one!(q)- |> Map.drop([:__meta__, :domain])- end-- defp get_session_by_domain(domain) do- q = from s in Plausible.ClickhouseSession, where: s.domain == ^domain-- Plausible.ClickhouseRepo.one!(q)- |> Map.drop([:__meta__, :domain])- end-end