Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Split clickhouse repos - making the main one read only (#2826)

hq1154ce3a44cbd
diff --git a/config/runtime.exs b/config/runtime.exs
index 3151029a43..50804dd256 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -289,7 +289,10 @@ config :plausible, Plausible.ClickhouseRepo,
queue_target: 500,
queue_interval: 2000,
url: ch_db_url,
- transport_opts: ch_transport_opts
+ transport_opts: ch_transport_opts,
+ settings: [
+ readonly: 1
+ ]
config :plausible, Plausible.IngestRepo,
loggers: [Ecto.LogEntry],
@@ -313,6 +316,14 @@ config :plausible, Plausible.AsyncInsertRepo,
wait_for_async_insert: 0
]
+config :plausible, Plausible.ImportDeletionRepo,
+ loggers: [Ecto.LogEntry],
+ queue_target: 500,
+ queue_interval: 2000,
+ url: ch_db_url,
+ transport_opts: ch_transport_opts,
+ pool_size: 1
+
case mailer_adapter do
"Bamboo.PostmarkAdapter" ->
config :plausible, Plausible.Mailer,
diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex
index 27c1d697f5..b2f75f8f6f 100644
--- a/lib/plausible/application.ex
+++ b/lib/plausible/application.ex
@@ -11,6 +11,7 @@ defmodule Plausible.Application do
Plausible.ClickhouseRepo,
Plausible.IngestRepo,
Plausible.AsyncInsertRepo,
+ Plausible.ImportDeletionRepo,
Plausible.Ingestion.Counters,
{Finch, name: Plausible.Finch, pools: finch_pool_config()},
{Phoenix.PubSub, name: Plausible.PubSub},
diff --git a/lib/plausible/import_deletion_repo.ex b/lib/plausible/import_deletion_repo.ex
new file mode 100644
index 0000000000..3c8a012d8a
--- /dev/null
+++ b/lib/plausible/import_deletion_repo.ex
@@ -0,0 +1,17 @@
+defmodule Plausible.ImportDeletionRepo do
+ @moduledoc """
+ A dedicated repo for import related mutations
+ """
+
+ use Ecto.Repo,
+ otp_app: :plausible,
+ adapter: Ecto.Adapters.ClickHouse
+
+ defmacro __using__(_) do
+ quote do
+ alias Plausible.ImportDeletionRepo
+ import Ecto
+ import Ecto.Query, only: [from: 1, from: 2]
+ end
+ end
+end
diff --git a/lib/plausible/purge.ex b/lib/plausible/purge.ex
index a075108c4c..4288f74b53 100644
--- a/lib/plausible/purge.ex
+++ b/lib/plausible/purge.ex
@@ -17,7 +17,7 @@ defmodule Plausible.Purge do
def delete_imported_stats!(site) do
Enum.each(Plausible.Imported.tables(), fn table ->
sql = "ALTER TABLE #{table} DELETE WHERE site_id = {$0:UInt64}"
- Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, sql, [site.id])
+ Ecto.Adapters.SQL.query!(Plausible.ImportDeletionRepo, sql, [site.id])
end)
clear_stats_start_date!(site)
diff --git a/lib/plausible/site/admin.ex b/lib/plausible/site/admin.ex
index 032f196624..6333b1d7db 100644
--- a/lib/plausible/site/admin.ex
+++ b/lib/plausible/site/admin.ex
@@ -53,18 +53,6 @@ defmodule Plausible.SiteAdmin do
]
end
- def list_actions(_conn) do
- [
- transfer_data: %{
- name: "Transfer data",
- inputs: [
- %{name: "domain", title: "to domain", default: nil}
- ],
- action: fn _conn, sites, params -> transfer_data(sites, params) end
- }
- ]
- end
-
defp format_date(date) do
Timex.format!(date, "{Mshort} {D}, {YYYY}")
end
@@ -83,58 +71,6 @@ defmodule Plausible.SiteAdmin do
|> Enum.join(", ")
end
- def transfer_data([from_site], params) do
- if Plausible.v2?() do
- {:error, "Transfers temporarily unspported with v2"}
- else
- to_site = Repo.get_by(Plausible.Site, domain: params["domain"])
-
- if to_site do
- event_q = event_transfer_query(from_site.domain, to_site.domain)
- {:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, event_q, [], timeout: 30_000)
-
- session_q = session_transfer_query(from_site.domain, to_site.domain)
-
- {:ok, _} =
- Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, session_q, [], timeout: 30_000)
-
- start_date = Plausible.Stats.Clickhouse.pageview_start_date_local(from_site)
-
- {:ok, _} =
- to_site
- |> Plausible.Site.set_stats_start_date(start_date)
- |> Plausible.Site.set_native_stats_start_at(from_site.native_stats_start_at)
- |> Repo.update()
-
- :ok
- else
- {:error, "Cannot transfer to non-existing domain"}
- end
- end
- end
-
- def transfer_data(_, _), do: {:error, "Please select exactly one site for this action"}
-
- def session_transfer_query(from_domain, to_domain) do
- fields = get_struct_fields(Plausible.ClickhouseSession)
-
- "INSERT INTO sessions (" <>
- stringify_fields(fields) <>
- ") SELECT " <>
- stringify_fields(fields, to_domain, from_domain) <>
- " FROM (SELECT * FROM sessions WHERE domain='#{from_domain}')"
- end
-
- def event_transfer_query(from_domain, to_domain) do
- fields = get_struct_fields(Plausible.ClickhouseEvent)
-
- "INSERT INTO events (" <>
- stringify_fields(fields) <>
- ") SELECT " <>
- stringify_fields(fields, to_domain, from_domain) <>
- " FROM (SELECT * FROM events WHERE domain='#{from_domain}')"
- end
-
def get_struct_fields(module) do
module.__struct__()
|> Map.drop([:__meta__, :__struct__])
@@ -143,19 +79,6 @@ defmodule Plausible.SiteAdmin do
|> Enum.sort()
end
- defp stringify_fields(fields), do: Enum.join(fields, ", ")
-
- defp stringify_fields(fields, domain_value, transferred_from_value) do
- Enum.map(fields, fn field ->
- case field do
- "domain" -> "'#{domain_value}' as domain"
- "transferred_from" -> "'#{transferred_from_value}' as transferred_from"
- _ -> field
- end
- end)
- |> stringify_fields()
- end
-
def create_changeset(schema, attrs), do: Plausible.Site.crm_changeset(schema, attrs)
def update_changeset(schema, attrs), do: Plausible.Site.crm_changeset(schema, attrs)
end
diff --git a/test/plausible/site/admin_test.exs b/test/plausible/site/admin_test.exs
deleted file mode 100644
index 99d5429c67..0000000000
--- a/test/plausible/site/admin_test.exs
+++ /dev/null
@@ -1,130 +0,0 @@
-defmodule Plausible.SiteAdminTest do
- use Plausible.DataCase, async: true
-
- alias Plausible.{
- SiteAdmin,
- ClickhouseRepo,
- ClickhouseEvent,
- ClickhouseSession
- }
-
- describe "transfer_data" do
- test "event and session structs remain the same after transfer" do
- if Plausible.v2?() do
- :ok
- else
- from_site = insert(:site)
- to_site = insert(:site)
-
- populate_stats(from_site, [build(:pageview)])
-
- event_before = get_event_by_domain(from_site.domain)
- session_before = get_session_by_domain(from_site.domain)
-
- SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
-
- event_after = get_event_by_domain(to_site.domain)
- session_after = get_session_by_domain(to_site.domain)
-
- assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}
- assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}
- assert session_before == %ClickhouseSession{session_after | transferred_from: ""}
- assert event_after.transferred_from == from_site.domain
- assert session_after.transferred_from == from_site.domain
- end
- end
-
- test "transfers all events and sessions" do
- if Plausible.v2?() do
- :ok
- else
- from_site = insert(:site)
- to_site = insert(:site)
-
- populate_stats(from_site, [
- build(:pageview, user_id: 123),
- build(:event, name: "Signup", user_id: 123),
- build(:pageview, user_id: 456),
- build(:event, name: "Signup", user_id: 789)
- ])
-
- SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
-
- transferred_events =
- ClickhouseRepo.all(
- from e in Plausible.ClickhouseEvent, where: e.domain == ^to_site.domain
- )
-
- transferred_sessions =
- ClickhouseRepo.all(
- from e in Plausible.ClickhouseSession, where: e.domain == ^to_site.domain
- )
-
- assert length(transferred_events) == 4
- assert length(transferred_sessions) == 3
- end
- end
-
- test "updates stats_start_date on site record" do
- if Plausible.v2?() do
- :ok
- else
- from_site = insert(:site)
- to_site = insert(:site)
-
- populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])])
-
- SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
-
- assert Repo.reload(to_site).stats_start_date == ~D[2022-01-01]
- end
- end
-
- test "updates native_stats_start_date on based on the from_site record" do
- if Plausible.v2?() do
- :ok
- else
- from_site = insert(:site, native_stats_start_at: ~N[2022-01-01 01:00:00])
- to_site = insert(:site)
-
- populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])])
-
- SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
-
- assert Repo.reload(to_site).native_stats_start_at == ~N[2022-01-01 01:00:00]
- end
- end
-
- test "session_transfer_query" do
- actual = SiteAdmin.session_transfer_query("from.com", "to.com")
-
- expected =
- "INSERT INTO sessions (browser, browser_version, city_geoname_id, country_code, domain, duration, entry_meta.key, entry_meta.value, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, duration, entry_meta.key, entry_meta.value, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM sessions WHERE domain='from.com')"
-
- assert actual == expected
- end
-
- test "event_transfer_query" do
- actual = SiteAdmin.event_transfer_query("from.com", "to.com")
-
- expected =
- "INSERT INTO events (browser, browser_version, city_geoname_id, country_code, domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM events WHERE domain='from.com')"
-
- assert actual == expected
- end
- end
-
- defp get_event_by_domain(domain) do
- q = from e in Plausible.ClickhouseEvent, where: e.domain == ^domain
-
- Plausible.ClickhouseRepo.one!(q)
- |> Map.drop([:__meta__, :domain])
- end
-
- defp get_session_by_domain(domain) do
- q = from s in Plausible.ClickhouseSession, where: s.domain == ^domain
-
- Plausible.ClickhouseRepo.one!(q)
- |> Map.drop([:__meta__, :domain])
- end
-end