Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Crm transfer data (#1749)

RobertJoonas492f47ba1e93
diff --git a/lib/plausible/event/clickhouse_schema.ex b/lib/plausible/event/clickhouse_schema.ex
index f5d5fe678c..cb957576a7 100644
--- a/lib/plausible/event/clickhouse_schema.ex
+++ b/lib/plausible/event/clickhouse_schema.ex
@@ -33,6 +33,7 @@ defmodule Plausible.ClickhouseEvent do
field :"meta.key", {:array, :string}, default: []
field :"meta.value", {:array, :string}, default: []
+ field :transferred_from, :string, default: ""
end
def new(attrs) do
diff --git a/lib/plausible/session/clickhouse_schema.ex b/lib/plausible/session/clickhouse_schema.ex
index ccf653049f..00b0b1f310 100644
--- a/lib/plausible/session/clickhouse_schema.ex
+++ b/lib/plausible/session/clickhouse_schema.ex
@@ -37,6 +37,8 @@ defmodule Plausible.ClickhouseSession do
field :browser, :string
field :browser_version, :string
field :timestamp, :naive_datetime
+
+ field :transferred_from, :string, default: ""
end
def random_uint64() do
diff --git a/lib/plausible/site/admin.ex b/lib/plausible/site/admin.ex
index f2d2938d5b..32b6034c7e 100644
--- a/lib/plausible/site/admin.ex
+++ b/lib/plausible/site/admin.ex
@@ -1,5 +1,6 @@
defmodule Plausible.SiteAdmin do
use Plausible.Repo
+ import Ecto.Query
def search_fields(_schema) do
[
@@ -31,6 +32,22 @@ defmodule Plausible.SiteAdmin do
]
end
+ def list_actions(_conn) do
+ [
+ transfer_data: %{
+ name: "Transfer data",
+ inputs: [
+ %{name: "domain", title: "to domain", default: nil}
+ ],
+ action: fn _conn, sites, params -> transfer_data(sites, params) end
+ }
+ ]
+ end
+
+ defp format_date(date) do
+ Timex.format!(date, "{Mshort} {D}, {YYYY}")
+ end
+
defp get_owner_email(site) do
Enum.find(site.memberships, fn m -> m.role == :owner end).user.email
end
@@ -40,7 +57,67 @@ defmodule Plausible.SiteAdmin do
Enum.map(memberships, fn m -> m.user.email end) |> Enum.join(", ")
end
- defp format_date(date) do
- Timex.format!(date, "{Mshort} {D}, {YYYY}")
+ def transfer_data([site], params) do
+ from_domain = site.domain
+ to_domain = params["domain"]
+
+ if to_domain && domain_exists?(to_domain) do
+ event_q = event_transfer_query(from_domain, to_domain)
+ {:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, event_q)
+
+ session_q = session_transfer_query(from_domain, to_domain)
+ {:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, session_q)
+
+ :ok
+ else
+ {:error, "Cannot transfer to non-existing domain"}
+ end
+ end
+
+ def transfer_data(_, _), do: {:error, "Please select exactly one site for this action"}
+
+ defp domain_exists?(domain) do
+ Repo.exists?(from s in Plausible.Site, where: s.domain == ^domain)
+ end
+
+ def session_transfer_query(from_domain, to_domain) do
+ fields = get_struct_fields(Plausible.ClickhouseSession)
+
+ "INSERT INTO sessions (" <>
+ stringify_fields(fields) <>
+ ") SELECT " <>
+ stringify_fields(fields, to_domain, from_domain) <>
+ " FROM (SELECT * FROM sessions WHERE domain='#{from_domain}')"
+ end
+
+ def event_transfer_query(from_domain, to_domain) do
+ fields = get_struct_fields(Plausible.ClickhouseEvent)
+
+ "INSERT INTO events (" <>
+ stringify_fields(fields) <>
+ ") SELECT " <>
+ stringify_fields(fields, to_domain, from_domain) <>
+ " FROM (SELECT * FROM events WHERE domain='#{from_domain}')"
+ end
+
+ def get_struct_fields(module) do
+ module.__struct__()
+ |> Map.drop([:__meta__, :__struct__])
+ |> Map.keys()
+ |> Enum.map(&Atom.to_string/1)
+ |> Enum.sort()
+ end
+
+ defp stringify_fields(fields), do: Enum.join(fields, ", ")
+
+ defp stringify_fields(fields, domain_value, transferred_from_value) do
+ Enum.map(fields, fn field ->
+ case field do
+ "domain" -> "'#{domain_value}' as domain"
+ "transferred_from" -> "'#{transferred_from_value}' as transferred_from"
+ _ -> field
+ end
+ end)
+ |> stringify_fields()
end
end
diff --git a/mix.lock b/mix.lock
index 4ba1ece281..a556a16ed1 100644
--- a/mix.lock
+++ b/mix.lock
@@ -10,7 +10,7 @@
"certifi": {:hex, :certifi, "2.8.0", "d4fb0a6bb20b7c9c3643e22507e42f356ac090a1dcea9ab99e27e0376d695eba", [:rebar3], [], "hexpm", "6ac7efc1c6f8600b08d625292d4bbf584e14847ce1b6b5c44d983d273e1097ea"},
"chatterbox": {:hex, :ts_chatterbox, "0.11.0", "b8f372c706023eb0de5bf2976764edb27c70fe67052c88c1f6a66b3a5626847f", [:rebar3], [{:hpack, "~>0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "722fe2bad52913ab7e87d849fc6370375f0c961ffb2f0b5e6d647c9170c382a6"},
"clickhouse_ecto": {:git, "https://github.com/plausible/clickhouse_ecto.git", "7bc94cce111d3e9dbd8534fe96bd5195181826a2", []},
- "clickhousex": {:git, "https://github.com/plausible/clickhousex", "6405ac09b4fa103644bb4fe7fc0509fb48497927", []},
+ "clickhousex": {:git, "https://github.com/plausible/clickhousex", "f030155234ad045a08a9fdc263f471dd4cfea350", []},
"combination": {:hex, :combination, "0.0.3", "746aedca63d833293ec6e835aa1f34974868829b1486b1e1cb0685f0b2ae1f41", [:mix], [], "hexpm", "72b099f463df42ef7dc6371d250c7070b57b6c5902853f69deb894f79eda18ca"},
"combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
"comeonin": {:hex, :comeonin, "5.3.2", "5c2f893d05c56ae3f5e24c1b983c2d5dfb88c6d979c9287a76a7feb1e1d8d646", [:mix], [], "hexpm", "d0993402844c49539aeadb3fe46a3c9bd190f1ecf86b6f9ebd71957534c95f04"},
diff --git a/priv/clickhouse_repo/migrations/20220310104931_add_transferred_from.exs b/priv/clickhouse_repo/migrations/20220310104931_add_transferred_from.exs
new file mode 100644
index 0000000000..0e72288c7e
--- /dev/null
+++ b/priv/clickhouse_repo/migrations/20220310104931_add_transferred_from.exs
@@ -0,0 +1,13 @@
+defmodule Plausible.ClickhouseRepo.Migrations.AddTransferredFrom do
+ use Ecto.Migration
+
+ def change do
+ alter table(:events) do
+ add(:transferred_from, :string)
+ end
+
+ alter table(:sessions) do
+ add(:transferred_from, :string)
+ end
+ end
+end
diff --git a/test/plausible/site/admin_test.exs b/test/plausible/site/admin_test.exs
new file mode 100644
index 0000000000..15b916c94b
--- /dev/null
+++ b/test/plausible/site/admin_test.exs
@@ -0,0 +1,82 @@
+defmodule Plausible.SiteAdminTest do
+ use Plausible.DataCase
+ import Plausible.TestUtils
+ alias Plausible.{SiteAdmin, ClickhouseRepo, ClickhouseEvent, ClickhouseSession}
+
+ test "event and session structs remain the same after transfer" do
+ from_site = insert(:site)
+ to_site = insert(:site)
+
+ populate_stats(from_site, [build(:pageview)])
+
+ event_before = get_event_by_domain(from_site.domain)
+ session_before = get_session_by_domain(from_site.domain)
+
+ SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
+
+ event_after = get_event_by_domain(to_site.domain)
+ session_after = get_session_by_domain(to_site.domain)
+
+ assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}
+ assert session_before == %ClickhouseSession{session_after | transferred_from: ""}
+ assert event_after.transferred_from == from_site.domain
+ assert session_after.transferred_from == from_site.domain
+ end
+
+ test "transfers all events and sessions" do
+ from_site = insert(:site)
+ to_site = insert(:site)
+
+ populate_stats(from_site, [
+ build(:pageview, user_id: 123),
+ build(:event, name: "Signup", user_id: 123),
+ build(:pageview, user_id: 456),
+ build(:event, name: "Signup", user_id: 789)
+ ])
+
+ SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
+
+ transferred_events =
+ ClickhouseRepo.all(from e in Plausible.ClickhouseEvent, where: e.domain == ^to_site.domain)
+
+ transferred_sessions =
+ ClickhouseRepo.all(
+ from e in Plausible.ClickhouseSession, where: e.domain == ^to_site.domain
+ )
+
+ assert length(transferred_events) == 4
+ assert length(transferred_sessions) == 3
+ end
+
+ test "session_transfer_query" do
+ actual = SiteAdmin.session_transfer_query("from.com", "to.com")
+
+ expected =
+ "INSERT INTO sessions (browser, browser_version, city_geoname_id, country_code, domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM sessions WHERE domain='from.com')"
+
+ assert actual == expected
+ end
+
+ test "event_transfer_query" do
+ actual = SiteAdmin.event_transfer_query("from.com", "to.com")
+
+ expected =
+ "INSERT INTO events (browser, browser_version, city_geoname_id, country_code, domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM events WHERE domain='from.com')"
+
+ assert actual == expected
+ end
+
+ defp get_event_by_domain(domain) do
+ q = from e in Plausible.ClickhouseEvent, where: e.domain == ^domain
+
+ Plausible.ClickhouseRepo.one!(q)
+ |> Map.drop([:__meta__, :domain])
+ end
+
+ defp get_session_by_domain(domain) do
+ q = from s in Plausible.ClickhouseSession, where: s.domain == ^domain
+
+ Plausible.ClickhouseRepo.one!(q)
+ |> Map.drop([:__meta__, :domain])
+ end
+end
diff --git a/test/plausible/sites_test.exs b/test/plausible/site/sites_test.exs
similarity index 100%
rename from test/plausible/sites_test.exs
rename to test/plausible/site/sites_test.exs