Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Crm transfer data (#1749)
RobertJoonas492f47ba1e93
diff --git a/lib/plausible/event/clickhouse_schema.ex b/lib/plausible/event/clickhouse_schema.exindex f5d5fe678c..cb957576a7 100644--- a/lib/plausible/event/clickhouse_schema.ex+++ b/lib/plausible/event/clickhouse_schema.ex@@ -33,6 +33,7 @@ defmodule Plausible.ClickhouseEvent dofield :"meta.key", {:array, :string}, default: []field :"meta.value", {:array, :string}, default: []+ field :transferred_from, :string, default: ""enddef new(attrs) dodiff --git a/lib/plausible/session/clickhouse_schema.ex b/lib/plausible/session/clickhouse_schema.exindex ccf653049f..00b0b1f310 100644--- a/lib/plausible/session/clickhouse_schema.ex+++ b/lib/plausible/session/clickhouse_schema.ex@@ -37,6 +37,8 @@ defmodule Plausible.ClickhouseSession dofield :browser, :stringfield :browser_version, :stringfield :timestamp, :naive_datetime++ field :transferred_from, :string, default: ""enddef random_uint64() dodiff --git a/lib/plausible/site/admin.ex b/lib/plausible/site/admin.exindex f2d2938d5b..32b6034c7e 100644--- a/lib/plausible/site/admin.ex+++ b/lib/plausible/site/admin.ex@@ -1,5 +1,6 @@defmodule Plausible.SiteAdmin douse Plausible.Repo+ import Ecto.Querydef search_fields(_schema) do[@@ -31,6 +32,22 @@ defmodule Plausible.SiteAdmin do]end+ def list_actions(_conn) do+ [+ transfer_data: %{+ name: "Transfer data",+ inputs: [+ %{name: "domain", title: "to domain", default: nil}+ ],+ action: fn _conn, sites, params -> transfer_data(sites, params) end+ }+ ]+ end++ defp format_date(date) do+ Timex.format!(date, "{Mshort} {D}, {YYYY}")+ end+defp get_owner_email(site) doEnum.find(site.memberships, fn m -> m.role == :owner end).user.emailend@@ -40,7 +57,67 @@ defmodule Plausible.SiteAdmin doEnum.map(memberships, fn m -> m.user.email end) |> Enum.join(", ")end- defp format_date(date) do- Timex.format!(date, "{Mshort} {D}, {YYYY}")+ def transfer_data([site], params) do+ from_domain = site.domain+ to_domain = params["domain"]++ if to_domain && domain_exists?(to_domain) do+ event_q = event_transfer_query(from_domain, to_domain)+ {:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, event_q)++ session_q = session_transfer_query(from_domain, to_domain)+ {:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, session_q)++ :ok+ else+ {:error, "Cannot transfer to non-existing domain"}+ end+ end++ def transfer_data(_, _), do: {:error, "Please select exactly one site for this action"}++ defp domain_exists?(domain) do+ Repo.exists?(from s in Plausible.Site, where: s.domain == ^domain)+ end++ def session_transfer_query(from_domain, to_domain) do+ fields = get_struct_fields(Plausible.ClickhouseSession)++ "INSERT INTO sessions (" <>+ stringify_fields(fields) <>+ ") SELECT " <>+ stringify_fields(fields, to_domain, from_domain) <>+ " FROM (SELECT * FROM sessions WHERE domain='#{from_domain}')"+ end++ def event_transfer_query(from_domain, to_domain) do+ fields = get_struct_fields(Plausible.ClickhouseEvent)++ "INSERT INTO events (" <>+ stringify_fields(fields) <>+ ") SELECT " <>+ stringify_fields(fields, to_domain, from_domain) <>+ " FROM (SELECT * FROM events WHERE domain='#{from_domain}')"+ end++ def get_struct_fields(module) do+ module.__struct__()+ |> Map.drop([:__meta__, :__struct__])+ |> Map.keys()+ |> Enum.map(&Atom.to_string/1)+ |> Enum.sort()+ end++ defp stringify_fields(fields), do: Enum.join(fields, ", ")++ defp stringify_fields(fields, domain_value, transferred_from_value) do+ Enum.map(fields, fn field ->+ case field do+ "domain" -> "'#{domain_value}' as domain"+ "transferred_from" -> "'#{transferred_from_value}' as transferred_from"+ _ -> field+ end+ end)+ |> stringify_fields()endenddiff --git a/mix.lock b/mix.lockindex 4ba1ece281..a556a16ed1 100644--- a/mix.lock+++ b/mix.lock@@ -10,7 +10,7 @@"certifi": {:hex, :certifi, "2.8.0", "d4fb0a6bb20b7c9c3643e22507e42f356ac090a1dcea9ab99e27e0376d695eba", [:rebar3], [], "hexpm", "6ac7efc1c6f8600b08d625292d4bbf584e14847ce1b6b5c44d983d273e1097ea"},"chatterbox": {:hex, :ts_chatterbox, "0.11.0", "b8f372c706023eb0de5bf2976764edb27c70fe67052c88c1f6a66b3a5626847f", [:rebar3], [{:hpack, "~>0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "722fe2bad52913ab7e87d849fc6370375f0c961ffb2f0b5e6d647c9170c382a6"},"clickhouse_ecto": {:git, "https://github.com/plausible/clickhouse_ecto.git", "7bc94cce111d3e9dbd8534fe96bd5195181826a2", []},- "clickhousex": {:git, "https://github.com/plausible/clickhousex", "6405ac09b4fa103644bb4fe7fc0509fb48497927", []},+ "clickhousex": {:git, "https://github.com/plausible/clickhousex", "f030155234ad045a08a9fdc263f471dd4cfea350", []},"combination": {:hex, :combination, "0.0.3", "746aedca63d833293ec6e835aa1f34974868829b1486b1e1cb0685f0b2ae1f41", [:mix], [], "hexpm", "72b099f463df42ef7dc6371d250c7070b57b6c5902853f69deb894f79eda18ca"},"combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},"comeonin": {:hex, :comeonin, "5.3.2", "5c2f893d05c56ae3f5e24c1b983c2d5dfb88c6d979c9287a76a7feb1e1d8d646", [:mix], [], "hexpm", "d0993402844c49539aeadb3fe46a3c9bd190f1ecf86b6f9ebd71957534c95f04"},diff --git a/priv/clickhouse_repo/migrations/20220310104931_add_transferred_from.exs b/priv/clickhouse_repo/migrations/20220310104931_add_transferred_from.exsnew file mode 100644index 0000000000..0e72288c7e--- /dev/null+++ b/priv/clickhouse_repo/migrations/20220310104931_add_transferred_from.exs@@ -0,0 +1,13 @@+defmodule Plausible.ClickhouseRepo.Migrations.AddTransferredFrom do+ use Ecto.Migration++ def change do+ alter table(:events) do+ add(:transferred_from, :string)+ end++ alter table(:sessions) do+ add(:transferred_from, :string)+ end+ end+enddiff --git a/test/plausible/site/admin_test.exs b/test/plausible/site/admin_test.exsnew file mode 100644index 0000000000..15b916c94b--- /dev/null+++ b/test/plausible/site/admin_test.exs@@ -0,0 +1,82 @@+defmodule Plausible.SiteAdminTest do+ use Plausible.DataCase+ import Plausible.TestUtils+ alias Plausible.{SiteAdmin, ClickhouseRepo, ClickhouseEvent, ClickhouseSession}++ test "event and session structs remain the same after transfer" do+ from_site = insert(:site)+ to_site = insert(:site)++ populate_stats(from_site, [build(:pageview)])++ event_before = get_event_by_domain(from_site.domain)+ session_before = get_session_by_domain(from_site.domain)++ SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})++ event_after = get_event_by_domain(to_site.domain)+ session_after = get_session_by_domain(to_site.domain)++ assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}+ assert session_before == %ClickhouseSession{session_after | transferred_from: ""}+ assert event_after.transferred_from == from_site.domain+ assert session_after.transferred_from == from_site.domain+ end++ test "transfers all events and sessions" do+ from_site = insert(:site)+ to_site = insert(:site)++ populate_stats(from_site, [+ build(:pageview, user_id: 123),+ build(:event, name: "Signup", user_id: 123),+ build(:pageview, user_id: 456),+ build(:event, name: "Signup", user_id: 789)+ ])++ SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})++ transferred_events =+ ClickhouseRepo.all(from e in Plausible.ClickhouseEvent, where: e.domain == ^to_site.domain)++ transferred_sessions =+ ClickhouseRepo.all(+ from e in Plausible.ClickhouseSession, where: e.domain == ^to_site.domain+ )++ assert length(transferred_events) == 4+ assert length(transferred_sessions) == 3+ end++ test "session_transfer_query" do+ actual = SiteAdmin.session_transfer_query("from.com", "to.com")++ expected =+ "INSERT INTO sessions (browser, browser_version, city_geoname_id, country_code, domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM sessions WHERE domain='from.com')"++ assert actual == expected+ end++ test "event_transfer_query" do+ actual = SiteAdmin.event_transfer_query("from.com", "to.com")++ expected =+ "INSERT INTO events (browser, browser_version, city_geoname_id, country_code, domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM events WHERE domain='from.com')"++ assert actual == expected+ end++ defp get_event_by_domain(domain) do+ q = from e in Plausible.ClickhouseEvent, where: e.domain == ^domain++ Plausible.ClickhouseRepo.one!(q)+ |> Map.drop([:__meta__, :domain])+ end++ defp get_session_by_domain(domain) do+ q = from s in Plausible.ClickhouseSession, where: s.domain == ^domain++ Plausible.ClickhouseRepo.one!(q)+ |> Map.drop([:__meta__, :domain])+ end+enddiff --git a/test/plausible/sites_test.exs b/test/plausible/site/sites_test.exssimilarity index 100%rename from test/plausible/sites_test.exsrename to test/plausible/site/sites_test.exs