Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Add stats_start_date field
Uku Taht333de87ceba1
diff --git a/lib/plausible/site/admin.ex b/lib/plausible/site/admin.exindex 32b6034c7e..4c8fc863bc 100644--- a/lib/plausible/site/admin.ex+++ b/lib/plausible/site/admin.ex@@ -57,17 +57,22 @@ defmodule Plausible.SiteAdmin doEnum.map(memberships, fn m -> m.user.email end) |> Enum.join(", ")end- def transfer_data([site], params) do- from_domain = site.domain- to_domain = params["domain"]+ def transfer_data([from_site], params) do+ to_site = Repo.get_by(Plausible.Site, domain: params["domain"])- if to_domain && domain_exists?(to_domain) do- event_q = event_transfer_query(from_domain, to_domain)+ if to_site do+ event_q = event_transfer_query(from_site.domain, to_site.domain){:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, event_q)- session_q = session_transfer_query(from_domain, to_domain)+ session_q = session_transfer_query(from_site.domain, to_site.domain){:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, session_q)+ start_date = Plausible.Stats.Clickhouse.pageview_start_date_local(from_site)++ {:ok, _} =+ Plausible.Site.set_stats_start_date(to_site, start_date)+ |> Repo.update()+:okelse{:error, "Cannot transfer to non-existing domain"}diff --git a/lib/plausible/site/schema.ex b/lib/plausible/site/schema.exindex 127dffc01c..1a4afe8d65 100644--- a/lib/plausible/site/schema.ex+++ b/lib/plausible/site/schema.ex@@ -22,6 +22,7 @@ defmodule Plausible.Site dofield :public, :booleanfield :locked, :booleanfield :has_stats, :boolean+ field :stats_start_date, :dateembeds_one :imported_data, Plausible.Site.ImportedData, on_replace: :update@@ -63,6 +64,10 @@ defmodule Plausible.Site dochange(site, has_stats: has_stats_val)end+ def set_stats_start_date(site, val) do+ change(site, stats_start_date: val)+ end+def start_import(site, start_date, end_date, imported_source, status \\ "importing") dochange(site,imported_data: %{@@ -75,7 +80,10 @@ defmodule Plausible.Site doenddef import_success(site) do- change(site, imported_data: %{status: "ok"})+ change(site,+ stats_start_date: site.imported_data.start_date,+ imported_data: %{status: "ok"}+ )enddef import_failure(site) dodiff --git a/lib/plausible/sites.ex b/lib/plausible/sites.exindex 25bba9c846..27ba78f04a 100644--- a/lib/plausible/sites.ex+++ b/lib/plausible/sites.ex@@ -38,6 +38,21 @@ defmodule Plausible.Sites doendend+ def stats_start_date(site) do+ if site.stats_start_date do+ site.stats_start_date+ else+ start_date = Plausible.Stats.Clickhouse.pageview_start_date_local(site)++ if start_date do+ Plausible.Site.set_stats_start_date(site, start_date)+ |> Repo.update()++ start_date+ end+ end+ end+def has_stats?(site) doif site.has_stats dotruediff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.exindex 7dc3705cff..3d8ce1d636 100644--- a/lib/plausible/stats/clickhouse.ex+++ b/lib/plausible/stats/clickhouse.ex@@ -16,7 +16,7 @@ defmodule Plausible.Stats.Clickhouse docase datetime do# no stats for this domain yet~N[1970-01-01 00:00:00] ->- Timex.today(site.timezone)+ nil_ ->Timex.Timezone.convert(datetime, "UTC")diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.exindex 728e835f46..53f70ffa84 100644--- a/lib/plausible_web/controllers/site_controller.ex+++ b/lib/plausible_web/controllers/site_controller.ex@@ -701,7 +701,10 @@ defmodule PlausibleWeb.SiteController dosite = conn.assigns[:site]start_date = Plausible.Google.Api.get_analytics_start_date(view_id, access_token)- end_date = Plausible.Stats.Clickhouse.pageview_start_date_local(site)++ end_date =+ Plausible.Stats.Clickhouse.pageview_start_date_local(site) || Timex.today(site.timezone)+{:ok, view_ids} = Plausible.Google.Api.get_analytics_view_ids(access_token)conndiff --git a/priv/repo/migrations/20220405124819_add_stats_start_date.exs b/priv/repo/migrations/20220405124819_add_stats_start_date.exsnew file mode 100644index 0000000000..250d1f9a98--- /dev/null+++ b/priv/repo/migrations/20220405124819_add_stats_start_date.exs@@ -0,0 +1,9 @@+defmodule Plausible.Repo.Migrations.AddStatsStartDate do+ use Ecto.Migration++ def change do+ alter table(:sites) do+ add :stats_start_date, :date+ end+ end+enddiff --git a/test/plausible/site/admin_test.exs b/test/plausible/site/admin_test.exsindex 15b916c94b..f5cd92538c 100644--- a/test/plausible/site/admin_test.exs+++ b/test/plausible/site/admin_test.exs@@ -3,67 +3,82 @@ defmodule Plausible.SiteAdminTest doimport Plausible.TestUtilsalias Plausible.{SiteAdmin, ClickhouseRepo, ClickhouseEvent, ClickhouseSession}- test "event and session structs remain the same after transfer" do- from_site = insert(:site)- to_site = insert(:site)+ describe "transfer_data" do+ test "event and session structs remain the same after transfer" do+ from_site = insert(:site)+ to_site = insert(:site)- populate_stats(from_site, [build(:pageview)])+ populate_stats(from_site, [build(:pageview)])- event_before = get_event_by_domain(from_site.domain)- session_before = get_session_by_domain(from_site.domain)+ event_before = get_event_by_domain(from_site.domain)+ session_before = get_session_by_domain(from_site.domain)- SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})+ SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})- event_after = get_event_by_domain(to_site.domain)- session_after = get_session_by_domain(to_site.domain)+ event_after = get_event_by_domain(to_site.domain)+ session_after = get_session_by_domain(to_site.domain)- assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}- assert session_before == %ClickhouseSession{session_after | transferred_from: ""}- assert event_after.transferred_from == from_site.domain- assert session_after.transferred_from == from_site.domain- end+ assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}+ assert session_before == %ClickhouseSession{session_after | transferred_from: ""}+ assert event_after.transferred_from == from_site.domain+ assert session_after.transferred_from == from_site.domain+ end- test "transfers all events and sessions" do- from_site = insert(:site)- to_site = insert(:site)+ test "transfers all events and sessions" do+ from_site = insert(:site)+ to_site = insert(:site)- populate_stats(from_site, [- build(:pageview, user_id: 123),- build(:event, name: "Signup", user_id: 123),- build(:pageview, user_id: 456),- build(:event, name: "Signup", user_id: 789)- ])+ populate_stats(from_site, [+ build(:pageview, user_id: 123),+ build(:event, name: "Signup", user_id: 123),+ build(:pageview, user_id: 456),+ build(:event, name: "Signup", user_id: 789)+ ])- SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})+ SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})- transferred_events =- ClickhouseRepo.all(from e in Plausible.ClickhouseEvent, where: e.domain == ^to_site.domain)+ transferred_events =+ ClickhouseRepo.all(+ from e in Plausible.ClickhouseEvent, where: e.domain == ^to_site.domain+ )- transferred_sessions =- ClickhouseRepo.all(- from e in Plausible.ClickhouseSession, where: e.domain == ^to_site.domain- )+ transferred_sessions =+ ClickhouseRepo.all(+ from e in Plausible.ClickhouseSession, where: e.domain == ^to_site.domain+ )- assert length(transferred_events) == 4- assert length(transferred_sessions) == 3- end+ assert length(transferred_events) == 4+ assert length(transferred_sessions) == 3+ end- test "session_transfer_query" do- actual = SiteAdmin.session_transfer_query("from.com", "to.com")+ test "updates stats_start_date on site record" do+ from_site = insert(:site)+ to_site = insert(:site)- expected =- "INSERT INTO sessions (browser, browser_version, city_geoname_id, country_code, domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM sessions WHERE domain='from.com')"+ populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])])- assert actual == expected- end+ SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})++ assert Repo.reload(to_site).stats_start_date == ~D[2022-01-01]+ end++ test "session_transfer_query" do+ actual = SiteAdmin.session_transfer_query("from.com", "to.com")++ expected =+ "INSERT INTO sessions (browser, browser_version, city_geoname_id, country_code, domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM sessions WHERE domain='from.com')"++ assert actual == expected+ end- test "event_transfer_query" do- actual = SiteAdmin.event_transfer_query("from.com", "to.com")+ test "event_transfer_query" do+ actual = SiteAdmin.event_transfer_query("from.com", "to.com")- expected =- "INSERT INTO events (browser, browser_version, city_geoname_id, country_code, domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM events WHERE domain='from.com')"+ expected =+ "INSERT INTO events (browser, browser_version, city_geoname_id, country_code, domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM events WHERE domain='from.com')"- assert actual == expected+ assert actual == expected+ endenddefp get_event_by_domain(domain) dodiff --git a/test/plausible/site/sites_test.exs b/test/plausible/site/sites_test.exsindex 077c694d47..492bfa44a7 100644--- a/test/plausible/site/sites_test.exs+++ b/test/plausible/site/sites_test.exs@@ -19,6 +19,37 @@ defmodule Plausible.SitesTest doendend+ describe "stats_start_date" do+ test "is nil if site has no stats" do+ site = insert(:site)++ assert Sites.stats_start_date(site) == nil+ end++ test "is date if first pageview if site does have stats" do+ site = insert(:site)++ populate_stats(site, [+ build(:pageview)+ ])++ assert Sites.stats_start_date(site) == Timex.today(site.timezone)+ end++ test "memoizes value of start date" do+ site = insert(:site)++ assert site.stats_start_date == nil++ populate_stats(site, [+ build(:pageview)+ ])++ assert Sites.stats_start_date(site) == Timex.today(site.timezone)+ assert Repo.reload!(site).stats_start_date == Timex.today(site.timezone)+ end+ end+describe "has_stats?" dotest "is false if site has no stats" dosite = insert(:site)diff --git a/test/workers/import_google_analytics_test.exs b/test/workers/import_google_analytics_test.exsindex 5caa6d36ed..1852945a47 100644--- a/test/workers/import_google_analytics_test.exs+++ b/test/workers/import_google_analytics_test.exs@@ -6,6 +6,7 @@ defmodule Plausible.Workers.ImportGoogleAnalyticsTest doalias Plausible.Workers.ImportGoogleAnalytics@imported_data %Plausible.Site.ImportedData{+ start_date: Timex.today() |> Timex.shift(days: -7),end_date: Timex.today(),source: "Google Analytics",status: "importing"@@ -40,6 +41,35 @@ defmodule Plausible.Workers.ImportGoogleAnalyticsTest doassert Repo.reload!(site).imported_data.status == "ok"end+ test "updates the stats_start_date field for the site after succesful import" do+ user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))+ site = insert(:site, members: [user], imported_data: @imported_data)++ api_stub =+ stub(Plausible.Google.Api, :import_analytics, fn _site,+ _view_id,+ _start_date,+ _end_date,+ _access_token ->+ {:ok, nil}+ end)++ ImportGoogleAnalytics.perform(+ %Oban.Job{+ args: %{+ "site_id" => site.id,+ "view_id" => "view_id",+ "start_date" => "2020-01-01",+ "end_date" => "2022-01-01",+ "access_token" => "token"+ }+ },+ api_stub+ )++ assert Repo.reload!(site).stats_start_date == @imported_data.start_date+ end+test "sends email to owner after succesful import" douser = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))site = insert(:site, members: [user], imported_data: @imported_data)