Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Add stats_start_date field

Uku Taht333de87ceba1
diff --git a/lib/plausible/site/admin.ex b/lib/plausible/site/admin.ex
index 32b6034c7e..4c8fc863bc 100644
--- a/lib/plausible/site/admin.ex
+++ b/lib/plausible/site/admin.ex
@@ -57,17 +57,22 @@ defmodule Plausible.SiteAdmin do
Enum.map(memberships, fn m -> m.user.email end) |> Enum.join(", ")
end
- def transfer_data([site], params) do
- from_domain = site.domain
- to_domain = params["domain"]
+ def transfer_data([from_site], params) do
+ to_site = Repo.get_by(Plausible.Site, domain: params["domain"])
- if to_domain && domain_exists?(to_domain) do
- event_q = event_transfer_query(from_domain, to_domain)
+ if to_site do
+ event_q = event_transfer_query(from_site.domain, to_site.domain)
{:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, event_q)
- session_q = session_transfer_query(from_domain, to_domain)
+ session_q = session_transfer_query(from_site.domain, to_site.domain)
{:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, session_q)
+ start_date = Plausible.Stats.Clickhouse.pageview_start_date_local(from_site)
+
+ {:ok, _} =
+ Plausible.Site.set_stats_start_date(to_site, start_date)
+ |> Repo.update()
+
:ok
else
{:error, "Cannot transfer to non-existing domain"}
diff --git a/lib/plausible/site/schema.ex b/lib/plausible/site/schema.ex
index 127dffc01c..1a4afe8d65 100644
--- a/lib/plausible/site/schema.ex
+++ b/lib/plausible/site/schema.ex
@@ -22,6 +22,7 @@ defmodule Plausible.Site do
field :public, :boolean
field :locked, :boolean
field :has_stats, :boolean
+ field :stats_start_date, :date
embeds_one :imported_data, Plausible.Site.ImportedData, on_replace: :update
@@ -63,6 +64,10 @@ defmodule Plausible.Site do
change(site, has_stats: has_stats_val)
end
+ def set_stats_start_date(site, val) do
+ change(site, stats_start_date: val)
+ end
+
def start_import(site, start_date, end_date, imported_source, status \\ "importing") do
change(site,
imported_data: %{
@@ -75,7 +80,10 @@ defmodule Plausible.Site do
end
def import_success(site) do
- change(site, imported_data: %{status: "ok"})
+ change(site,
+ stats_start_date: site.imported_data.start_date,
+ imported_data: %{status: "ok"}
+ )
end
def import_failure(site) do
diff --git a/lib/plausible/sites.ex b/lib/plausible/sites.ex
index 25bba9c846..27ba78f04a 100644
--- a/lib/plausible/sites.ex
+++ b/lib/plausible/sites.ex
@@ -38,6 +38,21 @@ defmodule Plausible.Sites do
end
end
+ def stats_start_date(site) do
+ if site.stats_start_date do
+ site.stats_start_date
+ else
+ start_date = Plausible.Stats.Clickhouse.pageview_start_date_local(site)
+
+ if start_date do
+ Plausible.Site.set_stats_start_date(site, start_date)
+ |> Repo.update()
+
+ start_date
+ end
+ end
+ end
+
def has_stats?(site) do
if site.has_stats do
true
diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex
index 7dc3705cff..3d8ce1d636 100644
--- a/lib/plausible/stats/clickhouse.ex
+++ b/lib/plausible/stats/clickhouse.ex
@@ -16,7 +16,7 @@ defmodule Plausible.Stats.Clickhouse do
case datetime do
# no stats for this domain yet
~N[1970-01-01 00:00:00] ->
- Timex.today(site.timezone)
+ nil
_ ->
Timex.Timezone.convert(datetime, "UTC")
diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex
index 728e835f46..53f70ffa84 100644
--- a/lib/plausible_web/controllers/site_controller.ex
+++ b/lib/plausible_web/controllers/site_controller.ex
@@ -701,7 +701,10 @@ defmodule PlausibleWeb.SiteController do
site = conn.assigns[:site]
start_date = Plausible.Google.Api.get_analytics_start_date(view_id, access_token)
- end_date = Plausible.Stats.Clickhouse.pageview_start_date_local(site)
+
+ end_date =
+ Plausible.Stats.Clickhouse.pageview_start_date_local(site) || Timex.today(site.timezone)
+
{:ok, view_ids} = Plausible.Google.Api.get_analytics_view_ids(access_token)
conn
diff --git a/priv/repo/migrations/20220405124819_add_stats_start_date.exs b/priv/repo/migrations/20220405124819_add_stats_start_date.exs
new file mode 100644
index 0000000000..250d1f9a98
--- /dev/null
+++ b/priv/repo/migrations/20220405124819_add_stats_start_date.exs
@@ -0,0 +1,9 @@
+defmodule Plausible.Repo.Migrations.AddStatsStartDate do
+ use Ecto.Migration
+
+ def change do
+ alter table(:sites) do
+ add :stats_start_date, :date
+ end
+ end
+end
diff --git a/test/plausible/site/admin_test.exs b/test/plausible/site/admin_test.exs
index 15b916c94b..f5cd92538c 100644
--- a/test/plausible/site/admin_test.exs
+++ b/test/plausible/site/admin_test.exs
@@ -3,67 +3,82 @@ defmodule Plausible.SiteAdminTest do
import Plausible.TestUtils
alias Plausible.{SiteAdmin, ClickhouseRepo, ClickhouseEvent, ClickhouseSession}
- test "event and session structs remain the same after transfer" do
- from_site = insert(:site)
- to_site = insert(:site)
+ describe "transfer_data" do
+ test "event and session structs remain the same after transfer" do
+ from_site = insert(:site)
+ to_site = insert(:site)
- populate_stats(from_site, [build(:pageview)])
+ populate_stats(from_site, [build(:pageview)])
- event_before = get_event_by_domain(from_site.domain)
- session_before = get_session_by_domain(from_site.domain)
+ event_before = get_event_by_domain(from_site.domain)
+ session_before = get_session_by_domain(from_site.domain)
- SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
+ SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
- event_after = get_event_by_domain(to_site.domain)
- session_after = get_session_by_domain(to_site.domain)
+ event_after = get_event_by_domain(to_site.domain)
+ session_after = get_session_by_domain(to_site.domain)
- assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}
- assert session_before == %ClickhouseSession{session_after | transferred_from: ""}
- assert event_after.transferred_from == from_site.domain
- assert session_after.transferred_from == from_site.domain
- end
+ assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}
+ assert session_before == %ClickhouseSession{session_after | transferred_from: ""}
+ assert event_after.transferred_from == from_site.domain
+ assert session_after.transferred_from == from_site.domain
+ end
- test "transfers all events and sessions" do
- from_site = insert(:site)
- to_site = insert(:site)
+ test "transfers all events and sessions" do
+ from_site = insert(:site)
+ to_site = insert(:site)
- populate_stats(from_site, [
- build(:pageview, user_id: 123),
- build(:event, name: "Signup", user_id: 123),
- build(:pageview, user_id: 456),
- build(:event, name: "Signup", user_id: 789)
- ])
+ populate_stats(from_site, [
+ build(:pageview, user_id: 123),
+ build(:event, name: "Signup", user_id: 123),
+ build(:pageview, user_id: 456),
+ build(:event, name: "Signup", user_id: 789)
+ ])
- SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
+ SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
- transferred_events =
- ClickhouseRepo.all(from e in Plausible.ClickhouseEvent, where: e.domain == ^to_site.domain)
+ transferred_events =
+ ClickhouseRepo.all(
+ from e in Plausible.ClickhouseEvent, where: e.domain == ^to_site.domain
+ )
- transferred_sessions =
- ClickhouseRepo.all(
- from e in Plausible.ClickhouseSession, where: e.domain == ^to_site.domain
- )
+ transferred_sessions =
+ ClickhouseRepo.all(
+ from e in Plausible.ClickhouseSession, where: e.domain == ^to_site.domain
+ )
- assert length(transferred_events) == 4
- assert length(transferred_sessions) == 3
- end
+ assert length(transferred_events) == 4
+ assert length(transferred_sessions) == 3
+ end
- test "session_transfer_query" do
- actual = SiteAdmin.session_transfer_query("from.com", "to.com")
+ test "updates stats_start_date on site record" do
+ from_site = insert(:site)
+ to_site = insert(:site)
- expected =
- "INSERT INTO sessions (browser, browser_version, city_geoname_id, country_code, domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM sessions WHERE domain='from.com')"
+ populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])])
- assert actual == expected
- end
+ SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
+
+ assert Repo.reload(to_site).stats_start_date == ~D[2022-01-01]
+ end
+
+ test "session_transfer_query" do
+ actual = SiteAdmin.session_transfer_query("from.com", "to.com")
+
+ expected =
+ "INSERT INTO sessions (browser, browser_version, city_geoname_id, country_code, domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM sessions WHERE domain='from.com')"
+
+ assert actual == expected
+ end
- test "event_transfer_query" do
- actual = SiteAdmin.event_transfer_query("from.com", "to.com")
+ test "event_transfer_query" do
+ actual = SiteAdmin.event_transfer_query("from.com", "to.com")
- expected =
- "INSERT INTO events (browser, browser_version, city_geoname_id, country_code, domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM events WHERE domain='from.com')"
+ expected =
+ "INSERT INTO events (browser, browser_version, city_geoname_id, country_code, domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM events WHERE domain='from.com')"
- assert actual == expected
+ assert actual == expected
+ end
end
defp get_event_by_domain(domain) do
diff --git a/test/plausible/site/sites_test.exs b/test/plausible/site/sites_test.exs
index 077c694d47..492bfa44a7 100644
--- a/test/plausible/site/sites_test.exs
+++ b/test/plausible/site/sites_test.exs
@@ -19,6 +19,37 @@ defmodule Plausible.SitesTest do
end
end
+ describe "stats_start_date" do
+ test "is nil if site has no stats" do
+ site = insert(:site)
+
+ assert Sites.stats_start_date(site) == nil
+ end
+
+ test "is date if first pageview if site does have stats" do
+ site = insert(:site)
+
+ populate_stats(site, [
+ build(:pageview)
+ ])
+
+ assert Sites.stats_start_date(site) == Timex.today(site.timezone)
+ end
+
+ test "memoizes value of start date" do
+ site = insert(:site)
+
+ assert site.stats_start_date == nil
+
+ populate_stats(site, [
+ build(:pageview)
+ ])
+
+ assert Sites.stats_start_date(site) == Timex.today(site.timezone)
+ assert Repo.reload!(site).stats_start_date == Timex.today(site.timezone)
+ end
+ end
+
describe "has_stats?" do
test "is false if site has no stats" do
site = insert(:site)
diff --git a/test/workers/import_google_analytics_test.exs b/test/workers/import_google_analytics_test.exs
index 5caa6d36ed..1852945a47 100644
--- a/test/workers/import_google_analytics_test.exs
+++ b/test/workers/import_google_analytics_test.exs
@@ -6,6 +6,7 @@ defmodule Plausible.Workers.ImportGoogleAnalyticsTest do
alias Plausible.Workers.ImportGoogleAnalytics
@imported_data %Plausible.Site.ImportedData{
+ start_date: Timex.today() |> Timex.shift(days: -7),
end_date: Timex.today(),
source: "Google Analytics",
status: "importing"
@@ -40,6 +41,35 @@ defmodule Plausible.Workers.ImportGoogleAnalyticsTest do
assert Repo.reload!(site).imported_data.status == "ok"
end
+ test "updates the stats_start_date field for the site after succesful import" do
+ user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))
+ site = insert(:site, members: [user], imported_data: @imported_data)
+
+ api_stub =
+ stub(Plausible.Google.Api, :import_analytics, fn _site,
+ _view_id,
+ _start_date,
+ _end_date,
+ _access_token ->
+ {:ok, nil}
+ end)
+
+ ImportGoogleAnalytics.perform(
+ %Oban.Job{
+ args: %{
+ "site_id" => site.id,
+ "view_id" => "view_id",
+ "start_date" => "2020-01-01",
+ "end_date" => "2022-01-01",
+ "access_token" => "token"
+ }
+ },
+ api_stub
+ )
+
+ assert Repo.reload!(site).stats_start_date == @imported_data.start_date
+ end
+
test "sends email to owner after succesful import" do
user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))
site = insert(:site, members: [user], imported_data: @imported_data)