Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Improve merge imported (#4003)

RobertJoonasfce909d041c8
diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex
index c110a4b625..795c400de5 100644
--- a/lib/plausible/stats/breakdown.ex
+++ b/lib/plausible/stats/breakdown.ex
@@ -310,7 +310,6 @@ defmodule Plausible.Stats.Breakdown do
q
|> apply_pagination(pagination)
|> ClickhouseRepo.all()
- |> transform_keys(%{operating_system: :os})
|> Util.keep_requested_metrics(metrics)
end
@@ -601,8 +600,7 @@ defmodule Plausible.Stats.Breakdown do
s in q,
group_by: s.operating_system,
select_merge: %{
- operating_system:
- fragment("if(empty(?), ?, ?)", s.operating_system, @not_set, s.operating_system)
+ os: fragment("if(empty(?), ?, ?)", s.operating_system, @not_set, s.operating_system)
},
order_by: {:asc, s.operating_system}
)
@@ -651,7 +649,6 @@ defmodule Plausible.Stats.Breakdown do
end
defp group_by_field_names("event:props:" <> _prop), do: [:name]
- defp group_by_field_names("visit:os"), do: [:operating_system]
defp group_by_field_names("visit:os_version"), do: [:os, :os_version]
defp group_by_field_names("visit:browser_version"), do: [:browser, :browser_version]
diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex
index 0346751ad3..ccd8a04cd7 100644
--- a/lib/plausible/stats/imported.ex
+++ b/lib/plausible/stats/imported.ex
@@ -6,6 +6,30 @@ defmodule Plausible.Stats.Imported do
import Plausible.Stats.Fragments
@no_ref "Direct / None"
+ @not_set "(not set)"
+
+ @property_to_table_mappings %{
+ "visit:source" => "imported_sources",
+ "visit:referrer" => "imported_sources",
+ "visit:utm_source" => "imported_sources",
+ "visit:utm_medium" => "imported_sources",
+ "visit:utm_campaign" => "imported_sources",
+ "visit:utm_term" => "imported_sources",
+ "visit:utm_content" => "imported_sources",
+ "visit:entry_page" => "imported_entry_pages",
+ "visit:exit_page" => "imported_exit_pages",
+ "visit:country" => "imported_locations",
+ "visit:region" => "imported_locations",
+ "visit:city" => "imported_locations",
+ "visit:device" => "imported_devices",
+ "visit:browser" => "imported_browsers",
+ "visit:browser_version" => "imported_browsers",
+ "visit:os" => "imported_operating_systems",
+ "visit:os_version" => "imported_operating_systems",
+ "event:page" => "imported_pages"
+ }
+
+ @imported_properties Map.keys(@property_to_table_mappings)
def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{include_imported: false}, _),
do: native_q
@@ -56,268 +80,46 @@ defmodule Plausible.Stats.Imported do
def merge_imported(q, _, %Query{include_imported: false}, _, _), do: q
def merge_imported(q, _, _, _, [:events | _]), do: q
- # GA only has 'source'
- def merge_imported(q, _, _, "utm_source", _), do: q
def merge_imported(q, site, query, property, metrics)
- when property in [
- "visit:source",
- "visit:utm_medium",
- "visit:utm_campaign",
- "visit:utm_term",
- "visit:utm_content",
- "visit:entry_page",
- "visit:exit_page",
- "visit:country",
- "visit:region",
- "visit:city",
- "visit:device",
- "visit:browser",
- "visit:os",
- "event:page"
- ] do
- {table, dim} =
- case property do
- "visit:country" ->
- {"imported_locations", :country}
-
- "visit:region" ->
- {"imported_locations", :region}
-
- "visit:city" ->
- {"imported_locations", :city}
-
- "visit:utm_medium" ->
- {"imported_sources", :utm_medium}
-
- "visit:utm_campaign" ->
- {"imported_sources", :utm_campaign}
-
- "visit:utm_term" ->
- {"imported_sources", :utm_term}
-
- "visit:utm_content" ->
- {"imported_sources", :utm_content}
-
- "visit:os" ->
- {"imported_operating_systems", :operating_system}
-
- "event:page" ->
- {"imported_pages", :page}
-
- _ ->
- dim = String.trim_leading(property, "visit:")
- {"imported_#{dim}s", String.to_existing_atom(dim)}
- end
-
+ when property in @imported_properties do
+ table = Map.fetch!(@property_to_table_mappings, property)
+ dim = Plausible.Stats.Filters.without_prefix(property)
import_ids = site.complete_import_ids
imported_q =
from(
i in table,
- group_by: field(i, ^dim),
where: i.site_id == ^site.id,
where: i.import_id in ^import_ids,
where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,
where: i.visitors > 0,
select: %{}
)
+ |> maybe_filter_by_breakdown_property(query.filters[property], dim)
+ |> group_imported_by(dim)
|> select_imported_metrics(metrics)
- imported_q =
- case query.filters[property] do
- {:is_not, value} ->
- value = if value == @no_ref, do: "", else: value
- where(imported_q, [i], field(i, ^dim) != ^value)
-
- {:member, list} ->
- where(imported_q, [i], field(i, ^dim) in ^list)
+ join_on =
+ case dim do
+ :os_version ->
+ dynamic([s, i], s.os == i.os and s.os_version == i.os_version)
- _ ->
- imported_q
- end
+ :browser_version ->
+ dynamic([s, i], s.browser == i.browser and s.browser_version == i.browser_version)
- imported_q =
- case dim do
- :source ->
- imported_q
- |> select_merge([i], %{
- source: fragment("if(empty(?), ?, ?)", i.source, @no_ref, i.source)
- })
-
- :utm_medium ->
- imported_q
- |> where([i], fragment("not empty(?)", i.utm_medium))
- |> select_merge([i], %{
- utm_medium: i.utm_medium
- })
-
- :utm_campaign ->
- imported_q
- |> where([i], fragment("not empty(?)", i.utm_campaign))
- |> select_merge([i], %{
- utm_campaign: i.utm_campaign
- })
-
- :utm_term ->
- imported_q
- |> where([i], fragment("not empty(?)", i.utm_term))
- |> select_merge([i], %{
- utm_term: i.utm_term
- })
-
- :utm_content ->
- imported_q
- |> where([i], fragment("not empty(?)", i.utm_content))
- |> select_merge([i], %{
- utm_content: i.utm_content
- })
-
- :page ->
- imported_q
- |> select_merge([i], %{
- page: i.page,
- time_on_page: sum(i.time_on_page)
- })
-
- :entry_page ->
- imported_q
- |> select_merge([i], %{
- entry_page: i.entry_page
- })
-
- :exit_page ->
- imported_q
- |> select_merge([i], %{exit_page: i.exit_page})
-
- :country ->
- imported_q |> where([i], i.country != "ZZ") |> select_merge([i], %{country: i.country})
-
- :region ->
- imported_q |> where([i], i.region != "") |> select_merge([i], %{region: i.region})
-
- :city ->
- imported_q
- |> where([i], i.city != 0 and not is_nil(i.city))
- |> select_merge([i], %{city: i.city})
-
- :device ->
- imported_q |> select_merge([i], %{device: i.device})
-
- :browser ->
- imported_q |> select_merge([i], %{browser: i.browser})
-
- :operating_system ->
- imported_q |> select_merge([i], %{operating_system: i.operating_system})
+ dim ->
+ dynamic([s, i], field(s, ^dim) == field(i, ^dim))
end
- q =
- from(s in Ecto.Query.subquery(q),
- full_join: i in subquery(imported_q),
- on: field(s, ^dim) == field(i, ^dim),
- select: %{}
- )
- |> select_joined_metrics(metrics)
- |> apply_order_by(metrics)
-
- case dim do
- :source ->
- q
- |> select_merge([s, i], %{
- source: fragment("if(empty(?), ?, ?)", s.source, i.source, s.source)
- })
-
- :utm_medium ->
- q
- |> select_merge([s, i], %{
- utm_medium: fragment("if(empty(?), ?, ?)", s.utm_medium, i.utm_medium, s.utm_medium)
- })
-
- :utm_source ->
- q
- |> select_merge([s, i], %{
- utm_source: fragment("if(empty(?), ?, ?)", s.utm_source, i.utm_source, s.utm_source)
- })
-
- :utm_campaign ->
- q
- |> select_merge([s, i], %{
- utm_campaign:
- fragment("if(empty(?), ?, ?)", s.utm_campaign, i.utm_campaign, s.utm_campaign)
- })
-
- :utm_term ->
- q
- |> select_merge([s, i], %{
- utm_term: fragment("if(empty(?), ?, ?)", s.utm_term, i.utm_term, s.utm_term)
- })
-
- :utm_content ->
- q
- |> select_merge([s, i], %{
- utm_content: fragment("if(empty(?), ?, ?)", s.utm_content, i.utm_content, s.utm_content)
- })
-
- :page ->
- q
- |> select_merge([s, i], %{
- page: fragment("if(empty(?), ?, ?)", i.page, s.page, i.page)
- })
-
- :entry_page ->
- q
- |> select_merge([s, i], %{
- entry_page: fragment("if(empty(?), ?, ?)", i.entry_page, s.entry_page, i.entry_page)
- })
-
- :exit_page ->
- q
- |> select_merge([s, i], %{
- exit_page: fragment("if(empty(?), ?, ?)", i.exit_page, s.exit_page, i.exit_page)
- })
-
- :country ->
- q
- |> select_merge([i, s], %{
- country: fragment("if(empty(?), ?, ?)", s.country, i.country, s.country)
- })
-
- :region ->
- q
- |> select_merge([i, s], %{
- region: fragment("if(empty(?), ?, ?)", s.region, i.region, s.region)
- })
-
- :city ->
- q
- |> select_merge([s, i], %{
- city: fragment("coalesce(nullif(?, 0), ?)", i.city, s.city)
- })
-
- :device ->
- q
- |> select_merge([i, s], %{
- device: fragment("if(empty(?), ?, ?)", s.device, i.device, s.device)
- })
-
- :browser ->
- q
- |> select_merge([i, s], %{
- browser: fragment("if(empty(?), ?, ?)", s.browser, i.browser, s.browser)
- })
-
- :operating_system ->
- q
- |> select_merge([i, s], %{
- operating_system:
- fragment(
- "if(empty(?), ?, ?)",
- s.operating_system,
- i.operating_system,
- s.operating_system
- )
- })
- end
+ from(s in Ecto.Query.subquery(q),
+ full_join: i in subquery(imported_q),
+ on: ^join_on,
+ select: %{}
+ )
+ |> select_joined_dimension(dim)
+ |> select_joined_metrics(metrics)
+ |> apply_order_by(metrics)
end
def merge_imported(q, site, query, :aggregate, metrics) do
@@ -352,6 +154,12 @@ defmodule Plausible.Stats.Imported do
)
end
+ defp maybe_filter_by_breakdown_property(q, {:member, list}, dim) do
+ where(q, [i], field(i, ^dim) in ^list)
+ end
+
+ defp maybe_filter_by_breakdown_property(q, _, _), do: q
+
defp select_imported_metrics(q, []), do: q
defp select_imported_metrics(q, [:visitors | rest]) do
@@ -448,6 +256,128 @@ defmodule Plausible.Stats.Imported do
|> select_imported_metrics(rest)
end
+ defp group_imported_by(q, dim) when dim in [:source, :referrer] do
+ q
+ |> group_by([i], field(i, ^dim))
+ |> select_merge([i], %{
+ ^dim => fragment("if(empty(?), ?, ?)", field(i, ^dim), @no_ref, field(i, ^dim))
+ })
+ end
+
+ defp group_imported_by(q, dim)
+ when dim in [:utm_source, :utm_medium, :utm_campaign, :utm_term, :utm_content] do
+ q
+ |> group_by([i], field(i, ^dim))
+ |> where([i], fragment("not empty(?)", field(i, ^dim)))
+ |> select_merge([i], %{^dim => field(i, ^dim)})
+ end
+
+ defp group_imported_by(q, :page) do
+ q
+ |> group_by([i], i.page)
+ |> select_merge([i], %{page: i.page, time_on_page: sum(i.time_on_page)})
+ end
+
+ defp group_imported_by(q, :country) do
+ q
+ |> group_by([i], i.country)
+ |> where([i], i.country != "ZZ")
+ |> select_merge([i], %{country: i.country})
+ end
+
+ defp group_imported_by(q, :region) do
+ q
+ |> group_by([i], i.region)
+ |> where([i], i.region != "")
+ |> select_merge([i], %{region: i.region})
+ end
+
+ defp group_imported_by(q, :city) do
+ q
+ |> group_by([i], i.city)
+ |> where([i], i.city != 0 and not is_nil(i.city))
+ |> select_merge([i], %{city: i.city})
+ end
+
+ defp group_imported_by(q, dim) when dim in [:device, :browser] do
+ q
+ |> group_by([i], field(i, ^dim))
+ |> select_merge([i], %{
+ ^dim => fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim))
+ })
+ end
+
+ defp group_imported_by(q, :browser_version) do
+ q
+ |> group_by([i], [i.browser, i.browser_version])
+ |> select_merge([i], %{
+ browser: fragment("if(empty(?), ?, ?)", i.browser, @not_set, i.browser),
+ browser_version:
+ fragment(
+ "if(empty(?), ?, ?)",
+ i.browser_version,
+ @not_set,
+ i.browser_version
+ )
+ })
+ end
+
+ defp group_imported_by(q, :os) do
+ q
+ |> group_by([i], i.operating_system)
+ |> select_merge([i], %{
+ os: fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system)
+ })
+ end
+
+ defp group_imported_by(q, :os_version) do
+ q
+ |> group_by([i], [i.operating_system, i.operating_system_version])
+ |> select_merge([i], %{
+ os: fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system),
+ os_version:
+ fragment(
+ "if(empty(?), ?, ?)",
+ i.operating_system_version,
+ @not_set,
+ i.operating_system_version
+ )
+ })
+ end
+
+ defp group_imported_by(q, dim) when dim in [:entry_page, :exit_page] do
+ q
+ |> group_by([i], field(i, ^dim))
+ |> select_merge([i], %{^dim => field(i, ^dim)})
+ end
+
+ defp select_joined_dimension(q, :city) do
+ select_merge(q, [s, i], %{
+ city: fragment("greatest(?,?)", i.city, s.city)
+ })
+ end
+
+ defp select_joined_dimension(q, :os_version) do
+ select_merge(q, [s, i], %{
+ os: fragment("if(empty(?), ?, ?)", s.os, i.os, s.os),
+ os_version: fragment("if(empty(?), ?, ?)", s.os_version, i.os_version, s.os_version)
+ })
+ end
+
+ defp select_joined_dimension(q, :browser_version) do
+ select_merge(q, [s, i], %{
+ browser: fragment("if(empty(?), ?, ?)", s.browser, i.browser, s.browser),
+ browser_version:
+ fragment("if(empty(?), ?, ?)", s.browser_version, i.browser_version, s.browser_version)
+ })
+ end
+
+ defp select_joined_dimension(q, dim) do
+ select_merge(q, [s, i], %{
+ ^dim => fragment("if(empty(?), ?, ?)", field(s, ^dim), field(i, ^dim), field(s, ^dim))
+ })
+ end
+
defp select_joined_metrics(q, []), do: q
# TODO: Reverse-engineering the native data bounces and total visit
# durations to combine with imported data is inefficient. Instead both
diff --git a/test/plausible/imported/google_analytics4_test.exs b/test/plausible/imported/google_analytics4_test.exs
index d81b36bfe0..6874441834 100644
--- a/test/plausible/imported/google_analytics4_test.exs
+++ b/test/plausible/imported/google_analytics4_test.exs
@@ -36,8 +36,8 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
user,
label: "properties/123456",
property: "properties/123456",
- start_date: ~D[2024-02-20],
- end_date: Date.utc_today(),
+ start_date: ~D[2024-01-01],
+ end_date: ~D[2024-01-31],
access_token: "redacted_access_token",
refresh_token: "redacted_refresh_token",
token_expires_at: DateTime.to_iso8601(past)
diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
index 65bfd89e08..42c50d12c5 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
@@ -329,6 +329,91 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
}
end
+ test "breaks down all metrics by visit:referrer with imported data", %{conn: conn, site: site} do
+ site =
+ site
+ |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")
+ |> Plausible.Repo.update!()
+
+ populate_stats(site, [
+ build(:pageview, referrer: "site.com", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, referrer: "site.com/1", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:imported_sources,
+ referrer: "site.com",
+ date: ~D[2021-01-01],
+ visitors: 2,
+ visits: 2,
+ pageviews: 2,
+ bounces: 1,
+ visit_duration: 120
+ ),
+ build(:imported_sources,
+ referrer: "site.com/2",
+ date: ~D[2021-01-01],
+ visitors: 2,
+ visits: 2,
+ pageviews: 2,
+ bounces: 2,
+ visit_duration: 0
+ ),
+ build(:imported_sources,
+ date: ~D[2021-01-01],
+ visitors: 10,
+ visits: 11,
+ pageviews: 50,
+ bounces: 0,
+ visit_duration: 1100
+ )
+ ])
+
+ conn =
+ get(conn, "/api/v1/stats/breakdown", %{
+ "site_id" => site.domain,
+ "period" => "day",
+ "metrics" => "visitors,visits,pageviews,bounce_rate,visit_duration",
+ "date" => "2021-01-01",
+ "property" => "visit:referrer",
+ "with_imported" => "true"
+ })
+
+ assert json_response(conn, 200) == %{
+ "results" => [
+ %{
+ "referrer" => "Direct / None",
+ "visitors" => 10,
+ "visits" => 11,
+ "pageviews" => 50,
+ "bounce_rate" => 0,
+ "visit_duration" => 100
+ },
+ %{
+ "referrer" => "site.com",
+ "visitors" => 3,
+ "visits" => 3,
+ "pageviews" => 3,
+ "bounce_rate" => 67.0,
+ "visit_duration" => 40
+ },
+ %{
+ "referrer" => "site.com/2",
+ "visitors" => 2,
+ "visits" => 2,
+ "pageviews" => 2,
+ "bounce_rate" => 100.0,
+ "visit_duration" => 0
+ },
+ %{
+ "referrer" => "site.com/1",
+ "visitors" => 1,
+ "visits" => 1,
+ "pageviews" => 1,
+ "bounce_rate" => 100.0,
+ "visit_duration" => 0
+ }
+ ]
+ }
+ end
+
test "breakdown by visit:utm_medium", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
@@ -391,6 +476,83 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
}
end
+ test "breaks down all metrics by visit:utm_source with imported data", %{conn: conn, site: site} do
+ site =
+ site
+ |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")
+ |> Plausible.Repo.update!()
+
+ populate_stats(site, [
+ build(:pageview, utm_source: "SomeUTMSource", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, utm_source: "SomeUTMSource-1", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:imported_sources,
+ utm_source: "SomeUTMSource",
+ date: ~D[2021-01-01],
+ visitors: 2,
+ visits: 2,
+ pageviews: 2,
+ bounces: 1,
+ visit_duration: 120
+ ),
+ build(:imported_sources,
+ utm_source: "SomeUTMSource-2",
+ date: ~D[2021-01-01],
+ visitors: 2,
+ visits: 2,
+ pageviews: 2,
+ bounces: 2,
+ visit_duration: 0
+ ),
+ build(:imported_sources,
+ date: ~D[2021-01-01],
+ visitors: 10,
+ visits: 11,
+ pageviews: 50,
+ bounces: 0,
+ visit_duration: 1100
+ )
+ ])
+
+ conn =
+ get(conn, "/api/v1/stats/breakdown", %{
+ "site_id" => site.domain,
+ "period" => "day",
+ "metrics" => "visitors,visits,pageviews,bounce_rate,visit_duration",
+ "date" => "2021-01-01",
+ "property" => "visit:utm_source",
+ "with_imported" => "true"
+ })
+
+ assert json_response(conn, 200) == %{
+ "results" => [
+ %{
+ "utm_source" => "SomeUTMSource",
+ "visitors" => 3,
+ "visits" => 3,
+ "pageviews" => 3,
+ "bounce_rate" => 67.0,
+ "visit_duration" => 40
+ },
+ %{
+ "utm_source" => "SomeUTMSource-2",
+ "visitors" => 2,
+ "visits" => 2,
+ "pageviews" => 2,
+ "bounce_rate" => 100.0,
+ "visit_duration" => 0
+ },
+ %{
+ "utm_source" => "SomeUTMSource-1",
+ "visitors" => 1,
+ "visits" => 1,
+ "pageviews" => 1,
+ "bounce_rate" => 100.0,
+ "visit_duration" => 0
+ }
+ ]
+ }
+ end
+
test "breakdown by visit:utm_campaign", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
diff --git a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs
index 7d4e94890d..8f0486fa37 100644
--- a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs
@@ -166,10 +166,27 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
%{"name" => "(not set)", "visitors" => 1, "percentage" => 100.0}
]
end
+
+ test "select empty imported_browsers as (not set), merging with the native (not set)", %{
+ conn: conn,
+ site: site
+ } do
+ populate_stats(site, [
+ build(:pageview, user_id: 123),
+ build(:imported_browsers, visitors: 1),
+ build(:imported_visitors, visitors: 1)
+ ])
+
+ conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day&with_imported=true")
+
+ assert json_response(conn, 200) == [
+ %{"name" => "(not set)", "visitors" => 2, "percentage" => 100.0}
+ ]
+ end
end
describe "GET /api/stats/:domain/browser-versions" do
- setup [:create_user, :log_in, :create_new_site]
+ setup [:create_user, :log_in, :create_new_site, :add_imported_data]
test "returns correct conversion_rate when browser_version clashes across browsers", %{
conn: conn,
@@ -259,5 +276,52 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do
}
]
end
+
+ test "with imported data", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview,
+ browser: "Chrome",
+ browser_version: "121",
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:pageview,
+ browser: "Chrome",
+ browser_version: "110",
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:imported_browsers,
+ date: ~D[2021-01-01],
+ browser: "Chrome",
+ browser_version: "121",
+ visitors: 5
+ ),
+ build(:imported_browsers,
+ date: ~D[2021-01-01],
+ browser: "Firefox",
+ browser_version: "121",
+ visitors: 3
+ ),
+ build(:imported_browsers, date: ~D[2021-01-01], visitors: 10),
+ build(:imported_visitors, date: ~D[2021-01-01], visitors: 18)
+ ])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/browser-versions?period=day&date=2021-01-01&with_imported=true"
+ )
+
+ assert json_response(conn, 200) == [
+ %{
+ "browser" => "(not set)",
+ "name" => "(not set)",
+ "visitors" => 10,
+ "percentage" => 50.0
+ },
+ %{"browser" => "Chrome", "name" => "121", "visitors" => 6, "percentage" => 30.0},
+ %{"browser" => "Firefox", "name" => "121", "visitors" => 3, "percentage" => 15.0},
+ %{"browser" => "Chrome", "name" => "110", "visitors" => 1, "percentage" => 5.0}
+ ]
+ end
end
end
diff --git a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs
index d2f5b70a40..6a2f9d8127 100644
--- a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs
@@ -1,7 +1,7 @@
defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do
use PlausibleWeb.ConnCase
- describe "GET /api/stats/:domain/operating_systems" do
+ describe "GET /api/stats/:domain/operating-systems" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
test "returns operating systems by unique visitors", %{conn: conn, site: site} do
@@ -46,6 +46,22 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do
]
end
+ test "select empty imported_operating_systems as (not set), merging with the native (not set)",
+ %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, user_id: 123),
+ build(:imported_operating_systems, visitors: 1),
+ build(:imported_visitors, visitors: 1)
+ ])
+
+ conn =
+ get(conn, "/api/stats/#{site.domain}/operating-systems?period=day&with_imported=true")
+
+ assert json_response(conn, 200) == [
+ %{"name" => "(not set)", "visitors" => 2, "percentage" => 100.0}
+ ]
+ end
+
test "calculates conversion_rate when filtering for goal", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, user_id: 1, operating_system: "Mac"),
@@ -195,7 +211,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do
end
describe "GET /api/stats/:domain/operating-system-versions" do
- setup [:create_user, :log_in, :create_new_site]
+ setup [:create_user, :log_in, :create_new_site, :add_imported_data]
test "returns top OS versions by unique visitors", %{conn: conn, site: site} do
populate_stats(site, [
@@ -230,5 +246,52 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do
%{"name" => "10.15", "visitors" => 1, "percentage" => 33.3, "os" => "Mac"}
]
end
+
+ test "with imported data", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview,
+ operating_system: "Mac",
+ operating_system_version: "11",
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:pageview,
+ operating_system: "Mac",
+ operating_system_version: "12",
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:imported_operating_systems,
+ date: ~D[2021-01-01],
+ operating_system: "Mac",
+ operating_system_version: "11",
+ visitors: 5
+ ),
+ build(:imported_operating_systems,
+ date: ~D[2021-01-01],
+ operating_system: "Windows",
+ operating_system_version: "11",
+ visitors: 3
+ ),
+ build(:imported_operating_systems, date: ~D[2021-01-01], visitors: 10),
+ build(:imported_visitors, date: ~D[2021-01-01], visitors: 18)
+ ])
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/operating-system-versions?period=day&date=2021-01-01&with_imported=true"
+ )
+
+ assert json_response(conn, 200) == [
+ %{
+ "os" => "(not set)",
+ "name" => "(not set)",
+ "visitors" => 10,
+ "percentage" => 50.0
+ },
+ %{"os" => "Mac", "name" => "11", "visitors" => 6, "percentage" => 30.0},
+ %{"os" => "Windows", "name" => "11", "visitors" => 3, "percentage" => 15.0},
+ %{"os" => "Mac", "name" => "12", "visitors" => 1, "percentage" => 5.0}
+ ]
+ end
end
end
diff --git a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs
index 40a23df35c..a25520be2f 100644
--- a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs
@@ -1,7 +1,7 @@
defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do
use PlausibleWeb.ConnCase
- describe "GET /api/stats/:domain/browsers" do
+ describe "GET /api/stats/:domain/screen-sizes" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
test "returns screen sizes by new visitors", %{conn: conn, site: site} do
@@ -72,6 +72,23 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do
]
end
+ test "select empty imported_devices as (not set), merging with the native (not set)", %{
+ conn: conn,
+ site: site
+ } do
+ populate_stats(site, [
+ build(:pageview, user_id: 123),
+ build(:imported_devices, visitors: 1),
+ build(:imported_visitors, visitors: 1)
+ ])
+
+ conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day&with_imported=true")
+
+ assert json_response(conn, 200) == [
+ %{"name" => "(not set)", "visitors" => 2, "percentage" => 100.0}
+ ]
+ end
+
test "returns screen sizes with :is filter on custom pageview props", %{
conn: conn,
site: site