Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Improve merge imported (#4003)
RobertJoonasfce909d041c8
diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex c110a4b625..795c400de5 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -310,7 +310,6 @@ defmodule Plausible.Stats.Breakdown doq|> apply_pagination(pagination)|> ClickhouseRepo.all()- |> transform_keys(%{operating_system: :os})|> Util.keep_requested_metrics(metrics)end@@ -601,8 +600,7 @@ defmodule Plausible.Stats.Breakdown dos in q,group_by: s.operating_system,select_merge: %{- operating_system:- fragment("if(empty(?), ?, ?)", s.operating_system, @not_set, s.operating_system)+ os: fragment("if(empty(?), ?, ?)", s.operating_system, @not_set, s.operating_system)},order_by: {:asc, s.operating_system})@@ -651,7 +649,6 @@ defmodule Plausible.Stats.Breakdown doenddefp group_by_field_names("event:props:" <> _prop), do: [:name]- defp group_by_field_names("visit:os"), do: [:operating_system]defp group_by_field_names("visit:os_version"), do: [:os, :os_version]defp group_by_field_names("visit:browser_version"), do: [:browser, :browser_version]diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.exindex 0346751ad3..ccd8a04cd7 100644--- a/lib/plausible/stats/imported.ex+++ b/lib/plausible/stats/imported.ex@@ -6,6 +6,30 @@ defmodule Plausible.Stats.Imported doimport Plausible.Stats.Fragments@no_ref "Direct / None"+ @not_set "(not set)"++ @property_to_table_mappings %{+ "visit:source" => "imported_sources",+ "visit:referrer" => "imported_sources",+ "visit:utm_source" => "imported_sources",+ "visit:utm_medium" => "imported_sources",+ "visit:utm_campaign" => "imported_sources",+ "visit:utm_term" => "imported_sources",+ "visit:utm_content" => "imported_sources",+ "visit:entry_page" => "imported_entry_pages",+ "visit:exit_page" => "imported_exit_pages",+ "visit:country" => "imported_locations",+ "visit:region" => "imported_locations",+ "visit:city" => "imported_locations",+ "visit:device" => "imported_devices",+ "visit:browser" => "imported_browsers",+ "visit:browser_version" => "imported_browsers",+ "visit:os" => "imported_operating_systems",+ "visit:os_version" => "imported_operating_systems",+ "event:page" => "imported_pages"+ }++ @imported_properties Map.keys(@property_to_table_mappings)def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{include_imported: false}, _),do: native_q@@ -56,268 +80,46 @@ defmodule Plausible.Stats.Imported dodef merge_imported(q, _, %Query{include_imported: false}, _, _), do: qdef merge_imported(q, _, _, _, [:events | _]), do: q- # GA only has 'source'- def merge_imported(q, _, _, "utm_source", _), do: qdef merge_imported(q, site, query, property, metrics)- when property in [- "visit:source",- "visit:utm_medium",- "visit:utm_campaign",- "visit:utm_term",- "visit:utm_content",- "visit:entry_page",- "visit:exit_page",- "visit:country",- "visit:region",- "visit:city",- "visit:device",- "visit:browser",- "visit:os",- "event:page"- ] do- {table, dim} =- case property do- "visit:country" ->- {"imported_locations", :country}-- "visit:region" ->- {"imported_locations", :region}-- "visit:city" ->- {"imported_locations", :city}-- "visit:utm_medium" ->- {"imported_sources", :utm_medium}-- "visit:utm_campaign" ->- {"imported_sources", :utm_campaign}-- "visit:utm_term" ->- {"imported_sources", :utm_term}-- "visit:utm_content" ->- {"imported_sources", :utm_content}-- "visit:os" ->- {"imported_operating_systems", :operating_system}-- "event:page" ->- {"imported_pages", :page}-- _ ->- dim = String.trim_leading(property, "visit:")- {"imported_#{dim}s", String.to_existing_atom(dim)}- end-+ when property in @imported_properties do+ table = Map.fetch!(@property_to_table_mappings, property)+ dim = Plausible.Stats.Filters.without_prefix(property)import_ids = site.complete_import_idsimported_q =from(i in table,- group_by: field(i, ^dim),where: i.site_id == ^site.id,where: i.import_id in ^import_ids,where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,where: i.visitors > 0,select: %{})+ |> maybe_filter_by_breakdown_property(query.filters[property], dim)+ |> group_imported_by(dim)|> select_imported_metrics(metrics)- imported_q =- case query.filters[property] do- {:is_not, value} ->- value = if value == @no_ref, do: "", else: value- where(imported_q, [i], field(i, ^dim) != ^value)-- {:member, list} ->- where(imported_q, [i], field(i, ^dim) in ^list)+ join_on =+ case dim do+ :os_version ->+ dynamic([s, i], s.os == i.os and s.os_version == i.os_version)- _ ->- imported_q- end+ :browser_version ->+ dynamic([s, i], s.browser == i.browser and s.browser_version == i.browser_version)- imported_q =- case dim do- :source ->- imported_q- |> select_merge([i], %{- source: fragment("if(empty(?), ?, ?)", i.source, @no_ref, i.source)- })-- :utm_medium ->- imported_q- |> where([i], fragment("not empty(?)", i.utm_medium))- |> select_merge([i], %{- utm_medium: i.utm_medium- })-- :utm_campaign ->- imported_q- |> where([i], fragment("not empty(?)", i.utm_campaign))- |> select_merge([i], %{- utm_campaign: i.utm_campaign- })-- :utm_term ->- imported_q- |> where([i], fragment("not empty(?)", i.utm_term))- |> select_merge([i], %{- utm_term: i.utm_term- })-- :utm_content ->- imported_q- |> where([i], fragment("not empty(?)", i.utm_content))- |> select_merge([i], %{- utm_content: i.utm_content- })-- :page ->- imported_q- |> select_merge([i], %{- page: i.page,- time_on_page: sum(i.time_on_page)- })-- :entry_page ->- imported_q- |> select_merge([i], %{- entry_page: i.entry_page- })-- :exit_page ->- imported_q- |> select_merge([i], %{exit_page: i.exit_page})-- :country ->- imported_q |> where([i], i.country != "ZZ") |> select_merge([i], %{country: i.country})-- :region ->- imported_q |> where([i], i.region != "") |> select_merge([i], %{region: i.region})-- :city ->- imported_q- |> where([i], i.city != 0 and not is_nil(i.city))- |> select_merge([i], %{city: i.city})-- :device ->- imported_q |> select_merge([i], %{device: i.device})-- :browser ->- imported_q |> select_merge([i], %{browser: i.browser})-- :operating_system ->- imported_q |> select_merge([i], %{operating_system: i.operating_system})+ dim ->+ dynamic([s, i], field(s, ^dim) == field(i, ^dim))end- q =- from(s in Ecto.Query.subquery(q),- full_join: i in subquery(imported_q),- on: field(s, ^dim) == field(i, ^dim),- select: %{}- )- |> select_joined_metrics(metrics)- |> apply_order_by(metrics)-- case dim do- :source ->- q- |> select_merge([s, i], %{- source: fragment("if(empty(?), ?, ?)", s.source, i.source, s.source)- })-- :utm_medium ->- q- |> select_merge([s, i], %{- utm_medium: fragment("if(empty(?), ?, ?)", s.utm_medium, i.utm_medium, s.utm_medium)- })-- :utm_source ->- q- |> select_merge([s, i], %{- utm_source: fragment("if(empty(?), ?, ?)", s.utm_source, i.utm_source, s.utm_source)- })-- :utm_campaign ->- q- |> select_merge([s, i], %{- utm_campaign:- fragment("if(empty(?), ?, ?)", s.utm_campaign, i.utm_campaign, s.utm_campaign)- })-- :utm_term ->- q- |> select_merge([s, i], %{- utm_term: fragment("if(empty(?), ?, ?)", s.utm_term, i.utm_term, s.utm_term)- })-- :utm_content ->- q- |> select_merge([s, i], %{- utm_content: fragment("if(empty(?), ?, ?)", s.utm_content, i.utm_content, s.utm_content)- })-- :page ->- q- |> select_merge([s, i], %{- page: fragment("if(empty(?), ?, ?)", i.page, s.page, i.page)- })-- :entry_page ->- q- |> select_merge([s, i], %{- entry_page: fragment("if(empty(?), ?, ?)", i.entry_page, s.entry_page, i.entry_page)- })-- :exit_page ->- q- |> select_merge([s, i], %{- exit_page: fragment("if(empty(?), ?, ?)", i.exit_page, s.exit_page, i.exit_page)- })-- :country ->- q- |> select_merge([i, s], %{- country: fragment("if(empty(?), ?, ?)", s.country, i.country, s.country)- })-- :region ->- q- |> select_merge([i, s], %{- region: fragment("if(empty(?), ?, ?)", s.region, i.region, s.region)- })-- :city ->- q- |> select_merge([s, i], %{- city: fragment("coalesce(nullif(?, 0), ?)", i.city, s.city)- })-- :device ->- q- |> select_merge([i, s], %{- device: fragment("if(empty(?), ?, ?)", s.device, i.device, s.device)- })-- :browser ->- q- |> select_merge([i, s], %{- browser: fragment("if(empty(?), ?, ?)", s.browser, i.browser, s.browser)- })-- :operating_system ->- q- |> select_merge([i, s], %{- operating_system:- fragment(- "if(empty(?), ?, ?)",- s.operating_system,- i.operating_system,- s.operating_system- )- })- end+ from(s in Ecto.Query.subquery(q),+ full_join: i in subquery(imported_q),+ on: ^join_on,+ select: %{}+ )+ |> select_joined_dimension(dim)+ |> select_joined_metrics(metrics)+ |> apply_order_by(metrics)enddef merge_imported(q, site, query, :aggregate, metrics) do@@ -352,6 +154,12 @@ defmodule Plausible.Stats.Imported do)end+ defp maybe_filter_by_breakdown_property(q, {:member, list}, dim) do+ where(q, [i], field(i, ^dim) in ^list)+ end++ defp maybe_filter_by_breakdown_property(q, _, _), do: q+defp select_imported_metrics(q, []), do: qdefp select_imported_metrics(q, [:visitors | rest]) do@@ -448,6 +256,128 @@ defmodule Plausible.Stats.Imported do|> select_imported_metrics(rest)end+ defp group_imported_by(q, dim) when dim in [:source, :referrer] do+ q+ |> group_by([i], field(i, ^dim))+ |> select_merge([i], %{+ ^dim => fragment("if(empty(?), ?, ?)", field(i, ^dim), @no_ref, field(i, ^dim))+ })+ end++ defp group_imported_by(q, dim)+ when dim in [:utm_source, :utm_medium, :utm_campaign, :utm_term, :utm_content] do+ q+ |> group_by([i], field(i, ^dim))+ |> where([i], fragment("not empty(?)", field(i, ^dim)))+ |> select_merge([i], %{^dim => field(i, ^dim)})+ end++ defp group_imported_by(q, :page) do+ q+ |> group_by([i], i.page)+ |> select_merge([i], %{page: i.page, time_on_page: sum(i.time_on_page)})+ end++ defp group_imported_by(q, :country) do+ q+ |> group_by([i], i.country)+ |> where([i], i.country != "ZZ")+ |> select_merge([i], %{country: i.country})+ end++ defp group_imported_by(q, :region) do+ q+ |> group_by([i], i.region)+ |> where([i], i.region != "")+ |> select_merge([i], %{region: i.region})+ end++ defp group_imported_by(q, :city) do+ q+ |> group_by([i], i.city)+ |> where([i], i.city != 0 and not is_nil(i.city))+ |> select_merge([i], %{city: i.city})+ end++ defp group_imported_by(q, dim) when dim in [:device, :browser] do+ q+ |> group_by([i], field(i, ^dim))+ |> select_merge([i], %{+ ^dim => fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim))+ })+ end++ defp group_imported_by(q, :browser_version) do+ q+ |> group_by([i], [i.browser, i.browser_version])+ |> select_merge([i], %{+ browser: fragment("if(empty(?), ?, ?)", i.browser, @not_set, i.browser),+ browser_version:+ fragment(+ "if(empty(?), ?, ?)",+ i.browser_version,+ @not_set,+ i.browser_version+ )+ })+ end++ defp group_imported_by(q, :os) do+ q+ |> group_by([i], i.operating_system)+ |> select_merge([i], %{+ os: fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system)+ })+ end++ defp group_imported_by(q, :os_version) do+ q+ |> group_by([i], [i.operating_system, i.operating_system_version])+ |> select_merge([i], %{+ os: fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system),+ os_version:+ fragment(+ "if(empty(?), ?, ?)",+ i.operating_system_version,+ @not_set,+ i.operating_system_version+ )+ })+ end++ defp group_imported_by(q, dim) when dim in [:entry_page, :exit_page] do+ q+ |> group_by([i], field(i, ^dim))+ |> select_merge([i], %{^dim => field(i, ^dim)})+ end++ defp select_joined_dimension(q, :city) do+ select_merge(q, [s, i], %{+ city: fragment("greatest(?,?)", i.city, s.city)+ })+ end++ defp select_joined_dimension(q, :os_version) do+ select_merge(q, [s, i], %{+ os: fragment("if(empty(?), ?, ?)", s.os, i.os, s.os),+ os_version: fragment("if(empty(?), ?, ?)", s.os_version, i.os_version, s.os_version)+ })+ end++ defp select_joined_dimension(q, :browser_version) do+ select_merge(q, [s, i], %{+ browser: fragment("if(empty(?), ?, ?)", s.browser, i.browser, s.browser),+ browser_version:+ fragment("if(empty(?), ?, ?)", s.browser_version, i.browser_version, s.browser_version)+ })+ end++ defp select_joined_dimension(q, dim) do+ select_merge(q, [s, i], %{+ ^dim => fragment("if(empty(?), ?, ?)", field(s, ^dim), field(i, ^dim), field(s, ^dim))+ })+ end+defp select_joined_metrics(q, []), do: q# TODO: Reverse-engineering the native data bounces and total visit# durations to combine with imported data is inefficient. Instead bothdiff --git a/test/plausible/imported/google_analytics4_test.exs b/test/plausible/imported/google_analytics4_test.exsindex d81b36bfe0..6874441834 100644--- a/test/plausible/imported/google_analytics4_test.exs+++ b/test/plausible/imported/google_analytics4_test.exs@@ -36,8 +36,8 @@ defmodule Plausible.Imported.GoogleAnalytics4Test douser,label: "properties/123456",property: "properties/123456",- start_date: ~D[2024-02-20],- end_date: Date.utc_today(),+ start_date: ~D[2024-01-01],+ end_date: ~D[2024-01-31],access_token: "redacted_access_token",refresh_token: "redacted_refresh_token",token_expires_at: DateTime.to_iso8601(past)diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exsindex 65bfd89e08..42c50d12c5 100644--- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs@@ -329,6 +329,91 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do}end+ test "breaks down all metrics by visit:referrer with imported data", %{conn: conn, site: site} do+ site =+ site+ |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")+ |> Plausible.Repo.update!()++ populate_stats(site, [+ build(:pageview, referrer: "site.com", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, referrer: "site.com/1", timestamp: ~N[2021-01-01 00:00:00]),+ build(:imported_sources,+ referrer: "site.com",+ date: ~D[2021-01-01],+ visitors: 2,+ visits: 2,+ pageviews: 2,+ bounces: 1,+ visit_duration: 120+ ),+ build(:imported_sources,+ referrer: "site.com/2",+ date: ~D[2021-01-01],+ visitors: 2,+ visits: 2,+ pageviews: 2,+ bounces: 2,+ visit_duration: 0+ ),+ build(:imported_sources,+ date: ~D[2021-01-01],+ visitors: 10,+ visits: 11,+ pageviews: 50,+ bounces: 0,+ visit_duration: 1100+ )+ ])++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "period" => "day",+ "metrics" => "visitors,visits,pageviews,bounce_rate,visit_duration",+ "date" => "2021-01-01",+ "property" => "visit:referrer",+ "with_imported" => "true"+ })++ assert json_response(conn, 200) == %{+ "results" => [+ %{+ "referrer" => "Direct / None",+ "visitors" => 10,+ "visits" => 11,+ "pageviews" => 50,+ "bounce_rate" => 0,+ "visit_duration" => 100+ },+ %{+ "referrer" => "site.com",+ "visitors" => 3,+ "visits" => 3,+ "pageviews" => 3,+ "bounce_rate" => 67.0,+ "visit_duration" => 40+ },+ %{+ "referrer" => "site.com/2",+ "visitors" => 2,+ "visits" => 2,+ "pageviews" => 2,+ "bounce_rate" => 100.0,+ "visit_duration" => 0+ },+ %{+ "referrer" => "site.com/1",+ "visitors" => 1,+ "visits" => 1,+ "pageviews" => 1,+ "bounce_rate" => 100.0,+ "visit_duration" => 0+ }+ ]+ }+ end+test "breakdown by visit:utm_medium", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview,@@ -391,6 +476,83 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do}end+ test "breaks down all metrics by visit:utm_source with imported data", %{conn: conn, site: site} do+ site =+ site+ |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")+ |> Plausible.Repo.update!()++ populate_stats(site, [+ build(:pageview, utm_source: "SomeUTMSource", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, utm_source: "SomeUTMSource-1", timestamp: ~N[2021-01-01 00:00:00]),+ build(:imported_sources,+ utm_source: "SomeUTMSource",+ date: ~D[2021-01-01],+ visitors: 2,+ visits: 2,+ pageviews: 2,+ bounces: 1,+ visit_duration: 120+ ),+ build(:imported_sources,+ utm_source: "SomeUTMSource-2",+ date: ~D[2021-01-01],+ visitors: 2,+ visits: 2,+ pageviews: 2,+ bounces: 2,+ visit_duration: 0+ ),+ build(:imported_sources,+ date: ~D[2021-01-01],+ visitors: 10,+ visits: 11,+ pageviews: 50,+ bounces: 0,+ visit_duration: 1100+ )+ ])++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "period" => "day",+ "metrics" => "visitors,visits,pageviews,bounce_rate,visit_duration",+ "date" => "2021-01-01",+ "property" => "visit:utm_source",+ "with_imported" => "true"+ })++ assert json_response(conn, 200) == %{+ "results" => [+ %{+ "utm_source" => "SomeUTMSource",+ "visitors" => 3,+ "visits" => 3,+ "pageviews" => 3,+ "bounce_rate" => 67.0,+ "visit_duration" => 40+ },+ %{+ "utm_source" => "SomeUTMSource-2",+ "visitors" => 2,+ "visits" => 2,+ "pageviews" => 2,+ "bounce_rate" => 100.0,+ "visit_duration" => 0+ },+ %{+ "utm_source" => "SomeUTMSource-1",+ "visitors" => 1,+ "visits" => 1,+ "pageviews" => 1,+ "bounce_rate" => 100.0,+ "visit_duration" => 0+ }+ ]+ }+ end+test "breakdown by visit:utm_campaign", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview,diff --git a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs b/test/plausible_web/controllers/api/stats_controller/browsers_test.exsindex 7d4e94890d..8f0486fa37 100644--- a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs@@ -166,10 +166,27 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do%{"name" => "(not set)", "visitors" => 1, "percentage" => 100.0}]end++ test "select empty imported_browsers as (not set), merging with the native (not set)", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview, user_id: 123),+ build(:imported_browsers, visitors: 1),+ build(:imported_visitors, visitors: 1)+ ])++ conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day&with_imported=true")++ assert json_response(conn, 200) == [+ %{"name" => "(not set)", "visitors" => 2, "percentage" => 100.0}+ ]+ endenddescribe "GET /api/stats/:domain/browser-versions" do- setup [:create_user, :log_in, :create_new_site]+ setup [:create_user, :log_in, :create_new_site, :add_imported_data]test "returns correct conversion_rate when browser_version clashes across browsers", %{conn: conn,@@ -259,5 +276,52 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do}]end++ test "with imported data", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ browser: "Chrome",+ browser_version: "121",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ browser: "Chrome",+ browser_version: "110",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:imported_browsers,+ date: ~D[2021-01-01],+ browser: "Chrome",+ browser_version: "121",+ visitors: 5+ ),+ build(:imported_browsers,+ date: ~D[2021-01-01],+ browser: "Firefox",+ browser_version: "121",+ visitors: 3+ ),+ build(:imported_browsers, date: ~D[2021-01-01], visitors: 10),+ build(:imported_visitors, date: ~D[2021-01-01], visitors: 18)+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/browser-versions?period=day&date=2021-01-01&with_imported=true"+ )++ assert json_response(conn, 200) == [+ %{+ "browser" => "(not set)",+ "name" => "(not set)",+ "visitors" => 10,+ "percentage" => 50.0+ },+ %{"browser" => "Chrome", "name" => "121", "visitors" => 6, "percentage" => 30.0},+ %{"browser" => "Firefox", "name" => "121", "visitors" => 3, "percentage" => 15.0},+ %{"browser" => "Chrome", "name" => "110", "visitors" => 1, "percentage" => 5.0}+ ]+ endendenddiff --git a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exsindex d2f5b70a40..6a2f9d8127 100644--- a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs@@ -1,7 +1,7 @@defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest douse PlausibleWeb.ConnCase- describe "GET /api/stats/:domain/operating_systems" do+ describe "GET /api/stats/:domain/operating-systems" dosetup [:create_user, :log_in, :create_new_site, :add_imported_data]test "returns operating systems by unique visitors", %{conn: conn, site: site} do@@ -46,6 +46,22 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do]end+ test "select empty imported_operating_systems as (not set), merging with the native (not set)",+ %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, user_id: 123),+ build(:imported_operating_systems, visitors: 1),+ build(:imported_visitors, visitors: 1)+ ])++ conn =+ get(conn, "/api/stats/#{site.domain}/operating-systems?period=day&with_imported=true")++ assert json_response(conn, 200) == [+ %{"name" => "(not set)", "visitors" => 2, "percentage" => 100.0}+ ]+ end+test "calculates conversion_rate when filtering for goal", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, user_id: 1, operating_system: "Mac"),@@ -195,7 +211,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest doenddescribe "GET /api/stats/:domain/operating-system-versions" do- setup [:create_user, :log_in, :create_new_site]+ setup [:create_user, :log_in, :create_new_site, :add_imported_data]test "returns top OS versions by unique visitors", %{conn: conn, site: site} dopopulate_stats(site, [@@ -230,5 +246,52 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do%{"name" => "10.15", "visitors" => 1, "percentage" => 33.3, "os" => "Mac"}]end++ test "with imported data", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ operating_system: "Mac",+ operating_system_version: "11",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ operating_system: "Mac",+ operating_system_version: "12",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:imported_operating_systems,+ date: ~D[2021-01-01],+ operating_system: "Mac",+ operating_system_version: "11",+ visitors: 5+ ),+ build(:imported_operating_systems,+ date: ~D[2021-01-01],+ operating_system: "Windows",+ operating_system_version: "11",+ visitors: 3+ ),+ build(:imported_operating_systems, date: ~D[2021-01-01], visitors: 10),+ build(:imported_visitors, date: ~D[2021-01-01], visitors: 18)+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/operating-system-versions?period=day&date=2021-01-01&with_imported=true"+ )++ assert json_response(conn, 200) == [+ %{+ "os" => "(not set)",+ "name" => "(not set)",+ "visitors" => 10,+ "percentage" => 50.0+ },+ %{"os" => "Mac", "name" => "11", "visitors" => 6, "percentage" => 30.0},+ %{"os" => "Windows", "name" => "11", "visitors" => 3, "percentage" => 15.0},+ %{"os" => "Mac", "name" => "12", "visitors" => 1, "percentage" => 5.0}+ ]+ endendenddiff --git a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exsindex 40a23df35c..a25520be2f 100644--- a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs@@ -1,7 +1,7 @@defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest douse PlausibleWeb.ConnCase- describe "GET /api/stats/:domain/browsers" do+ describe "GET /api/stats/:domain/screen-sizes" dosetup [:create_user, :log_in, :create_new_site, :add_imported_data]test "returns screen sizes by new visitors", %{conn: conn, site: site} do@@ -72,6 +72,23 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do]end+ test "select empty imported_devices as (not set), merging with the native (not set)", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview, user_id: 123),+ build(:imported_devices, visitors: 1),+ build(:imported_visitors, visitors: 1)+ ])++ conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day&with_imported=true")++ assert json_response(conn, 200) == [+ %{"name" => "(not set)", "visitors" => 2, "percentage" => 100.0}+ ]+ end+test "returns screen sizes with :is filter on custom pageview props", %{conn: conn,site: site