Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Improve CSV importer tests and visit duration query for imported data (#4114)
Adrian Gruntkowski7876802d055d
diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.exindex 0d24634459..b6623587aa 100644--- a/lib/plausible/stats/imported.ex+++ b/lib/plausible/stats/imported.ex@@ -632,7 +632,7 @@ defmodule Plausible.Stats.Imported do"""if(? + ? > 0,- round((? + ? * ?) / (? + ?), 1),+ round((? + ? * ?) / (? + ?), 0),0)""",diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exsindex e32102eaaf..b752702731 100644--- a/test/plausible/imported/csv_importer_test.exs+++ b/test/plausible/imported/csv_importer_test.exs@@ -561,24 +561,14 @@ defmodule Plausible.Imported.CSVImporterTest do}end- breakdown = fn params_or_site, by, opts ->- metrics =- Keyword.get(opts, :metrics, "visitors,visits,pageviews,visit_duration,bounce_rate")-- filters = Keyword.get(opts, :filters, "")-- by_without_prefix =- by- |> String.trim_leading("event:")- |> String.trim_leading("visit:")- |> String.trim_leading("props:")+ breakdown = fn params_or_site, by ->+ by_without_prefix = by |> Plausible.Stats.Filters.without_prefix() |> to_string()params =case params_or_site do%Plausible.Site{} = site ->common_params.(site)- |> Map.put("metrics", metrics)- |> Map.put("filters", filters)+ |> Map.put("metrics", "visitors,visits,pageviews,visit_duration,bounce_rate")|> Map.put("limit", 1000)|> Map.put("property", by)@@ -592,6 +582,22 @@ defmodule Plausible.Imported.CSVImporterTest do)end+ goal_breakdown = fn site, by, filters ->+ by_without_prefix = by |> Plausible.Stats.Filters.without_prefix() |> to_string()++ params =+ common_params.(site)+ |> Map.put("metrics", "visitors,events,conversion_rate")+ |> Map.put("filters", filters)+ |> Map.put("limit", 1000)+ |> Map.put("property", by)++ Enum.sort_by(+ results.("/api/v1/stats/breakdown", params),+ &Map.fetch!(&1, by_without_prefix)+ )+ end+# timeseriestimeseries_params = fn site ->Map.put(@@ -634,37 +640,19 @@ defmodule Plausible.Imported.CSVImporterTest do|> Map.put("property", "event:page")end- exported_pages = breakdown.(pages_params.(exported_site), "visit:page", [])- imported_pages = breakdown.(pages_params.(imported_site), "visit:page", [])+ exported_pages = breakdown.(pages_params.(exported_site), "visit:page")+ imported_pages = breakdown.(pages_params.(imported_site), "visit:page")pairwise(exported_pages, imported_pages, fn exported, imported ->assert exported["page"] == imported["page"]assert exported["pageviews"] == imported["pageviews"]+ assert exported["visit_duration"] == imported["visit_duration"]assert exported["bounce_rate"] == imported["bounce_rate"]# time on page is not being exported/imported right nowassert imported["time_on_page"] == 0end)- # NOTE: page breakdown's visitors difference is up to almost 43%- assert summary(field(exported_pages, "visit_duration")) == [0, 0.0, 13.0, 46.5, 264]- assert summary(field(imported_pages, "visit_duration")) == [0.0, 0.0, 13.3, 46.6, 264.5]-- assert summary(- pairwise(exported_pages, imported_pages, fn exported, imported ->- e = exported["visit_duration"]- i = imported["visit_duration"]-- if is_number(e) and is_number(i) and i > 0 do- abs(1 - e / i)- else- # both nil or both zero- assert e == i- _no_diff = 0- end- end)- ) == [0.0, 0.0, 0.0, 0.008812019016100597, 0.4285714285714286]-# NOTE: page breakdown's visitors difference is up to 28%assert summary(field(exported_pages, "visitors")) == [1, 1.0, 5.5, 17.75, 495]assert summary(field(imported_pages, "visitors")) == [1, 1.75, 5.5, 20.0, 511]@@ -694,8 +682,8 @@ defmodule Plausible.Imported.CSVImporterTest do) == [0, 0.0, 0.0, 0.0, 0.06666666666666665]# sources- exported_sources = breakdown.(exported_site, "visit:source", [])- imported_sources = breakdown.(imported_site, "visit:source", [])+ exported_sources = breakdown.(exported_site, "visit:source")+ imported_sources = breakdown.(imported_site, "visit:source")pairwise(exported_sources, imported_sources, fn exported, imported ->assert exported["source"] == imported["source"]@@ -716,12 +704,12 @@ defmodule Plausible.Imported.CSVImporterTest do) == [0.0, 0.0, 0.0, 0.0, 0.11111111111111116]# utm mediums- assert breakdown.(exported_site, "visit:utm_medium", []) ==- breakdown.(imported_site, "visit:utm_medium", [])+ assert breakdown.(exported_site, "visit:utm_medium") ==+ breakdown.(imported_site, "visit:utm_medium")# entry pages- exported_entry_pages = breakdown.(exported_site, "visit:entry_page", [])- imported_entry_pages = breakdown.(imported_site, "visit:entry_page", [])+ exported_entry_pages = breakdown.(exported_site, "visit:entry_page")+ imported_entry_pages = breakdown.(imported_site, "visit:entry_page")pairwise(exported_entry_pages, imported_entry_pages, fn exported, imported ->assert exported["entry_page"] == imported["entry_page"]@@ -742,17 +730,53 @@ defmodule Plausible.Imported.CSVImporterTest do) == [0, 0, 0, 0, 0.5]# cities- exported_cities = breakdown.(exported_site, "visit:city", [])- imported_cities = breakdown.(imported_site, "visit:city", [])+ exported_cities = breakdown.(exported_site, "visit:city")+ imported_cities = breakdown.(imported_site, "visit:city")pairwise(exported_cities, imported_cities, fn exported, imported ->assert exported["city"] == imported["city"]- assert_in_delta exported["bounce_rate"], imported["bounce_rate"], 4assert exported["pageviews"] == imported["pageviews"]- assert_in_delta exported["visit_duration"], imported["visit_duration"], 2assert_in_delta exported["visits"], imported["visits"], 1end)+ # NOTE: city breakdown's bounce rate difference is up to 4%+ assert summary(field(exported_cities, "bounce_rate")) == [0, 87.0, 100.0, 100.0, 100]+ assert summary(field(imported_cities, "bounce_rate")) == [0.0, 87.0, 100.0, 100.0, 100.0]++ assert summary(+ pairwise(exported_cities, imported_cities, fn exported, imported ->+ e = exported["bounce_rate"]+ i = imported["bounce_rate"]++ if is_number(e) and is_number(i) and i > 0 do+ abs(1 - e / i)+ else+ # both nil or both zero+ assert e == i+ _no_diff = 0+ end+ end)+ ) == [0.0, 0.0, 0.0, 0.0, 0.03614457831325302]++ # NOTE: city breakdown's visit duration difference is up to 14%+ assert summary(field(exported_cities, "visit_duration")) == [0, 0.0, 0.0, 1.0, 1718]+ assert summary(field(imported_cities, "visit_duration")) == [0.0, 0.0, 0.0, 1.0, 1718.0]++ assert summary(+ pairwise(exported_cities, imported_cities, fn exported, imported ->+ e = exported["visit_duration"]+ i = imported["visit_duration"]++ if is_number(e) and is_number(i) and i > 0 do+ abs(1 - e / i)+ else+ # both nil or both zero+ assert e == i+ _no_diff = 0+ end+ end)+ ) == [0, 0.0, 0.0, 0.0, 0.1428571428571429]+# NOTE: city breakdown's visitors relative difference is up to 27%assert summary(field(exported_cities, "visitors")) == [1, 1.0, 1.0, 2.0, 22]assert summary(field(imported_cities, "visitors")) == [1, 1.0, 1.0, 2.0, 30]@@ -768,8 +792,8 @@ defmodule Plausible.Imported.CSVImporterTest do) == [0, 0.0, 0.0, 0.0, 0.2666666666666667]# devices- exported_devices = breakdown.(exported_site, "visit:device", [])- imported_devices = breakdown.(imported_site, "visit:device", [])+ exported_devices = breakdown.(exported_site, "visit:device")+ imported_devices = breakdown.(imported_site, "visit:device")pairwise(exported_devices, imported_devices, fn exported, imported ->assert exported["device"] == imported["device"]@@ -790,8 +814,8 @@ defmodule Plausible.Imported.CSVImporterTest do) == [0.0, 0.0, 0.011378002528445008, 0.025028787232157956, 0.031847133757961776]# browsers- exported_browsers = breakdown.(exported_site, "visit:browser", [])- imported_browsers = breakdown.(imported_site, "visit:browser", [])+ exported_browsers = breakdown.(exported_site, "visit:browser")+ imported_browsers = breakdown.(imported_site, "visit:browser")pairwise(exported_browsers, imported_browsers, fn exported, imported ->assert exported["browser"] == imported["browser"]@@ -812,8 +836,8 @@ defmodule Plausible.Imported.CSVImporterTest do) == [0.0, 0.0, 0.0, 0.0, 0.037122969837587005]# os- exported_os = breakdown.(exported_site, "visit:os", [])- imported_os = breakdown.(imported_site, "visit:os", [])+ exported_os = breakdown.(exported_site, "visit:os")+ imported_os = breakdown.(imported_site, "visit:os")pairwise(exported_os, imported_os, fn exported, imported ->assert exported["os"] == imported["os"]@@ -834,8 +858,8 @@ defmodule Plausible.Imported.CSVImporterTest do) == [0.0, 0.0, 0.0, 0.01754385964912286, 0.045592705167173286]# os versions- exported_os_versions = breakdown.(exported_site, "visit:os_version", [])- imported_os_versions = breakdown.(imported_site, "visit:os_version", [])+ exported_os_versions = breakdown.(exported_site, "visit:os_version")+ imported_os_versions = breakdown.(imported_site, "visit:os_version")pairwise(exported_os_versions, imported_os_versions, fn exported, imported ->assert exported["os_version"] == imported["os_version"]@@ -856,11 +880,8 @@ defmodule Plausible.Imported.CSVImporterTest do) == [0.0, 0.0, 0.0, 0.0, 0.33333333333333337]# goals- exported_goals =- breakdown.(exported_site, "event:goal", metrics: "visitors,events,conversion_rate")-- imported_goals =- breakdown.(imported_site, "event:goal", metrics: "visitors,events,conversion_rate")+ exported_goals = goal_breakdown.(exported_site, "event:goal", "")+ imported_goals = goal_breakdown.(imported_site, "event:goal", "")# NOTE: goal breakdown's visitors difference is up to 20%assert summary(field(exported_goals, "visitors")) == [1, 6.5, 12.0, 41.0, 70]@@ -907,16 +928,10 @@ defmodule Plausible.Imported.CSVImporterTest do# url property breakdownexported_url_props =- breakdown.(exported_site, "event:props:url",- filters: "event:goal==Outbound Link: Click",- metrics: "visitors,events,conversion_rate"- )+ goal_breakdown.(exported_site, "event:props:url", "event:goal==Outbound Link: Click")imported_url_props =- breakdown.(imported_site, "event:props:url",- filters: "event:goal==Outbound Link: Click",- metrics: "visitors,events,conversion_rate"- )+ goal_breakdown.(imported_site, "event:props:url", "event:goal==Outbound Link: Click")pairwise(exported_url_props, imported_url_props, fn exported, imported ->assert exported["visitors"] == imported["visitors"]@@ -934,17 +949,8 @@ defmodule Plausible.Imported.CSVImporterTest do) == [0.0, 0.0, 0.0, 0.0, 0.19999999999999996]# path property breakdown- exported_path_props =- breakdown.(exported_site, "event:props:path",- filters: "event:goal==404",- metrics: "visitors,events,conversion_rate"- )-- imported_path_props =- breakdown.(imported_site, "event:props:path",- filters: "event:goal==404",- metrics: "visitors,events,conversion_rate"- )+ exported_path_props = goal_breakdown.(exported_site, "event:props:path", "event:goal==404")+ imported_path_props = goal_breakdown.(imported_site, "event:props:path", "event:goal==404")pairwise(exported_path_props, imported_path_props, fn exported, imported ->assert exported["visitors"] == imported["visitors"]diff --git a/test/plausible/imported/google_analytics4_test.exs b/test/plausible/imported/google_analytics4_test.exsindex 7bde5f0673..7253877c4c 100644--- a/test/plausible/imported/google_analytics4_test.exs+++ b/test/plausible/imported/google_analytics4_test.exs@@ -503,7 +503,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do"bounce_rate" => 58.0,"pageviews" => 491,"utm_medium" => "referral",- "visit_duration" => 27.5,+ "visit_duration" => 27.0,"visitors" => 294,"visits" => 298}@@ -521,7 +521,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.first(results) == %{"bounce_rate" => 35.0,"pageviews" => 838,- "visit_duration" => 43.1,+ "visit_duration" => 43.0,"visitors" => 675,"visits" => 712,"entry_page" => "/brza-kukuruza"@@ -549,7 +549,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do"bounce_rate" => 35.0,"city" => 792_680,"pageviews" => 1650,- "visit_duration" => 38.9,+ "visit_duration" => 39.0,"visitors" => 1233,"visits" => 1273}@@ -575,7 +575,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.first(results) == %{"bounce_rate" => 38.0,"pageviews" => 7041,- "visit_duration" => 36.6,+ "visit_duration" => 37.0,"visitors" => 5277,"visits" => 5532,"device" => "Mobile"@@ -584,7 +584,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.last(results) == %{"bounce_rate" => 37.0,"pageviews" => 143,- "visit_duration" => 59.8,+ "visit_duration" => 60.0,"visitors" => 97,"visits" => 100,"device" => "Tablet"@@ -602,7 +602,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.first(results) == %{"bounce_rate" => 33.0,"pageviews" => 8143,- "visit_duration" => 50.2,+ "visit_duration" => 50.0,"visitors" => 4625,"visits" => 4655,"browser" => "Chrome"@@ -629,7 +629,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.first(results) == %{"bounce_rate" => 34.0,"pageviews" => 5827,- "visit_duration" => 40.6,+ "visit_duration" => 41.0,"visitors" => 4319,"visits" => 4495,"os" => "Android"@@ -658,7 +658,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do"os" => "Android","os_version" => "13.0.0","pageviews" => 1673,- "visit_duration" => 42.4,+ "visit_duration" => 42.0,"visitors" => 1247,"visits" => 1295}diff --git a/test/plausible_web/controllers/api/stats_controller/imported_test.exs b/test/plausible_web/controllers/api/stats_controller/imported_test.exsindex d460733e72..42eafec869 100644--- a/test/plausible_web/controllers/api/stats_controller/imported_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/imported_test.exs@@ -514,7 +514,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do"name" => "Sweden","visitors" => 3,"bounce_rate" => 67.0,- "visit_duration" => 33.3+ "visit_duration" => 33.0},%{"name" => "oat milk",@@ -1158,7 +1158,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest dovisit_duration = Enum.find(top_stats, fn stat -> stat["name"] == "Visit duration" end)- assert visit_duration["value"] == 3_479_033+ assert visit_duration["value"] == 3_479_032endtest "skips empty dates from import", %{conn: conn, site: site, import_id: import_id} do