Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Improve CSV importer tests and visit duration query for imported data (#4114)

Adrian Gruntkowski7876802d055d
diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex
index 0d24634459..b6623587aa 100644
--- a/lib/plausible/stats/imported.ex
+++ b/lib/plausible/stats/imported.ex
@@ -632,7 +632,7 @@ defmodule Plausible.Stats.Imported do
"""
if(
? + ? > 0,
- round((? + ? * ?) / (? + ?), 1),
+ round((? + ? * ?) / (? + ?), 0),
0
)
""",
diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exs
index e32102eaaf..b752702731 100644
--- a/test/plausible/imported/csv_importer_test.exs
+++ b/test/plausible/imported/csv_importer_test.exs
@@ -561,24 +561,14 @@ defmodule Plausible.Imported.CSVImporterTest do
}
end
- breakdown = fn params_or_site, by, opts ->
- metrics =
- Keyword.get(opts, :metrics, "visitors,visits,pageviews,visit_duration,bounce_rate")
-
- filters = Keyword.get(opts, :filters, "")
-
- by_without_prefix =
- by
- |> String.trim_leading("event:")
- |> String.trim_leading("visit:")
- |> String.trim_leading("props:")
+ breakdown = fn params_or_site, by ->
+ by_without_prefix = by |> Plausible.Stats.Filters.without_prefix() |> to_string()
params =
case params_or_site do
%Plausible.Site{} = site ->
common_params.(site)
- |> Map.put("metrics", metrics)
- |> Map.put("filters", filters)
+ |> Map.put("metrics", "visitors,visits,pageviews,visit_duration,bounce_rate")
|> Map.put("limit", 1000)
|> Map.put("property", by)
@@ -592,6 +582,22 @@ defmodule Plausible.Imported.CSVImporterTest do
)
end
+ goal_breakdown = fn site, by, filters ->
+ by_without_prefix = by |> Plausible.Stats.Filters.without_prefix() |> to_string()
+
+ params =
+ common_params.(site)
+ |> Map.put("metrics", "visitors,events,conversion_rate")
+ |> Map.put("filters", filters)
+ |> Map.put("limit", 1000)
+ |> Map.put("property", by)
+
+ Enum.sort_by(
+ results.("/api/v1/stats/breakdown", params),
+ &Map.fetch!(&1, by_without_prefix)
+ )
+ end
+
# timeseries
timeseries_params = fn site ->
Map.put(
@@ -634,37 +640,19 @@ defmodule Plausible.Imported.CSVImporterTest do
|> Map.put("property", "event:page")
end
- exported_pages = breakdown.(pages_params.(exported_site), "visit:page", [])
- imported_pages = breakdown.(pages_params.(imported_site), "visit:page", [])
+ exported_pages = breakdown.(pages_params.(exported_site), "visit:page")
+ imported_pages = breakdown.(pages_params.(imported_site), "visit:page")
pairwise(exported_pages, imported_pages, fn exported, imported ->
assert exported["page"] == imported["page"]
assert exported["pageviews"] == imported["pageviews"]
+ assert exported["visit_duration"] == imported["visit_duration"]
assert exported["bounce_rate"] == imported["bounce_rate"]
# time on page is not being exported/imported right now
assert imported["time_on_page"] == 0
end)
- # NOTE: page breakdown's visitors difference is up to almost 43%
- assert summary(field(exported_pages, "visit_duration")) == [0, 0.0, 13.0, 46.5, 264]
- assert summary(field(imported_pages, "visit_duration")) == [0.0, 0.0, 13.3, 46.6, 264.5]
-
- assert summary(
- pairwise(exported_pages, imported_pages, fn exported, imported ->
- e = exported["visit_duration"]
- i = imported["visit_duration"]
-
- if is_number(e) and is_number(i) and i > 0 do
- abs(1 - e / i)
- else
- # both nil or both zero
- assert e == i
- _no_diff = 0
- end
- end)
- ) == [0.0, 0.0, 0.0, 0.008812019016100597, 0.4285714285714286]
-
# NOTE: page breakdown's visitors difference is up to 28%
assert summary(field(exported_pages, "visitors")) == [1, 1.0, 5.5, 17.75, 495]
assert summary(field(imported_pages, "visitors")) == [1, 1.75, 5.5, 20.0, 511]
@@ -694,8 +682,8 @@ defmodule Plausible.Imported.CSVImporterTest do
) == [0, 0.0, 0.0, 0.0, 0.06666666666666665]
# sources
- exported_sources = breakdown.(exported_site, "visit:source", [])
- imported_sources = breakdown.(imported_site, "visit:source", [])
+ exported_sources = breakdown.(exported_site, "visit:source")
+ imported_sources = breakdown.(imported_site, "visit:source")
pairwise(exported_sources, imported_sources, fn exported, imported ->
assert exported["source"] == imported["source"]
@@ -716,12 +704,12 @@ defmodule Plausible.Imported.CSVImporterTest do
) == [0.0, 0.0, 0.0, 0.0, 0.11111111111111116]
# utm mediums
- assert breakdown.(exported_site, "visit:utm_medium", []) ==
- breakdown.(imported_site, "visit:utm_medium", [])
+ assert breakdown.(exported_site, "visit:utm_medium") ==
+ breakdown.(imported_site, "visit:utm_medium")
# entry pages
- exported_entry_pages = breakdown.(exported_site, "visit:entry_page", [])
- imported_entry_pages = breakdown.(imported_site, "visit:entry_page", [])
+ exported_entry_pages = breakdown.(exported_site, "visit:entry_page")
+ imported_entry_pages = breakdown.(imported_site, "visit:entry_page")
pairwise(exported_entry_pages, imported_entry_pages, fn exported, imported ->
assert exported["entry_page"] == imported["entry_page"]
@@ -742,17 +730,53 @@ defmodule Plausible.Imported.CSVImporterTest do
) == [0, 0, 0, 0, 0.5]
# cities
- exported_cities = breakdown.(exported_site, "visit:city", [])
- imported_cities = breakdown.(imported_site, "visit:city", [])
+ exported_cities = breakdown.(exported_site, "visit:city")
+ imported_cities = breakdown.(imported_site, "visit:city")
pairwise(exported_cities, imported_cities, fn exported, imported ->
assert exported["city"] == imported["city"]
- assert_in_delta exported["bounce_rate"], imported["bounce_rate"], 4
assert exported["pageviews"] == imported["pageviews"]
- assert_in_delta exported["visit_duration"], imported["visit_duration"], 2
assert_in_delta exported["visits"], imported["visits"], 1
end)
+ # NOTE: city breakdown's bounce rate difference is up to 4%
+ assert summary(field(exported_cities, "bounce_rate")) == [0, 87.0, 100.0, 100.0, 100]
+ assert summary(field(imported_cities, "bounce_rate")) == [0.0, 87.0, 100.0, 100.0, 100.0]
+
+ assert summary(
+ pairwise(exported_cities, imported_cities, fn exported, imported ->
+ e = exported["bounce_rate"]
+ i = imported["bounce_rate"]
+
+ if is_number(e) and is_number(i) and i > 0 do
+ abs(1 - e / i)
+ else
+ # both nil or both zero
+ assert e == i
+ _no_diff = 0
+ end
+ end)
+ ) == [0.0, 0.0, 0.0, 0.0, 0.03614457831325302]
+
+ # NOTE: city breakdown's visit duration difference is up to 14%
+ assert summary(field(exported_cities, "visit_duration")) == [0, 0.0, 0.0, 1.0, 1718]
+ assert summary(field(imported_cities, "visit_duration")) == [0.0, 0.0, 0.0, 1.0, 1718.0]
+
+ assert summary(
+ pairwise(exported_cities, imported_cities, fn exported, imported ->
+ e = exported["visit_duration"]
+ i = imported["visit_duration"]
+
+ if is_number(e) and is_number(i) and i > 0 do
+ abs(1 - e / i)
+ else
+ # both nil or both zero
+ assert e == i
+ _no_diff = 0
+ end
+ end)
+ ) == [0, 0.0, 0.0, 0.0, 0.1428571428571429]
+
# NOTE: city breakdown's visitors relative difference is up to 27%
assert summary(field(exported_cities, "visitors")) == [1, 1.0, 1.0, 2.0, 22]
assert summary(field(imported_cities, "visitors")) == [1, 1.0, 1.0, 2.0, 30]
@@ -768,8 +792,8 @@ defmodule Plausible.Imported.CSVImporterTest do
) == [0, 0.0, 0.0, 0.0, 0.2666666666666667]
# devices
- exported_devices = breakdown.(exported_site, "visit:device", [])
- imported_devices = breakdown.(imported_site, "visit:device", [])
+ exported_devices = breakdown.(exported_site, "visit:device")
+ imported_devices = breakdown.(imported_site, "visit:device")
pairwise(exported_devices, imported_devices, fn exported, imported ->
assert exported["device"] == imported["device"]
@@ -790,8 +814,8 @@ defmodule Plausible.Imported.CSVImporterTest do
) == [0.0, 0.0, 0.011378002528445008, 0.025028787232157956, 0.031847133757961776]
# browsers
- exported_browsers = breakdown.(exported_site, "visit:browser", [])
- imported_browsers = breakdown.(imported_site, "visit:browser", [])
+ exported_browsers = breakdown.(exported_site, "visit:browser")
+ imported_browsers = breakdown.(imported_site, "visit:browser")
pairwise(exported_browsers, imported_browsers, fn exported, imported ->
assert exported["browser"] == imported["browser"]
@@ -812,8 +836,8 @@ defmodule Plausible.Imported.CSVImporterTest do
) == [0.0, 0.0, 0.0, 0.0, 0.037122969837587005]
# os
- exported_os = breakdown.(exported_site, "visit:os", [])
- imported_os = breakdown.(imported_site, "visit:os", [])
+ exported_os = breakdown.(exported_site, "visit:os")
+ imported_os = breakdown.(imported_site, "visit:os")
pairwise(exported_os, imported_os, fn exported, imported ->
assert exported["os"] == imported["os"]
@@ -834,8 +858,8 @@ defmodule Plausible.Imported.CSVImporterTest do
) == [0.0, 0.0, 0.0, 0.01754385964912286, 0.045592705167173286]
# os versions
- exported_os_versions = breakdown.(exported_site, "visit:os_version", [])
- imported_os_versions = breakdown.(imported_site, "visit:os_version", [])
+ exported_os_versions = breakdown.(exported_site, "visit:os_version")
+ imported_os_versions = breakdown.(imported_site, "visit:os_version")
pairwise(exported_os_versions, imported_os_versions, fn exported, imported ->
assert exported["os_version"] == imported["os_version"]
@@ -856,11 +880,8 @@ defmodule Plausible.Imported.CSVImporterTest do
) == [0.0, 0.0, 0.0, 0.0, 0.33333333333333337]
# goals
- exported_goals =
- breakdown.(exported_site, "event:goal", metrics: "visitors,events,conversion_rate")
-
- imported_goals =
- breakdown.(imported_site, "event:goal", metrics: "visitors,events,conversion_rate")
+ exported_goals = goal_breakdown.(exported_site, "event:goal", "")
+ imported_goals = goal_breakdown.(imported_site, "event:goal", "")
# NOTE: goal breakdown's visitors difference is up to 20%
assert summary(field(exported_goals, "visitors")) == [1, 6.5, 12.0, 41.0, 70]
@@ -907,16 +928,10 @@ defmodule Plausible.Imported.CSVImporterTest do
# url property breakdown
exported_url_props =
- breakdown.(exported_site, "event:props:url",
- filters: "event:goal==Outbound Link: Click",
- metrics: "visitors,events,conversion_rate"
- )
+ goal_breakdown.(exported_site, "event:props:url", "event:goal==Outbound Link: Click")
imported_url_props =
- breakdown.(imported_site, "event:props:url",
- filters: "event:goal==Outbound Link: Click",
- metrics: "visitors,events,conversion_rate"
- )
+ goal_breakdown.(imported_site, "event:props:url", "event:goal==Outbound Link: Click")
pairwise(exported_url_props, imported_url_props, fn exported, imported ->
assert exported["visitors"] == imported["visitors"]
@@ -934,17 +949,8 @@ defmodule Plausible.Imported.CSVImporterTest do
) == [0.0, 0.0, 0.0, 0.0, 0.19999999999999996]
# path property breakdown
- exported_path_props =
- breakdown.(exported_site, "event:props:path",
- filters: "event:goal==404",
- metrics: "visitors,events,conversion_rate"
- )
-
- imported_path_props =
- breakdown.(imported_site, "event:props:path",
- filters: "event:goal==404",
- metrics: "visitors,events,conversion_rate"
- )
+ exported_path_props = goal_breakdown.(exported_site, "event:props:path", "event:goal==404")
+ imported_path_props = goal_breakdown.(imported_site, "event:props:path", "event:goal==404")
pairwise(exported_path_props, imported_path_props, fn exported, imported ->
assert exported["visitors"] == imported["visitors"]
diff --git a/test/plausible/imported/google_analytics4_test.exs b/test/plausible/imported/google_analytics4_test.exs
index 7bde5f0673..7253877c4c 100644
--- a/test/plausible/imported/google_analytics4_test.exs
+++ b/test/plausible/imported/google_analytics4_test.exs
@@ -503,7 +503,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
"bounce_rate" => 58.0,
"pageviews" => 491,
"utm_medium" => "referral",
- "visit_duration" => 27.5,
+ "visit_duration" => 27.0,
"visitors" => 294,
"visits" => 298
}
@@ -521,7 +521,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.first(results) == %{
"bounce_rate" => 35.0,
"pageviews" => 838,
- "visit_duration" => 43.1,
+ "visit_duration" => 43.0,
"visitors" => 675,
"visits" => 712,
"entry_page" => "/brza-kukuruza"
@@ -549,7 +549,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
"bounce_rate" => 35.0,
"city" => 792_680,
"pageviews" => 1650,
- "visit_duration" => 38.9,
+ "visit_duration" => 39.0,
"visitors" => 1233,
"visits" => 1273
}
@@ -575,7 +575,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.first(results) == %{
"bounce_rate" => 38.0,
"pageviews" => 7041,
- "visit_duration" => 36.6,
+ "visit_duration" => 37.0,
"visitors" => 5277,
"visits" => 5532,
"device" => "Mobile"
@@ -584,7 +584,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.last(results) == %{
"bounce_rate" => 37.0,
"pageviews" => 143,
- "visit_duration" => 59.8,
+ "visit_duration" => 60.0,
"visitors" => 97,
"visits" => 100,
"device" => "Tablet"
@@ -602,7 +602,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.first(results) == %{
"bounce_rate" => 33.0,
"pageviews" => 8143,
- "visit_duration" => 50.2,
+ "visit_duration" => 50.0,
"visitors" => 4625,
"visits" => 4655,
"browser" => "Chrome"
@@ -629,7 +629,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.first(results) == %{
"bounce_rate" => 34.0,
"pageviews" => 5827,
- "visit_duration" => 40.6,
+ "visit_duration" => 41.0,
"visitors" => 4319,
"visits" => 4495,
"os" => "Android"
@@ -658,7 +658,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
"os" => "Android",
"os_version" => "13.0.0",
"pageviews" => 1673,
- "visit_duration" => 42.4,
+ "visit_duration" => 42.0,
"visitors" => 1247,
"visits" => 1295
}
diff --git a/test/plausible_web/controllers/api/stats_controller/imported_test.exs b/test/plausible_web/controllers/api/stats_controller/imported_test.exs
index d460733e72..42eafec869 100644
--- a/test/plausible_web/controllers/api/stats_controller/imported_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/imported_test.exs
@@ -514,7 +514,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"name" => "Sweden",
"visitors" => 3,
"bounce_rate" => 67.0,
- "visit_duration" => 33.3
+ "visit_duration" => 33.0
},
%{
"name" => "oat milk",
@@ -1158,7 +1158,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
visit_duration = Enum.find(top_stats, fn stat -> stat["name"] == "Visit duration" end)
- assert visit_duration["value"] == 3_479_033
+ assert visit_duration["value"] == 3_479_032
end
test "skips empty dates from import", %{conn: conn, site: site, import_id: import_id} do