Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

More improvements for CSV export (#1446)

Matt Colligan29e37c8342ad
diff --git a/assets/js/dashboard/stats/visitor-graph.js b/assets/js/dashboard/stats/visitor-graph.js
index f83a4a4219..23c882e0f3 100644
--- a/assets/js/dashboard/stats/visitor-graph.js
+++ b/assets/js/dashboard/stats/visitor-graph.js
@@ -304,12 +304,18 @@ class LineGraph extends React.Component {
return stats
}
+ pollExportReady() {
+ if (document.cookie.includes('exporting')) {
+ setTimeout(this.pollExportReady.bind(this), 1000);
+ } else {
+ this.setState({exported: false})
+ }
+ }
+
downloadSpinner() {
this.setState({exported: true});
- setTimeout(
- () => this.setState({exported: false}),
- 4000
- );
+ document.cookie = "exporting=";
+ setTimeout(this.pollExportReady.bind(this), 1000);
}
downloadLink() {
diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex
index 162fe8fccf..8085fdb6b9 100644
--- a/lib/plausible/stats/breakdown.ex
+++ b/lib/plausible/stats/breakdown.ex
@@ -230,7 +230,7 @@ defmodule Plausible.Stats.Breakdown do
time_query = "
SELECT
p,
- sum(td)/count(case when p2 != p then 1 end) as avgTime
+ round(sum(td)/count(case when p2 != p then 1 end)) as avgTime
FROM
(SELECT
p,
diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex
index 42beaf8af9..9d84535f75 100644
--- a/lib/plausible_web/controllers/stats_controller.ex
+++ b/lib/plausible_web/controllers/stats_controller.ex
@@ -93,6 +93,7 @@ defmodule PlausibleWeb.StatsController do
conn
|> put_resp_content_type("application/zip")
|> put_resp_header("content-disposition", "attachment; filename=\"#{filename}\"")
+ |> delete_resp_cookie("exporting")
|> send_resp(200, zip_content)
end
diff --git a/test/plausible_web/controllers/CSVs/30d/browsers.csv b/test/plausible_web/controllers/CSVs/30d/browsers.csv
new file mode 100644
index 0000000000..a8b183d707
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/30d/browsers.csv
@@ -0,0 +1,3 @@
+name,visitors
+ABrowserName,2
+,1
diff --git a/test/plausible_web/controllers/CSVs/30d/conversions.csv b/test/plausible_web/controllers/CSVs/30d/conversions.csv
new file mode 100644
index 0000000000..f61d3a75c9
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/30d/conversions.csv
@@ -0,0 +1,2 @@
+name,unique_conversions,total_conversions
+Visit /some-other-page,1,1
diff --git a/test/plausible_web/controllers/CSVs/30d/countries.csv b/test/plausible_web/controllers/CSVs/30d/countries.csv
new file mode 100644
index 0000000000..550bdd69ca
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/30d/countries.csv
@@ -0,0 +1,2 @@
+name,visitors
+Estonia,2
diff --git a/test/plausible_web/controllers/CSVs/30d/devices.csv b/test/plausible_web/controllers/CSVs/30d/devices.csv
new file mode 100644
index 0000000000..6e6be88bce
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/30d/devices.csv
@@ -0,0 +1,2 @@
+name,visitors
+,3
diff --git a/test/plausible_web/controllers/CSVs/30d/entry_pages.csv b/test/plausible_web/controllers/CSVs/30d/entry_pages.csv
new file mode 100644
index 0000000000..8953831ae4
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/30d/entry_pages.csv
@@ -0,0 +1,2 @@
+name,unique_entrances,total_entrances,visit_duration
+/,3,3,20
diff --git a/test/plausible_web/controllers/CSVs/30d/exit_pages.csv b/test/plausible_web/controllers/CSVs/30d/exit_pages.csv
new file mode 100644
index 0000000000..53df3c5210
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/30d/exit_pages.csv
@@ -0,0 +1,3 @@
+name,unique_exits,total_exits,exit_rate
+/,2,2,66.0
+/some-other-page,1,1,100.0
diff --git a/test/plausible_web/controllers/CSVs/30d/operating_systems.csv b/test/plausible_web/controllers/CSVs/30d/operating_systems.csv
new file mode 100644
index 0000000000..6e6be88bce
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/30d/operating_systems.csv
@@ -0,0 +1,2 @@
+name,visitors
+,3
diff --git a/test/plausible_web/controllers/CSVs/30d/pages.csv b/test/plausible_web/controllers/CSVs/30d/pages.csv
new file mode 100644
index 0000000000..89301a9461
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/30d/pages.csv
@@ -0,0 +1,3 @@
+name,visitors,bounce_rate,time_on_page
+/,3,67,
+/some-other-page,1,,60.0
diff --git a/test/plausible_web/controllers/CSVs/30d/sources.csv b/test/plausible_web/controllers/CSVs/30d/sources.csv
new file mode 100644
index 0000000000..3b22e30a01
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/30d/sources.csv
@@ -0,0 +1,2 @@
+name,visitors,bounce_rate,visit_duration
+Google,1,0,60
diff --git a/test/plausible_web/controllers/CSVs/30d/utm_campaigns.csv b/test/plausible_web/controllers/CSVs/30d/utm_campaigns.csv
new file mode 100644
index 0000000000..934651b6b9
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/30d/utm_campaigns.csv
@@ -0,0 +1,2 @@
+name,visitors,bounce_rate,visit_duration
+ads,1,100,0
diff --git a/test/plausible_web/controllers/CSVs/30d/utm_mediums.csv b/test/plausible_web/controllers/CSVs/30d/utm_mediums.csv
new file mode 100644
index 0000000000..773a58cf75
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/30d/utm_mediums.csv
@@ -0,0 +1 @@
+name,visitors,bounce_rate,visit_duration
diff --git a/test/plausible_web/controllers/CSVs/30d/utm_sources.csv b/test/plausible_web/controllers/CSVs/30d/utm_sources.csv
new file mode 100644
index 0000000000..773a58cf75
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/30d/utm_sources.csv
@@ -0,0 +1 @@
+name,visitors,bounce_rate,visit_duration
diff --git a/test/plausible_web/controllers/CSVs/30d/visitors.csv b/test/plausible_web/controllers/CSVs/30d/visitors.csv
new file mode 100644
index 0000000000..c376875623
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/30d/visitors.csv
@@ -0,0 +1,32 @@
+date,visitors,pageviews,bounce_rate,visit_duration
+2021-09-20,1,1,100,0
+2021-09-21,0,0,,
+2021-09-22,0,0,,
+2021-09-23,0,0,,
+2021-09-24,0,0,,
+2021-09-25,0,0,,
+2021-09-26,0,0,,
+2021-09-27,0,0,,
+2021-09-28,0,0,,
+2021-09-29,0,0,,
+2021-09-30,0,0,,
+2021-10-01,0,0,,
+2021-10-02,0,0,,
+2021-10-03,0,0,,
+2021-10-04,0,0,,
+2021-10-05,0,0,,
+2021-10-06,0,0,,
+2021-10-07,0,0,,
+2021-10-08,0,0,,
+2021-10-09,0,0,,
+2021-10-10,0,0,,
+2021-10-11,0,0,,
+2021-10-12,0,0,,
+2021-10-13,0,0,,
+2021-10-14,0,0,,
+2021-10-15,0,0,,
+2021-10-16,0,0,,
+2021-10-17,0,0,,
+2021-10-18,0,0,,
+2021-10-19,1,1,100,0
+2021-10-20,1,2,0,60
diff --git a/test/plausible_web/controllers/CSVs/6m/browsers.csv b/test/plausible_web/controllers/CSVs/6m/browsers.csv
new file mode 100644
index 0000000000..6d9c5d6fa5
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/6m/browsers.csv
@@ -0,0 +1,3 @@
+name,visitors
+ABrowserName,3
+,1
diff --git a/test/plausible_web/controllers/CSVs/6m/conversions.csv b/test/plausible_web/controllers/CSVs/6m/conversions.csv
new file mode 100644
index 0000000000..f61d3a75c9
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/6m/conversions.csv
@@ -0,0 +1,2 @@
+name,unique_conversions,total_conversions
+Visit /some-other-page,1,1
diff --git a/test/plausible_web/controllers/CSVs/6m/countries.csv b/test/plausible_web/controllers/CSVs/6m/countries.csv
new file mode 100644
index 0000000000..4daa0cc026
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/6m/countries.csv
@@ -0,0 +1,2 @@
+name,visitors
+Estonia,3
diff --git a/test/plausible_web/controllers/CSVs/6m/devices.csv b/test/plausible_web/controllers/CSVs/6m/devices.csv
new file mode 100644
index 0000000000..259a48cdd4
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/6m/devices.csv
@@ -0,0 +1,2 @@
+name,visitors
+,4
diff --git a/test/plausible_web/controllers/CSVs/6m/entry_pages.csv b/test/plausible_web/controllers/CSVs/6m/entry_pages.csv
new file mode 100644
index 0000000000..016868c186
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/6m/entry_pages.csv
@@ -0,0 +1,2 @@
+name,unique_entrances,total_entrances,visit_duration
+/,4,4,15
diff --git a/test/plausible_web/controllers/CSVs/6m/exit_pages.csv b/test/plausible_web/controllers/CSVs/6m/exit_pages.csv
new file mode 100644
index 0000000000..d31eb55649
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/6m/exit_pages.csv
@@ -0,0 +1,3 @@
+name,unique_exits,total_exits,exit_rate
+/,3,3,75.0
+/some-other-page,1,1,100.0
diff --git a/test/plausible_web/controllers/CSVs/6m/operating_systems.csv b/test/plausible_web/controllers/CSVs/6m/operating_systems.csv
new file mode 100644
index 0000000000..259a48cdd4
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/6m/operating_systems.csv
@@ -0,0 +1,2 @@
+name,visitors
+,4
diff --git a/test/plausible_web/controllers/CSVs/6m/pages.csv b/test/plausible_web/controllers/CSVs/6m/pages.csv
new file mode 100644
index 0000000000..dc8d9adb4f
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/6m/pages.csv
@@ -0,0 +1,3 @@
+name,visitors,bounce_rate,time_on_page
+/,4,75,
+/some-other-page,1,,60.0
diff --git a/test/plausible_web/controllers/CSVs/6m/sources.csv b/test/plausible_web/controllers/CSVs/6m/sources.csv
new file mode 100644
index 0000000000..635bc508ad
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/6m/sources.csv
@@ -0,0 +1,2 @@
+name,visitors,bounce_rate,visit_duration
+Google,2,50,30
diff --git a/test/plausible_web/controllers/CSVs/6m/utm_campaigns.csv b/test/plausible_web/controllers/CSVs/6m/utm_campaigns.csv
new file mode 100644
index 0000000000..13779d3f0d
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/6m/utm_campaigns.csv
@@ -0,0 +1,2 @@
+name,visitors,bounce_rate,visit_duration
+ads,2,100,0
diff --git a/test/plausible_web/controllers/CSVs/6m/utm_mediums.csv b/test/plausible_web/controllers/CSVs/6m/utm_mediums.csv
new file mode 100644
index 0000000000..773a58cf75
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/6m/utm_mediums.csv
@@ -0,0 +1 @@
+name,visitors,bounce_rate,visit_duration
diff --git a/test/plausible_web/controllers/CSVs/6m/utm_sources.csv b/test/plausible_web/controllers/CSVs/6m/utm_sources.csv
new file mode 100644
index 0000000000..773a58cf75
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/6m/utm_sources.csv
@@ -0,0 +1 @@
+name,visitors,bounce_rate,visit_duration
diff --git a/test/plausible_web/controllers/CSVs/6m/visitors.csv b/test/plausible_web/controllers/CSVs/6m/visitors.csv
new file mode 100644
index 0000000000..cec87760f4
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/6m/visitors.csv
@@ -0,0 +1,7 @@
+date,visitors,pageviews,bounce_rate,visit_duration
+2021-05-01,1,1,100,0
+2021-06-01,0,0,,
+2021-07-01,0,0,,
+2021-08-01,0,0,,
+2021-09-01,1,1,100,0
+2021-10-01,2,3,50,30
diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exs
index ff0d68e036..e81d2f4725 100644
--- a/test/plausible_web/controllers/stats_controller_test.exs
+++ b/test/plausible_web/controllers/stats_controller_test.exs
@@ -48,20 +48,9 @@ defmodule PlausibleWeb.StatsControllerTest do
setup [:create_user, :create_new_site, :log_in]
test "exports data in zipped csvs", %{conn: conn, site: site} do
- populate_stats(site, [
- build(:pageview,
- country_code: "EE",
- timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -1),
- referrer_source: "Google"
- ),
- build(:pageview,
- utm_campaign: "ads",
- timestamp: Timex.shift(~N[2021-10-20 12:00:00], days: -1)
- )
- ])
-
+ populate_exported_stats(site)
conn = get(conn, "/" <> site.domain <> "/export?date=2021-10-20")
- assert_default_zip(conn)
+ assert_zip(conn, "30d")
end
end
@@ -70,149 +59,79 @@ defmodule PlausibleWeb.StatsControllerTest do
site = insert(:site, domain: "new-site.com")
link = insert(:shared_link, site: site)
- populate_stats(site, [
- build(:pageview,
- country_code: "EE",
- timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -1),
- referrer_source: "Google"
- ),
- build(:pageview,
- utm_campaign: "ads",
- timestamp: Timex.shift(~N[2021-10-20 12:00:00], days: -1)
- )
- ])
-
+ populate_exported_stats(site)
conn = get(conn, "/" <> site.domain <> "/export?auth=#{link.slug}&date=2021-10-20")
- assert_default_zip(conn)
+ assert_zip(conn, "30d")
end
end
- defp assert_default_zip(conn) do
- assert conn.status == 200
-
- assert {"content-type", "application/zip; charset=utf-8"} =
- List.keyfind(conn.resp_headers, "content-type", 0)
-
- {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])
-
- assert_csv(
- zip,
- 'visitors.csv',
- "date,visitors,pageviews,bounce_rate,visit_duration\r\n2021-09-20,0,0,,\r\n2021-09-21,0,0,,\r\n2021-09-22,0,0,,\r\n2021-09-23,0,0,,\r\n2021-09-24,0,0,,\r\n2021-09-25,0,0,,\r\n2021-09-26,0,0,,\r\n2021-09-27,0,0,,\r\n2021-09-28,0,0,,\r\n2021-09-29,0,0,,\r\n2021-09-30,0,0,,\r\n2021-10-01,0,0,,\r\n2021-10-02,0,0,,\r\n2021-10-03,0,0,,\r\n2021-10-04,0,0,,\r\n2021-10-05,0,0,,\r\n2021-10-06,0,0,,\r\n2021-10-07,0,0,,\r\n2021-10-08,0,0,,\r\n2021-10-09,0,0,,\r\n2021-10-10,0,0,,\r\n2021-10-11,0,0,,\r\n2021-10-12,0,0,,\r\n2021-10-13,0,0,,\r\n2021-10-14,0,0,,\r\n2021-10-15,0,0,,\r\n2021-10-16,0,0,,\r\n2021-10-17,0,0,,\r\n2021-10-18,0,0,,\r\n2021-10-19,1,1,100,0\r\n2021-10-20,1,1,100,0\r\n"
- )
-
- assert_csv(
- zip,
- 'sources.csv',
- "name,visitors,bounce_rate,visit_duration\r\nGoogle,1,100,0\r\n"
- )
-
- assert_csv(zip, 'utm_mediums.csv', "name,visitors,bounce_rate,visit_duration\r\n")
- assert_csv(zip, 'utm_sources.csv', "name,visitors,bounce_rate,visit_duration\r\n")
-
- assert_csv(
- zip,
- 'utm_campaigns.csv',
- "name,visitors,bounce_rate,visit_duration\r\nads,1,100,0\r\n"
- )
-
- assert_csv(zip, 'pages.csv', "name,visitors,bounce_rate,time_on_page\r\n/,2,100,\r\n")
-
- assert_csv(
- zip,
- 'entry_pages.csv',
- "name,unique_entrances,total_entrances,visit_duration\r\n/,2,2,0\r\n"
- )
-
- assert_csv(
- zip,
- 'exit_pages.csv',
- "name,unique_exits,total_exits,exit_rate\r\n/,2,2,100.0\r\n"
- )
-
- assert_csv(zip, 'countries.csv', "name,visitors\r\nEstonia,1\r\n")
- assert_csv(zip, 'browsers.csv', "name,visitors\r\n,2\r\n")
- assert_csv(zip, 'operating_systems.csv', "name,visitors\r\n,2\r\n")
- assert_csv(zip, 'devices.csv', "name,visitors\r\n,2\r\n")
- assert_csv(zip, 'conversions.csv', "name,unique_conversions,total_conversions\r\n")
- end
-
describe "GET /:website/export - for past 6 months" do
setup [:create_user, :create_new_site, :log_in]
test "exports 6 months of data in zipped csvs", %{conn: conn, site: site} do
- populate_stats(site, [
- build(:pageview,
- timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -1)
- ),
- build(:pageview,
- timestamp: Timex.shift(~N[2021-10-20 12:00:00], months: -1),
- country_code: "EE",
- browser: "ABrowserName"
- ),
- build(:pageview,
- timestamp: Timex.shift(~N[2021-10-20 12:00:00], months: -5),
- utm_campaign: "ads",
- country_code: "EE",
- referrer_source: "Google",
- browser: "ABrowserName"
- )
- ])
-
+ populate_exported_stats(site)
conn = get(conn, "/" <> site.domain <> "/export?period=6mo&date=2021-10-20")
- assert conn.status == 200
-
- assert {"content-type", "application/zip; charset=utf-8"} =
- List.keyfind(conn.resp_headers, "content-type", 0)
+ assert_zip(conn, "6m")
+ end
+ end
- {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])
+ defp assert_zip(conn, folder) do
+ assert conn.status == 200
- assert_csv(
- zip,
- 'visitors.csv',
- "date,visitors,pageviews,bounce_rate,visit_duration\r\n2021-05-01,1,1,100,0\r\n2021-06-01,0,0,,\r\n2021-07-01,0,0,,\r\n2021-08-01,0,0,,\r\n2021-09-01,1,1,100,0\r\n2021-10-01,1,1,100,0\r\n"
- )
+ assert {"content-type", "application/zip; charset=utf-8"} =
+ List.keyfind(conn.resp_headers, "content-type", 0)
- assert_csv(
- zip,
- 'sources.csv',
- "name,visitors,bounce_rate,visit_duration\r\nGoogle,1,100,0\r\n"
- )
+ {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])
- assert_csv(zip, 'utm_mediums.csv', "name,visitors,bounce_rate,visit_duration\r\n")
- assert_csv(zip, 'utm_sources.csv', "name,visitors,bounce_rate,visit_duration\r\n")
+ folder = Path.expand(folder, "test/plausible_web/controllers/CSVs")
- assert_csv(
- zip,
- 'utm_campaigns.csv',
- "name,visitors,bounce_rate,visit_duration\r\nads,1,100,0\r\n"
- )
+ Enum.map(zip, &assert_csv(&1, folder))
+ end
- assert_csv(zip, 'pages.csv', "name,visitors,bounce_rate,time_on_page\r\n/,3,100,\r\n")
+ defp assert_csv({file, downloaded}, folder) do
+ file = Path.expand(file, folder)
- assert_csv(
- zip,
- 'entry_pages.csv',
- "name,unique_entrances,total_entrances,visit_duration\r\n/,3,3,0\r\n"
- )
+ {:ok, content} = File.read(file)
+ assert downloaded == content
+ end
- assert_csv(
- zip,
- 'exit_pages.csv',
- "name,unique_exits,total_exits,exit_rate\r\n/,3,3,100.0\r\n"
+ defp populate_exported_stats(site) do
+ populate_stats(site, [
+ build(:pageview,
+ country_code: "EE",
+ pathname: "/",
+ timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -1),
+ referrer_source: "Google",
+ user_id: 123
+ ),
+ build(:pageview,
+ country_code: "EE",
+ pathname: "/some-other-page",
+ timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -2),
+ referrer_source: "Google",
+ user_id: 123
+ ),
+ build(:pageview,
+ pathname: "/",
+ utm_campaign: "ads",
+ timestamp: Timex.shift(~N[2021-10-20 12:00:00], days: -1),
+ browser: "ABrowserName"
+ ),
+ build(:pageview,
+ timestamp: Timex.shift(~N[2021-10-20 12:00:00], months: -1),
+ country_code: "EE",
+ browser: "ABrowserName"
+ ),
+ build(:pageview,
+ timestamp: Timex.shift(~N[2021-10-20 12:00:00], months: -5),
+ utm_campaign: "ads",
+ country_code: "EE",
+ referrer_source: "Google",
+ browser: "ABrowserName"
)
+ ])
- assert_csv(zip, 'countries.csv', "name,visitors\r\nEstonia,2\r\n")
- assert_csv(zip, 'browsers.csv', "name,visitors\r\nABrowserName,2\r\n,1\r\n")
- assert_csv(zip, 'operating_systems.csv', "name,visitors\r\n,3\r\n")
- assert_csv(zip, 'devices.csv', "name,visitors\r\n,3\r\n")
- assert_csv(zip, 'conversions.csv', "name,unique_conversions,total_conversions\r\n")
- end
- end
-
- defp assert_csv(zip, fileName, string) do
- {_, contents} = List.keyfind(zip, fileName, 0)
- assert to_string(contents) == string
+ insert(:goal, %{domain: site.domain, page_path: "/some-other-page"})
end
describe "GET /share/:slug" do