Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
More improvements for CSV export (#1446)
Matt Colligan29e37c8342ad
diff --git a/assets/js/dashboard/stats/visitor-graph.js b/assets/js/dashboard/stats/visitor-graph.jsindex f83a4a4219..23c882e0f3 100644--- a/assets/js/dashboard/stats/visitor-graph.js+++ b/assets/js/dashboard/stats/visitor-graph.js@@ -304,12 +304,18 @@ class LineGraph extends React.Component {return stats}+ pollExportReady() {+ if (document.cookie.includes('exporting')) {+ setTimeout(this.pollExportReady.bind(this), 1000);+ } else {+ this.setState({exported: false})+ }+ }+downloadSpinner() {this.setState({exported: true});- setTimeout(- () => this.setState({exported: false}),- 4000- );+ document.cookie = "exporting=";+ setTimeout(this.pollExportReady.bind(this), 1000);}downloadLink() {diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex 162fe8fccf..8085fdb6b9 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -230,7 +230,7 @@ defmodule Plausible.Stats.Breakdown dotime_query = "SELECTp,- sum(td)/count(case when p2 != p then 1 end) as avgTime+ round(sum(td)/count(case when p2 != p then 1 end)) as avgTimeFROM(SELECTp,diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.exindex 42beaf8af9..9d84535f75 100644--- a/lib/plausible_web/controllers/stats_controller.ex+++ b/lib/plausible_web/controllers/stats_controller.ex@@ -93,6 +93,7 @@ defmodule PlausibleWeb.StatsController doconn|> put_resp_content_type("application/zip")|> put_resp_header("content-disposition", "attachment; filename=\"#{filename}\"")+ |> delete_resp_cookie("exporting")|> send_resp(200, zip_content)enddiff --git a/test/plausible_web/controllers/CSVs/30d/browsers.csv b/test/plausible_web/controllers/CSVs/30d/browsers.csvnew file mode 100644index 0000000000..a8b183d707--- /dev/null+++ b/test/plausible_web/controllers/CSVs/30d/browsers.csv@@ -0,0 +1,3 @@+name,visitors+ABrowserName,2+,1diff --git a/test/plausible_web/controllers/CSVs/30d/conversions.csv b/test/plausible_web/controllers/CSVs/30d/conversions.csvnew file mode 100644index 0000000000..f61d3a75c9--- /dev/null+++ b/test/plausible_web/controllers/CSVs/30d/conversions.csv@@ -0,0 +1,2 @@+name,unique_conversions,total_conversions+Visit /some-other-page,1,1diff --git a/test/plausible_web/controllers/CSVs/30d/countries.csv b/test/plausible_web/controllers/CSVs/30d/countries.csvnew file mode 100644index 0000000000..550bdd69ca--- /dev/null+++ b/test/plausible_web/controllers/CSVs/30d/countries.csv@@ -0,0 +1,2 @@+name,visitors+Estonia,2diff --git a/test/plausible_web/controllers/CSVs/30d/devices.csv b/test/plausible_web/controllers/CSVs/30d/devices.csvnew file mode 100644index 0000000000..6e6be88bce--- /dev/null+++ b/test/plausible_web/controllers/CSVs/30d/devices.csv@@ -0,0 +1,2 @@+name,visitors+,3diff --git a/test/plausible_web/controllers/CSVs/30d/entry_pages.csv b/test/plausible_web/controllers/CSVs/30d/entry_pages.csvnew file mode 100644index 0000000000..8953831ae4--- /dev/null+++ b/test/plausible_web/controllers/CSVs/30d/entry_pages.csv@@ -0,0 +1,2 @@+name,unique_entrances,total_entrances,visit_duration+/,3,3,20diff --git a/test/plausible_web/controllers/CSVs/30d/exit_pages.csv b/test/plausible_web/controllers/CSVs/30d/exit_pages.csvnew file mode 100644index 0000000000..53df3c5210--- /dev/null+++ b/test/plausible_web/controllers/CSVs/30d/exit_pages.csv@@ -0,0 +1,3 @@+name,unique_exits,total_exits,exit_rate+/,2,2,66.0+/some-other-page,1,1,100.0diff --git a/test/plausible_web/controllers/CSVs/30d/operating_systems.csv b/test/plausible_web/controllers/CSVs/30d/operating_systems.csvnew file mode 100644index 0000000000..6e6be88bce--- /dev/null+++ b/test/plausible_web/controllers/CSVs/30d/operating_systems.csv@@ -0,0 +1,2 @@+name,visitors+,3diff --git a/test/plausible_web/controllers/CSVs/30d/pages.csv b/test/plausible_web/controllers/CSVs/30d/pages.csvnew file mode 100644index 0000000000..89301a9461--- /dev/null+++ b/test/plausible_web/controllers/CSVs/30d/pages.csv@@ -0,0 +1,3 @@+name,visitors,bounce_rate,time_on_page+/,3,67,+/some-other-page,1,,60.0diff --git a/test/plausible_web/controllers/CSVs/30d/sources.csv b/test/plausible_web/controllers/CSVs/30d/sources.csvnew file mode 100644index 0000000000..3b22e30a01--- /dev/null+++ b/test/plausible_web/controllers/CSVs/30d/sources.csv@@ -0,0 +1,2 @@+name,visitors,bounce_rate,visit_duration+Google,1,0,60diff --git a/test/plausible_web/controllers/CSVs/30d/utm_campaigns.csv b/test/plausible_web/controllers/CSVs/30d/utm_campaigns.csvnew file mode 100644index 0000000000..934651b6b9--- /dev/null+++ b/test/plausible_web/controllers/CSVs/30d/utm_campaigns.csv@@ -0,0 +1,2 @@+name,visitors,bounce_rate,visit_duration+ads,1,100,0diff --git a/test/plausible_web/controllers/CSVs/30d/utm_mediums.csv b/test/plausible_web/controllers/CSVs/30d/utm_mediums.csvnew file mode 100644index 0000000000..773a58cf75--- /dev/null+++ b/test/plausible_web/controllers/CSVs/30d/utm_mediums.csv@@ -0,0 +1 @@+name,visitors,bounce_rate,visit_durationdiff --git a/test/plausible_web/controllers/CSVs/30d/utm_sources.csv b/test/plausible_web/controllers/CSVs/30d/utm_sources.csvnew file mode 100644index 0000000000..773a58cf75--- /dev/null+++ b/test/plausible_web/controllers/CSVs/30d/utm_sources.csv@@ -0,0 +1 @@+name,visitors,bounce_rate,visit_durationdiff --git a/test/plausible_web/controllers/CSVs/30d/visitors.csv b/test/plausible_web/controllers/CSVs/30d/visitors.csvnew file mode 100644index 0000000000..c376875623--- /dev/null+++ b/test/plausible_web/controllers/CSVs/30d/visitors.csv@@ -0,0 +1,32 @@+date,visitors,pageviews,bounce_rate,visit_duration+2021-09-20,1,1,100,0+2021-09-21,0,0,,+2021-09-22,0,0,,+2021-09-23,0,0,,+2021-09-24,0,0,,+2021-09-25,0,0,,+2021-09-26,0,0,,+2021-09-27,0,0,,+2021-09-28,0,0,,+2021-09-29,0,0,,+2021-09-30,0,0,,+2021-10-01,0,0,,+2021-10-02,0,0,,+2021-10-03,0,0,,+2021-10-04,0,0,,+2021-10-05,0,0,,+2021-10-06,0,0,,+2021-10-07,0,0,,+2021-10-08,0,0,,+2021-10-09,0,0,,+2021-10-10,0,0,,+2021-10-11,0,0,,+2021-10-12,0,0,,+2021-10-13,0,0,,+2021-10-14,0,0,,+2021-10-15,0,0,,+2021-10-16,0,0,,+2021-10-17,0,0,,+2021-10-18,0,0,,+2021-10-19,1,1,100,0+2021-10-20,1,2,0,60diff --git a/test/plausible_web/controllers/CSVs/6m/browsers.csv b/test/plausible_web/controllers/CSVs/6m/browsers.csvnew file mode 100644index 0000000000..6d9c5d6fa5--- /dev/null+++ b/test/plausible_web/controllers/CSVs/6m/browsers.csv@@ -0,0 +1,3 @@+name,visitors+ABrowserName,3+,1diff --git a/test/plausible_web/controllers/CSVs/6m/conversions.csv b/test/plausible_web/controllers/CSVs/6m/conversions.csvnew file mode 100644index 0000000000..f61d3a75c9--- /dev/null+++ b/test/plausible_web/controllers/CSVs/6m/conversions.csv@@ -0,0 +1,2 @@+name,unique_conversions,total_conversions+Visit /some-other-page,1,1diff --git a/test/plausible_web/controllers/CSVs/6m/countries.csv b/test/plausible_web/controllers/CSVs/6m/countries.csvnew file mode 100644index 0000000000..4daa0cc026--- /dev/null+++ b/test/plausible_web/controllers/CSVs/6m/countries.csv@@ -0,0 +1,2 @@+name,visitors+Estonia,3diff --git a/test/plausible_web/controllers/CSVs/6m/devices.csv b/test/plausible_web/controllers/CSVs/6m/devices.csvnew file mode 100644index 0000000000..259a48cdd4--- /dev/null+++ b/test/plausible_web/controllers/CSVs/6m/devices.csv@@ -0,0 +1,2 @@+name,visitors+,4diff --git a/test/plausible_web/controllers/CSVs/6m/entry_pages.csv b/test/plausible_web/controllers/CSVs/6m/entry_pages.csvnew file mode 100644index 0000000000..016868c186--- /dev/null+++ b/test/plausible_web/controllers/CSVs/6m/entry_pages.csv@@ -0,0 +1,2 @@+name,unique_entrances,total_entrances,visit_duration+/,4,4,15diff --git a/test/plausible_web/controllers/CSVs/6m/exit_pages.csv b/test/plausible_web/controllers/CSVs/6m/exit_pages.csvnew file mode 100644index 0000000000..d31eb55649--- /dev/null+++ b/test/plausible_web/controllers/CSVs/6m/exit_pages.csv@@ -0,0 +1,3 @@+name,unique_exits,total_exits,exit_rate+/,3,3,75.0+/some-other-page,1,1,100.0diff --git a/test/plausible_web/controllers/CSVs/6m/operating_systems.csv b/test/plausible_web/controllers/CSVs/6m/operating_systems.csvnew file mode 100644index 0000000000..259a48cdd4--- /dev/null+++ b/test/plausible_web/controllers/CSVs/6m/operating_systems.csv@@ -0,0 +1,2 @@+name,visitors+,4diff --git a/test/plausible_web/controllers/CSVs/6m/pages.csv b/test/plausible_web/controllers/CSVs/6m/pages.csvnew file mode 100644index 0000000000..dc8d9adb4f--- /dev/null+++ b/test/plausible_web/controllers/CSVs/6m/pages.csv@@ -0,0 +1,3 @@+name,visitors,bounce_rate,time_on_page+/,4,75,+/some-other-page,1,,60.0diff --git a/test/plausible_web/controllers/CSVs/6m/sources.csv b/test/plausible_web/controllers/CSVs/6m/sources.csvnew file mode 100644index 0000000000..635bc508ad--- /dev/null+++ b/test/plausible_web/controllers/CSVs/6m/sources.csv@@ -0,0 +1,2 @@+name,visitors,bounce_rate,visit_duration+Google,2,50,30diff --git a/test/plausible_web/controllers/CSVs/6m/utm_campaigns.csv b/test/plausible_web/controllers/CSVs/6m/utm_campaigns.csvnew file mode 100644index 0000000000..13779d3f0d--- /dev/null+++ b/test/plausible_web/controllers/CSVs/6m/utm_campaigns.csv@@ -0,0 +1,2 @@+name,visitors,bounce_rate,visit_duration+ads,2,100,0diff --git a/test/plausible_web/controllers/CSVs/6m/utm_mediums.csv b/test/plausible_web/controllers/CSVs/6m/utm_mediums.csvnew file mode 100644index 0000000000..773a58cf75--- /dev/null+++ b/test/plausible_web/controllers/CSVs/6m/utm_mediums.csv@@ -0,0 +1 @@+name,visitors,bounce_rate,visit_durationdiff --git a/test/plausible_web/controllers/CSVs/6m/utm_sources.csv b/test/plausible_web/controllers/CSVs/6m/utm_sources.csvnew file mode 100644index 0000000000..773a58cf75--- /dev/null+++ b/test/plausible_web/controllers/CSVs/6m/utm_sources.csv@@ -0,0 +1 @@+name,visitors,bounce_rate,visit_durationdiff --git a/test/plausible_web/controllers/CSVs/6m/visitors.csv b/test/plausible_web/controllers/CSVs/6m/visitors.csvnew file mode 100644index 0000000000..cec87760f4--- /dev/null+++ b/test/plausible_web/controllers/CSVs/6m/visitors.csv@@ -0,0 +1,7 @@+date,visitors,pageviews,bounce_rate,visit_duration+2021-05-01,1,1,100,0+2021-06-01,0,0,,+2021-07-01,0,0,,+2021-08-01,0,0,,+2021-09-01,1,1,100,0+2021-10-01,2,3,50,30diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exsindex ff0d68e036..e81d2f4725 100644--- a/test/plausible_web/controllers/stats_controller_test.exs+++ b/test/plausible_web/controllers/stats_controller_test.exs@@ -48,20 +48,9 @@ defmodule PlausibleWeb.StatsControllerTest dosetup [:create_user, :create_new_site, :log_in]test "exports data in zipped csvs", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview,- country_code: "EE",- timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -1),- referrer_source: "Google"- ),- build(:pageview,- utm_campaign: "ads",- timestamp: Timex.shift(~N[2021-10-20 12:00:00], days: -1)- )- ])-+ populate_exported_stats(site)conn = get(conn, "/" <> site.domain <> "/export?date=2021-10-20")- assert_default_zip(conn)+ assert_zip(conn, "30d")endend@@ -70,149 +59,79 @@ defmodule PlausibleWeb.StatsControllerTest dosite = insert(:site, domain: "new-site.com")link = insert(:shared_link, site: site)- populate_stats(site, [- build(:pageview,- country_code: "EE",- timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -1),- referrer_source: "Google"- ),- build(:pageview,- utm_campaign: "ads",- timestamp: Timex.shift(~N[2021-10-20 12:00:00], days: -1)- )- ])-+ populate_exported_stats(site)conn = get(conn, "/" <> site.domain <> "/export?auth=#{link.slug}&date=2021-10-20")- assert_default_zip(conn)+ assert_zip(conn, "30d")endend- defp assert_default_zip(conn) do- assert conn.status == 200-- assert {"content-type", "application/zip; charset=utf-8"} =- List.keyfind(conn.resp_headers, "content-type", 0)-- {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])-- assert_csv(- zip,- 'visitors.csv',- "date,visitors,pageviews,bounce_rate,visit_duration\r\n2021-09-20,0,0,,\r\n2021-09-21,0,0,,\r\n2021-09-22,0,0,,\r\n2021-09-23,0,0,,\r\n2021-09-24,0,0,,\r\n2021-09-25,0,0,,\r\n2021-09-26,0,0,,\r\n2021-09-27,0,0,,\r\n2021-09-28,0,0,,\r\n2021-09-29,0,0,,\r\n2021-09-30,0,0,,\r\n2021-10-01,0,0,,\r\n2021-10-02,0,0,,\r\n2021-10-03,0,0,,\r\n2021-10-04,0,0,,\r\n2021-10-05,0,0,,\r\n2021-10-06,0,0,,\r\n2021-10-07,0,0,,\r\n2021-10-08,0,0,,\r\n2021-10-09,0,0,,\r\n2021-10-10,0,0,,\r\n2021-10-11,0,0,,\r\n2021-10-12,0,0,,\r\n2021-10-13,0,0,,\r\n2021-10-14,0,0,,\r\n2021-10-15,0,0,,\r\n2021-10-16,0,0,,\r\n2021-10-17,0,0,,\r\n2021-10-18,0,0,,\r\n2021-10-19,1,1,100,0\r\n2021-10-20,1,1,100,0\r\n"- )-- assert_csv(- zip,- 'sources.csv',- "name,visitors,bounce_rate,visit_duration\r\nGoogle,1,100,0\r\n"- )-- assert_csv(zip, 'utm_mediums.csv', "name,visitors,bounce_rate,visit_duration\r\n")- assert_csv(zip, 'utm_sources.csv', "name,visitors,bounce_rate,visit_duration\r\n")-- assert_csv(- zip,- 'utm_campaigns.csv',- "name,visitors,bounce_rate,visit_duration\r\nads,1,100,0\r\n"- )-- assert_csv(zip, 'pages.csv', "name,visitors,bounce_rate,time_on_page\r\n/,2,100,\r\n")-- assert_csv(- zip,- 'entry_pages.csv',- "name,unique_entrances,total_entrances,visit_duration\r\n/,2,2,0\r\n"- )-- assert_csv(- zip,- 'exit_pages.csv',- "name,unique_exits,total_exits,exit_rate\r\n/,2,2,100.0\r\n"- )-- assert_csv(zip, 'countries.csv', "name,visitors\r\nEstonia,1\r\n")- assert_csv(zip, 'browsers.csv', "name,visitors\r\n,2\r\n")- assert_csv(zip, 'operating_systems.csv', "name,visitors\r\n,2\r\n")- assert_csv(zip, 'devices.csv', "name,visitors\r\n,2\r\n")- assert_csv(zip, 'conversions.csv', "name,unique_conversions,total_conversions\r\n")- end-describe "GET /:website/export - for past 6 months" dosetup [:create_user, :create_new_site, :log_in]test "exports 6 months of data in zipped csvs", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview,- timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -1)- ),- build(:pageview,- timestamp: Timex.shift(~N[2021-10-20 12:00:00], months: -1),- country_code: "EE",- browser: "ABrowserName"- ),- build(:pageview,- timestamp: Timex.shift(~N[2021-10-20 12:00:00], months: -5),- utm_campaign: "ads",- country_code: "EE",- referrer_source: "Google",- browser: "ABrowserName"- )- ])-+ populate_exported_stats(site)conn = get(conn, "/" <> site.domain <> "/export?period=6mo&date=2021-10-20")- assert conn.status == 200-- assert {"content-type", "application/zip; charset=utf-8"} =- List.keyfind(conn.resp_headers, "content-type", 0)+ assert_zip(conn, "6m")+ end+ end- {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])+ defp assert_zip(conn, folder) do+ assert conn.status == 200- assert_csv(- zip,- 'visitors.csv',- "date,visitors,pageviews,bounce_rate,visit_duration\r\n2021-05-01,1,1,100,0\r\n2021-06-01,0,0,,\r\n2021-07-01,0,0,,\r\n2021-08-01,0,0,,\r\n2021-09-01,1,1,100,0\r\n2021-10-01,1,1,100,0\r\n"- )+ assert {"content-type", "application/zip; charset=utf-8"} =+ List.keyfind(conn.resp_headers, "content-type", 0)- assert_csv(- zip,- 'sources.csv',- "name,visitors,bounce_rate,visit_duration\r\nGoogle,1,100,0\r\n"- )+ {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])- assert_csv(zip, 'utm_mediums.csv', "name,visitors,bounce_rate,visit_duration\r\n")- assert_csv(zip, 'utm_sources.csv', "name,visitors,bounce_rate,visit_duration\r\n")+ folder = Path.expand(folder, "test/plausible_web/controllers/CSVs")- assert_csv(- zip,- 'utm_campaigns.csv',- "name,visitors,bounce_rate,visit_duration\r\nads,1,100,0\r\n"- )+ Enum.map(zip, &assert_csv(&1, folder))+ end- assert_csv(zip, 'pages.csv', "name,visitors,bounce_rate,time_on_page\r\n/,3,100,\r\n")+ defp assert_csv({file, downloaded}, folder) do+ file = Path.expand(file, folder)- assert_csv(- zip,- 'entry_pages.csv',- "name,unique_entrances,total_entrances,visit_duration\r\n/,3,3,0\r\n"- )+ {:ok, content} = File.read(file)+ assert downloaded == content+ end- assert_csv(- zip,- 'exit_pages.csv',- "name,unique_exits,total_exits,exit_rate\r\n/,3,3,100.0\r\n"+ defp populate_exported_stats(site) do+ populate_stats(site, [+ build(:pageview,+ country_code: "EE",+ pathname: "/",+ timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -1),+ referrer_source: "Google",+ user_id: 123+ ),+ build(:pageview,+ country_code: "EE",+ pathname: "/some-other-page",+ timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -2),+ referrer_source: "Google",+ user_id: 123+ ),+ build(:pageview,+ pathname: "/",+ utm_campaign: "ads",+ timestamp: Timex.shift(~N[2021-10-20 12:00:00], days: -1),+ browser: "ABrowserName"+ ),+ build(:pageview,+ timestamp: Timex.shift(~N[2021-10-20 12:00:00], months: -1),+ country_code: "EE",+ browser: "ABrowserName"+ ),+ build(:pageview,+ timestamp: Timex.shift(~N[2021-10-20 12:00:00], months: -5),+ utm_campaign: "ads",+ country_code: "EE",+ referrer_source: "Google",+ browser: "ABrowserName")+ ])- assert_csv(zip, 'countries.csv', "name,visitors\r\nEstonia,2\r\n")- assert_csv(zip, 'browsers.csv', "name,visitors\r\nABrowserName,2\r\n,1\r\n")- assert_csv(zip, 'operating_systems.csv', "name,visitors\r\n,3\r\n")- assert_csv(zip, 'devices.csv', "name,visitors\r\n,3\r\n")- assert_csv(zip, 'conversions.csv', "name,unique_conversions,total_conversions\r\n")- end- end-- defp assert_csv(zip, fileName, string) do- {_, contents} = List.keyfind(zip, fileName, 0)- assert to_string(contents) == string+ insert(:goal, %{domain: site.domain, page_path: "/some-other-page"})enddescribe "GET /share/:slug" do