Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Start putting together means to export all aggregate data (#1387)
Matt Colligan8138a6e3a564
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 025f55625c..6dc17286d7 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -19,6 +19,7 @@ All notable changes to this project will be documented in this file.- Add Conversion Rate to Top Sources, Top Pages Devices, Countries when filtered by a goal plausible/analytics#1299- Add list view for countries report in dashboard plausible/analytics#1381- Add ability to view more than 100 custom goal properties plausible/analytics#1353+- Data exported via the download button will contain CSV data for all visible graps in a zip file.### Fixed- Fix weekly report time range plausible/analytics#951diff --git a/assets/js/dashboard/stats/visitor-graph.js b/assets/js/dashboard/stats/visitor-graph.jsindex 6fe0ba508e..c359d78177 100644--- a/assets/js/dashboard/stats/visitor-graph.js+++ b/assets/js/dashboard/stats/visitor-graph.js@@ -303,7 +303,7 @@ class LineGraph extends React.Component {downloadLink() {if (this.props.query.period !== 'realtime') {- const endpoint = `/${encodeURIComponent(this.props.site.domain)}/visitors.csv${api.serializeQuery(this.props.query)}`+ const endpoint = `/${encodeURIComponent(this.props.site.domain)}/export${api.serializeQuery(this.props.query)}`return (<a href={endpoint} download>diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 409bb2c989..e9cd97baf2 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -216,7 +216,11 @@ defmodule PlausibleWeb.Api.StatsController do|> maybe_add_cr(site, query, pagination, "source", "visit:source")|> transform_keys(%{"source" => "name", "visitors" => "count"})- json(conn, res)+ if params["csv"] do+ res |> to_csv(["name", "count", "bounce_rate", "visit_duration"])+ else+ json(conn, res)+ endenddef utm_mediums(conn, params) do@@ -235,7 +239,11 @@ defmodule PlausibleWeb.Api.StatsController do|> maybe_add_cr(site, query, pagination, "utm_medium", "visit:utm_medium")|> transform_keys(%{"utm_medium" => "name", "visitors" => "count"})- json(conn, res)+ if params["csv"] do+ res |> to_csv(["name", "count", "bounce_rate", "visit_duration"])+ else+ json(conn, res)+ endenddef utm_campaigns(conn, params) do@@ -254,7 +262,11 @@ defmodule PlausibleWeb.Api.StatsController do|> maybe_add_cr(site, query, pagination, "utm_campaign", "visit:utm_campaign")|> transform_keys(%{"utm_campaign" => "name", "visitors" => "count"})- json(conn, res)+ if params["csv"] do+ res |> to_csv(["name", "count", "bounce_rate", "visit_duration"])+ else+ json(conn, res)+ endenddef utm_sources(conn, params) do@@ -273,7 +285,11 @@ defmodule PlausibleWeb.Api.StatsController do|> maybe_add_cr(site, query, pagination, "utm_source", "visit:utm_source")|> transform_keys(%{"utm_source" => "name", "visitors" => "count"})- json(conn, res)+ if params["csv"] do+ res |> to_csv(["name", "count", "bounce_rate", "visit_duration"])+ else+ json(conn, res)+ endenddef referrer_drilldown(conn, %{"referrer" => "Google"} = params) do@@ -344,7 +360,11 @@ defmodule PlausibleWeb.Api.StatsController do|> maybe_add_cr(site, query, pagination, "page", "event:page")|> transform_keys(%{"page" => "name", "visitors" => "count"})- json(conn, pages)+ if params["csv"] do+ pages |> to_csv(["name", "count", "bounce_rate", "time_on_page"])+ else+ json(conn, pages)+ endenddef entry_pages(conn, params) do@@ -358,7 +378,11 @@ defmodule PlausibleWeb.Api.StatsController do|> maybe_add_cr(site, query, pagination, "entry_page", "visit:entry_page")|> transform_keys(%{"entry_page" => "name", "visits" => "entries", "visitors" => "count"})- json(conn, entry_pages)+ if params["csv"] do+ entry_pages |> to_csv(["name", "count", "entries", "visit_duration"])+ else+ json(conn, entry_pages)+ endenddef exit_pages(conn, params) do@@ -398,7 +422,11 @@ defmodule PlausibleWeb.Api.StatsController doMap.put(exit_page, "exit_rate", exit_rate)end)- json(conn, exit_pages)+ if params["csv"] do+ exit_pages |> to_csv(["name", "count", "exits", "exit_rate"])+ else+ json(conn, exit_pages)+ endenddef countries(conn, params) do@@ -416,7 +444,11 @@ defmodule PlausibleWeb.Api.StatsController doend)|> add_percentages- json(conn, countries)+ if params["csv"] do+ countries |> to_csv(["name", "count"])+ else+ json(conn, countries)+ endenddef browsers(conn, params) do@@ -430,7 +462,11 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{"browser" => "name", "visitors" => "count"})|> add_percentages- json(conn, browsers)+ if params["csv"] do+ browsers |> to_csv(["name", "count"])+ else+ json(conn, browsers)+ endenddef browser_versions(conn, params) do@@ -458,7 +494,11 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{"os" => "name", "visitors" => "count"})|> add_percentages- json(conn, systems)+ if params["csv"] do+ systems |> to_csv(["name", "count"])+ else+ json(conn, systems)+ endenddef operating_system_versions(conn, params) do@@ -486,7 +526,11 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{"device" => "name", "visitors" => "count"})|> add_percentages- json(conn, sizes)+ if params["csv"] do+ sizes |> to_csv(["name", "count"])+ else+ json(conn, sizes)+ endenddefp calculate_cr(unique_visitors, converted_visitors) do@@ -526,7 +570,11 @@ defmodule PlausibleWeb.Api.StatsController do|> Map.put(:conversion_rate, calculate_cr(total_visitors, goal["count"]))end)- json(conn, conversions)+ if params["csv"] do+ conversions |> to_csv(["name", "count", "total_count"])+ else+ json(conn, conversions)+ endenddef prop_breakdown(conn, params) do@@ -634,4 +682,12 @@ defmodule PlausibleWeb.Api.StatsController dolistendend++ defp to_csv(list, headers) do+ list+ |> Enum.map(fn row -> Enum.map(headers, &row[&1]) end)+ |> (fn res -> [headers | res] end).()+ |> CSV.encode()+ |> Enum.join()+ endenddiff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.exindex 6a97d10c3b..bd4c02a691 100644--- a/lib/plausible_web/controllers/stats_controller.ex+++ b/lib/plausible_web/controllers/stats_controller.ex@@ -1,6 +1,7 @@defmodule PlausibleWeb.StatsController douse PlausibleWeb, :controlleruse Plausible.Repo+ alias PlausibleWeb.Apialias Plausible.Stats.Queryplug PlausibleWeb.AuthorizeSiteAccess when action in [:stats, :csv_export]@@ -37,9 +38,9 @@ defmodule PlausibleWeb.StatsController doendend- def csv_export(conn, %{"domain" => domain}) do+ def csv_export(conn, params) dosite = conn.assigns[:site]- query = Query.from(site.timezone, conn.params)+ query = Query.from(site.timezone, params)metrics =if query.filters["event:name"] do@@ -52,20 +53,40 @@ defmodule PlausibleWeb.StatsController doheaders = ["date" | metrics]- csv_content =+ visitors =Enum.map(graph, fn row -> Enum.map(headers, &row[&1]) end)|> (fn data -> [headers | data] end).()|> CSV.encode()- |> Enum.into([])|> Enum.join()filename =- "Plausible export #{domain} #{Timex.format!(query.date_range.first, "{ISOdate} ")} to #{Timex.format!(query.date_range.last, "{ISOdate} ")}.csv"+ "Plausible export #{params["domain"]} #{Timex.format!(query.date_range.first, "{ISOdate} ")} to #{Timex.format!(query.date_range.last, "{ISOdate} ")}.zip"++ params = Map.put(params, "limit", "1000")+ params = Map.put(params, "csv", "True")++ csvs = [+ {'visitors.csv', visitors},+ {'sources.csv', Api.StatsController.sources(conn, params)},+ {'utm_mediums.csv', Api.StatsController.utm_mediums(conn, params)},+ {'utm_sources.csv', Api.StatsController.utm_sources(conn, params)},+ {'utm_campaigns.csv', Api.StatsController.utm_campaigns(conn, params)},+ {'pages.csv', Api.StatsController.pages(conn, params)},+ {'entry_pages.csv', Api.StatsController.entry_pages(conn, params)},+ {'exit_pages.csv', Api.StatsController.exit_pages(conn, params)},+ {'countries.csv', Api.StatsController.countries(conn, params)},+ {'browsers.csv', Api.StatsController.browsers(conn, params)},+ {'operating_systems.csv', Api.StatsController.operating_systems(conn, params)},+ {'devices.csv', Api.StatsController.screen_sizes(conn, params)},+ {'conversions.csv', Api.StatsController.conversions(conn, params)}+ ]++ {:ok, {_, zip_content}} = :zip.create(filename, csvs, [:memory])conn- |> put_resp_content_type("text/csv")+ |> put_resp_content_type("application/zip")|> put_resp_header("content-disposition", "attachment; filename=\"#{filename}\"")- |> send_resp(200, csv_content)+ |> send_resp(200, zip_content)enddef shared_link(conn, %{"slug" => domain, "auth" => auth}) dodiff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.exindex 0b95355343..d96752e14f 100644--- a/lib/plausible_web/router.ex+++ b/lib/plausible_web/router.ex@@ -234,7 +234,7 @@ defmodule PlausibleWeb.Router dodelete "/:website", SiteController, :delete_sitedelete "/:website/stats", SiteController, :reset_stats- get "/:domain/visitors.csv", StatsController, :csv_export+ get "/:domain/export", StatsController, :csv_exportget "/:domain/*path", StatsController, :statsendenddiff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exsindex 224e006241..67aca70012 100644--- a/test/plausible_web/controllers/stats_controller_test.exs+++ b/test/plausible_web/controllers/stats_controller_test.exs@@ -44,31 +44,171 @@ defmodule PlausibleWeb.StatsControllerTest doendend- describe "GET /:website/visitors.csv" do- setup [:create_user, :log_in, :create_site]+ describe "GET /:website/export" do+ setup [:create_user, :create_new_site, :log_in]++ test "exports data in zipped csvs", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ country_code: "EE",+ timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -1),+ referrer_source: "Google"+ ),+ build(:pageview,+ utm_campaign: "ads",+ timestamp: Timex.shift(~N[2021-10-20 12:00:00], days: -1)+ )+ ])++ conn = get(conn, "/" <> site.domain <> "/export?date=2021-10-20")+ assert conn.status == 200++ assert {"content-type", "application/zip; charset=utf-8"} =+ List.keyfind(conn.resp_headers, "content-type", 0)++ {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])++ assert_csv(+ zip,+ 'visitors.csv',+ "date,visitors,pageviews,bounce_rate,visit_duration\r\n2021-09-20,0,0,,\r\n2021-09-21,0,0,,\r\n2021-09-22,0,0,,\r\n2021-09-23,0,0,,\r\n2021-09-24,0,0,,\r\n2021-09-25,0,0,,\r\n2021-09-26,0,0,,\r\n2021-09-27,0,0,,\r\n2021-09-28,0,0,,\r\n2021-09-29,0,0,,\r\n2021-09-30,0,0,,\r\n2021-10-01,0,0,,\r\n2021-10-02,0,0,,\r\n2021-10-03,0,0,,\r\n2021-10-04,0,0,,\r\n2021-10-05,0,0,,\r\n2021-10-06,0,0,,\r\n2021-10-07,0,0,,\r\n2021-10-08,0,0,,\r\n2021-10-09,0,0,,\r\n2021-10-10,0,0,,\r\n2021-10-11,0,0,,\r\n2021-10-12,0,0,,\r\n2021-10-13,0,0,,\r\n2021-10-14,0,0,,\r\n2021-10-15,0,0,,\r\n2021-10-16,0,0,,\r\n2021-10-17,0,0,,\r\n2021-10-18,0,0,,\r\n2021-10-19,1,1,100,0\r\n2021-10-20,1,1,100,0\r\n"+ )++ assert_csv(zip, 'sources.csv', "name,count,bounce_rate,visit_duration\r\nGoogle,1,,\r\n")+ assert_csv(zip, 'utm_mediums.csv', "name,count,bounce_rate,visit_duration\r\n")+ assert_csv(zip, 'utm_sources.csv', "name,count,bounce_rate,visit_duration\r\n")- test "exports graph as csv", %{conn: conn, site: site} do- today = Timex.today() |> Timex.format!("{ISOdate}")+ assert_csv(+ zip,+ 'utm_campaigns.csv',+ "name,count,bounce_rate,visit_duration\r\nads,1,100,0\r\n"+ )- conn = get(conn, "/" <> site.domain <> "/visitors.csv")- assert response(conn, 200) =~ "visitors,pageviews,bounce_rate,visit_duration"- assert response(conn, 200) =~ "#{today},3,3,0,0"+ assert_csv(zip, 'pages.csv', "name,count,bounce_rate,time_on_page\r\n/,2,,\r\n")+ assert_csv(zip, 'entry_pages.csv', "name,count,entries,visit_duration\r\n/,2,2,0\r\n")+ assert_csv(zip, 'exit_pages.csv', "name,count,exits,exit_rate\r\n/,2,2,100.0\r\n")+ assert_csv(zip, 'countries.csv', "name,count\r\nEST,1\r\n")+ assert_csv(zip, 'browsers.csv', "name,count\r\n,2\r\n")+ assert_csv(zip, 'operating_systems.csv', "name,count\r\n,2\r\n")+ assert_csv(zip, 'devices.csv', "name,count\r\n,2\r\n")+ assert_csv(zip, 'conversions.csv', "name,count,total_count\r\n")endend- describe "GET /:website/visitors.csv - via shared link" do- test "exports graph as csv", %{conn: conn} do- site = insert(:site, domain: "test-site.com")+ describe "GET /:website/export - via shared link" do+ test "exports data in zipped csvs", %{conn: conn} do+ site = insert(:site, domain: "new-site.com")link = insert(:shared_link, site: site)- today = Timex.today() |> Timex.format!("{ISOdate}")+ populate_stats(site, [+ build(:pageview,+ country_code: "EE",+ timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -1),+ referrer_source: "Google"+ ),+ build(:pageview,+ utm_campaign: "ads",+ timestamp: Timex.shift(~N[2021-10-20 12:00:00], days: -1)+ )+ ])++ conn = get(conn, "/" <> site.domain <> "/export?auth=#{link.slug}&date=2021-10-20")+ assert conn.status == 200++ assert {"content-type", "application/zip; charset=utf-8"} =+ List.keyfind(conn.resp_headers, "content-type", 0)++ {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])++ assert_csv(+ zip,+ 'visitors.csv',+ "date,visitors,pageviews,bounce_rate,visit_duration\r\n2021-09-20,0,0,,\r\n2021-09-21,0,0,,\r\n2021-09-22,0,0,,\r\n2021-09-23,0,0,,\r\n2021-09-24,0,0,,\r\n2021-09-25,0,0,,\r\n2021-09-26,0,0,,\r\n2021-09-27,0,0,,\r\n2021-09-28,0,0,,\r\n2021-09-29,0,0,,\r\n2021-09-30,0,0,,\r\n2021-10-01,0,0,,\r\n2021-10-02,0,0,,\r\n2021-10-03,0,0,,\r\n2021-10-04,0,0,,\r\n2021-10-05,0,0,,\r\n2021-10-06,0,0,,\r\n2021-10-07,0,0,,\r\n2021-10-08,0,0,,\r\n2021-10-09,0,0,,\r\n2021-10-10,0,0,,\r\n2021-10-11,0,0,,\r\n2021-10-12,0,0,,\r\n2021-10-13,0,0,,\r\n2021-10-14,0,0,,\r\n2021-10-15,0,0,,\r\n2021-10-16,0,0,,\r\n2021-10-17,0,0,,\r\n2021-10-18,0,0,,\r\n2021-10-19,1,1,100,0\r\n2021-10-20,1,1,100,0\r\n"+ )++ assert_csv(zip, 'sources.csv', "name,count,bounce_rate,visit_duration\r\nGoogle,1,,\r\n")+ assert_csv(zip, 'utm_mediums.csv', "name,count,bounce_rate,visit_duration\r\n")+ assert_csv(zip, 'utm_sources.csv', "name,count,bounce_rate,visit_duration\r\n")++ assert_csv(+ zip,+ 'utm_campaigns.csv',+ "name,count,bounce_rate,visit_duration\r\nads,1,100,0\r\n"+ )- conn = get(conn, "/" <> site.domain <> "/visitors.csv?auth=#{link.slug}")- assert response(conn, 200) =~ "visitors,pageviews,bounce_rate,visit_duration"- assert response(conn, 200) =~ "#{today},3,3,0,0"+ assert_csv(zip, 'pages.csv', "name,count,bounce_rate,time_on_page\r\n/,2,,\r\n")+ assert_csv(zip, 'entry_pages.csv', "name,count,entries,visit_duration\r\n/,2,2,0\r\n")+ assert_csv(zip, 'exit_pages.csv', "name,count,exits,exit_rate\r\n/,2,2,100.0\r\n")+ assert_csv(zip, 'countries.csv', "name,count\r\nEST,1\r\n")+ assert_csv(zip, 'browsers.csv', "name,count\r\n,2\r\n")+ assert_csv(zip, 'operating_systems.csv', "name,count\r\n,2\r\n")+ assert_csv(zip, 'devices.csv', "name,count\r\n,2\r\n")+ assert_csv(zip, 'conversions.csv', "name,count,total_count\r\n")endend+ describe "GET /:website/export - for past 6 months" do+ setup [:create_user, :create_new_site, :log_in]++ test "exports 6 months of data in zipped csvs", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ timestamp: relative_time(minutes: -1)+ ),+ build(:pageview,+ timestamp: relative_time(months: -1),+ country_code: "EE",+ browser: "ABrowserName"+ ),+ build(:pageview,+ timestamp: relative_time(months: -5),+ utm_campaign: "ads",+ country_code: "EE",+ referrer_source: "Google",+ browser: "ABrowserName"+ )+ ])++ conn = get(conn, "/" <> site.domain <> "/export?period=6mo&date=2021-10-20")+ assert conn.status == 200++ assert {"content-type", "application/zip; charset=utf-8"} =+ List.keyfind(conn.resp_headers, "content-type", 0)++ {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])++ assert_csv(+ zip,+ 'visitors.csv',+ "date,visitors,pageviews,bounce_rate,visit_duration\r\n2021-05-01,1,1,100,0\r\n2021-06-01,0,0,,\r\n2021-07-01,0,0,,\r\n2021-08-01,0,0,,\r\n2021-09-01,1,1,100,0\r\n2021-10-01,1,1,100,0\r\n"+ )++ assert_csv(zip, 'sources.csv', "name,count,bounce_rate,visit_duration\r\nGoogle,1,,\r\n")+ assert_csv(zip, 'utm_mediums.csv', "name,count,bounce_rate,visit_duration\r\n")+ assert_csv(zip, 'utm_sources.csv', "name,count,bounce_rate,visit_duration\r\n")++ assert_csv(+ zip,+ 'utm_campaigns.csv',+ "name,count,bounce_rate,visit_duration\r\nads,1,100,0\r\n"+ )++ assert_csv(zip, 'pages.csv', "name,count,bounce_rate,time_on_page\r\n/,3,,\r\n")+ assert_csv(zip, 'entry_pages.csv', "name,count,entries,visit_duration\r\n/,3,3,0\r\n")+ assert_csv(zip, 'exit_pages.csv', "name,count,exits,exit_rate\r\n/,3,3,100.0\r\n")+ assert_csv(zip, 'countries.csv', "name,count\r\nEST,2\r\n")+ assert_csv(zip, 'browsers.csv', "name,count\r\nABrowserName,2\r\n,1\r\n")+ assert_csv(zip, 'operating_systems.csv', "name,count\r\n,3\r\n")+ assert_csv(zip, 'devices.csv', "name,count\r\n,3\r\n")+ assert_csv(zip, 'conversions.csv', "name,count,total_count\r\n")+ end+ end++ defp assert_csv(zip, fileName, string) do+ {_, contents} = List.keyfind(zip, fileName, 0)+ assert to_string(contents) == string+ end+describe "GET /share/:slug" dotest "prompts a password for a password-protected link", %{conn: conn} dosite = insert(:site)