Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Start putting together means to export all aggregate data (#1387)

Matt Colligan8138a6e3a564
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 025f55625c..6dc17286d7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,6 +19,7 @@ All notable changes to this project will be documented in this file.
- Add Conversion Rate to Top Sources, Top Pages Devices, Countries when filtered by a goal plausible/analytics#1299
- Add list view for countries report in dashboard plausible/analytics#1381
- Add ability to view more than 100 custom goal properties plausible/analytics#1353
+- Data exported via the download button will contain CSV data for all visible graps in a zip file.
### Fixed
- Fix weekly report time range plausible/analytics#951
diff --git a/assets/js/dashboard/stats/visitor-graph.js b/assets/js/dashboard/stats/visitor-graph.js
index 6fe0ba508e..c359d78177 100644
--- a/assets/js/dashboard/stats/visitor-graph.js
+++ b/assets/js/dashboard/stats/visitor-graph.js
@@ -303,7 +303,7 @@ class LineGraph extends React.Component {
downloadLink() {
if (this.props.query.period !== 'realtime') {
- const endpoint = `/${encodeURIComponent(this.props.site.domain)}/visitors.csv${api.serializeQuery(this.props.query)}`
+ const endpoint = `/${encodeURIComponent(this.props.site.domain)}/export${api.serializeQuery(this.props.query)}`
return (
<a href={endpoint} download>
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index 409bb2c989..e9cd97baf2 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -216,7 +216,11 @@ defmodule PlausibleWeb.Api.StatsController do
|> maybe_add_cr(site, query, pagination, "source", "visit:source")
|> transform_keys(%{"source" => "name", "visitors" => "count"})
- json(conn, res)
+ if params["csv"] do
+ res |> to_csv(["name", "count", "bounce_rate", "visit_duration"])
+ else
+ json(conn, res)
+ end
end
def utm_mediums(conn, params) do
@@ -235,7 +239,11 @@ defmodule PlausibleWeb.Api.StatsController do
|> maybe_add_cr(site, query, pagination, "utm_medium", "visit:utm_medium")
|> transform_keys(%{"utm_medium" => "name", "visitors" => "count"})
- json(conn, res)
+ if params["csv"] do
+ res |> to_csv(["name", "count", "bounce_rate", "visit_duration"])
+ else
+ json(conn, res)
+ end
end
def utm_campaigns(conn, params) do
@@ -254,7 +262,11 @@ defmodule PlausibleWeb.Api.StatsController do
|> maybe_add_cr(site, query, pagination, "utm_campaign", "visit:utm_campaign")
|> transform_keys(%{"utm_campaign" => "name", "visitors" => "count"})
- json(conn, res)
+ if params["csv"] do
+ res |> to_csv(["name", "count", "bounce_rate", "visit_duration"])
+ else
+ json(conn, res)
+ end
end
def utm_sources(conn, params) do
@@ -273,7 +285,11 @@ defmodule PlausibleWeb.Api.StatsController do
|> maybe_add_cr(site, query, pagination, "utm_source", "visit:utm_source")
|> transform_keys(%{"utm_source" => "name", "visitors" => "count"})
- json(conn, res)
+ if params["csv"] do
+ res |> to_csv(["name", "count", "bounce_rate", "visit_duration"])
+ else
+ json(conn, res)
+ end
end
def referrer_drilldown(conn, %{"referrer" => "Google"} = params) do
@@ -344,7 +360,11 @@ defmodule PlausibleWeb.Api.StatsController do
|> maybe_add_cr(site, query, pagination, "page", "event:page")
|> transform_keys(%{"page" => "name", "visitors" => "count"})
- json(conn, pages)
+ if params["csv"] do
+ pages |> to_csv(["name", "count", "bounce_rate", "time_on_page"])
+ else
+ json(conn, pages)
+ end
end
def entry_pages(conn, params) do
@@ -358,7 +378,11 @@ defmodule PlausibleWeb.Api.StatsController do
|> maybe_add_cr(site, query, pagination, "entry_page", "visit:entry_page")
|> transform_keys(%{"entry_page" => "name", "visits" => "entries", "visitors" => "count"})
- json(conn, entry_pages)
+ if params["csv"] do
+ entry_pages |> to_csv(["name", "count", "entries", "visit_duration"])
+ else
+ json(conn, entry_pages)
+ end
end
def exit_pages(conn, params) do
@@ -398,7 +422,11 @@ defmodule PlausibleWeb.Api.StatsController do
Map.put(exit_page, "exit_rate", exit_rate)
end)
- json(conn, exit_pages)
+ if params["csv"] do
+ exit_pages |> to_csv(["name", "count", "exits", "exit_rate"])
+ else
+ json(conn, exit_pages)
+ end
end
def countries(conn, params) do
@@ -416,7 +444,11 @@ defmodule PlausibleWeb.Api.StatsController do
end)
|> add_percentages
- json(conn, countries)
+ if params["csv"] do
+ countries |> to_csv(["name", "count"])
+ else
+ json(conn, countries)
+ end
end
def browsers(conn, params) do
@@ -430,7 +462,11 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{"browser" => "name", "visitors" => "count"})
|> add_percentages
- json(conn, browsers)
+ if params["csv"] do
+ browsers |> to_csv(["name", "count"])
+ else
+ json(conn, browsers)
+ end
end
def browser_versions(conn, params) do
@@ -458,7 +494,11 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{"os" => "name", "visitors" => "count"})
|> add_percentages
- json(conn, systems)
+ if params["csv"] do
+ systems |> to_csv(["name", "count"])
+ else
+ json(conn, systems)
+ end
end
def operating_system_versions(conn, params) do
@@ -486,7 +526,11 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{"device" => "name", "visitors" => "count"})
|> add_percentages
- json(conn, sizes)
+ if params["csv"] do
+ sizes |> to_csv(["name", "count"])
+ else
+ json(conn, sizes)
+ end
end
defp calculate_cr(unique_visitors, converted_visitors) do
@@ -526,7 +570,11 @@ defmodule PlausibleWeb.Api.StatsController do
|> Map.put(:conversion_rate, calculate_cr(total_visitors, goal["count"]))
end)
- json(conn, conversions)
+ if params["csv"] do
+ conversions |> to_csv(["name", "count", "total_count"])
+ else
+ json(conn, conversions)
+ end
end
def prop_breakdown(conn, params) do
@@ -634,4 +682,12 @@ defmodule PlausibleWeb.Api.StatsController do
list
end
end
+
+ defp to_csv(list, headers) do
+ list
+ |> Enum.map(fn row -> Enum.map(headers, &row[&1]) end)
+ |> (fn res -> [headers | res] end).()
+ |> CSV.encode()
+ |> Enum.join()
+ end
end
diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex
index 6a97d10c3b..bd4c02a691 100644
--- a/lib/plausible_web/controllers/stats_controller.ex
+++ b/lib/plausible_web/controllers/stats_controller.ex
@@ -1,6 +1,7 @@
defmodule PlausibleWeb.StatsController do
use PlausibleWeb, :controller
use Plausible.Repo
+ alias PlausibleWeb.Api
alias Plausible.Stats.Query
plug PlausibleWeb.AuthorizeSiteAccess when action in [:stats, :csv_export]
@@ -37,9 +38,9 @@ defmodule PlausibleWeb.StatsController do
end
end
- def csv_export(conn, %{"domain" => domain}) do
+ def csv_export(conn, params) do
site = conn.assigns[:site]
- query = Query.from(site.timezone, conn.params)
+ query = Query.from(site.timezone, params)
metrics =
if query.filters["event:name"] do
@@ -52,20 +53,40 @@ defmodule PlausibleWeb.StatsController do
headers = ["date" | metrics]
- csv_content =
+ visitors =
Enum.map(graph, fn row -> Enum.map(headers, &row[&1]) end)
|> (fn data -> [headers | data] end).()
|> CSV.encode()
- |> Enum.into([])
|> Enum.join()
filename =
- "Plausible export #{domain} #{Timex.format!(query.date_range.first, "{ISOdate} ")} to #{Timex.format!(query.date_range.last, "{ISOdate} ")}.csv"
+ "Plausible export #{params["domain"]} #{Timex.format!(query.date_range.first, "{ISOdate} ")} to #{Timex.format!(query.date_range.last, "{ISOdate} ")}.zip"
+
+ params = Map.put(params, "limit", "1000")
+ params = Map.put(params, "csv", "True")
+
+ csvs = [
+ {'visitors.csv', visitors},
+ {'sources.csv', Api.StatsController.sources(conn, params)},
+ {'utm_mediums.csv', Api.StatsController.utm_mediums(conn, params)},
+ {'utm_sources.csv', Api.StatsController.utm_sources(conn, params)},
+ {'utm_campaigns.csv', Api.StatsController.utm_campaigns(conn, params)},
+ {'pages.csv', Api.StatsController.pages(conn, params)},
+ {'entry_pages.csv', Api.StatsController.entry_pages(conn, params)},
+ {'exit_pages.csv', Api.StatsController.exit_pages(conn, params)},
+ {'countries.csv', Api.StatsController.countries(conn, params)},
+ {'browsers.csv', Api.StatsController.browsers(conn, params)},
+ {'operating_systems.csv', Api.StatsController.operating_systems(conn, params)},
+ {'devices.csv', Api.StatsController.screen_sizes(conn, params)},
+ {'conversions.csv', Api.StatsController.conversions(conn, params)}
+ ]
+
+ {:ok, {_, zip_content}} = :zip.create(filename, csvs, [:memory])
conn
- |> put_resp_content_type("text/csv")
+ |> put_resp_content_type("application/zip")
|> put_resp_header("content-disposition", "attachment; filename=\"#{filename}\"")
- |> send_resp(200, csv_content)
+ |> send_resp(200, zip_content)
end
def shared_link(conn, %{"slug" => domain, "auth" => auth}) do
diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex
index 0b95355343..d96752e14f 100644
--- a/lib/plausible_web/router.ex
+++ b/lib/plausible_web/router.ex
@@ -234,7 +234,7 @@ defmodule PlausibleWeb.Router do
delete "/:website", SiteController, :delete_site
delete "/:website/stats", SiteController, :reset_stats
- get "/:domain/visitors.csv", StatsController, :csv_export
+ get "/:domain/export", StatsController, :csv_export
get "/:domain/*path", StatsController, :stats
end
end
diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exs
index 224e006241..67aca70012 100644
--- a/test/plausible_web/controllers/stats_controller_test.exs
+++ b/test/plausible_web/controllers/stats_controller_test.exs
@@ -44,31 +44,171 @@ defmodule PlausibleWeb.StatsControllerTest do
end
end
- describe "GET /:website/visitors.csv" do
- setup [:create_user, :log_in, :create_site]
+ describe "GET /:website/export" do
+ setup [:create_user, :create_new_site, :log_in]
+
+ test "exports data in zipped csvs", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview,
+ country_code: "EE",
+ timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -1),
+ referrer_source: "Google"
+ ),
+ build(:pageview,
+ utm_campaign: "ads",
+ timestamp: Timex.shift(~N[2021-10-20 12:00:00], days: -1)
+ )
+ ])
+
+ conn = get(conn, "/" <> site.domain <> "/export?date=2021-10-20")
+ assert conn.status == 200
+
+ assert {"content-type", "application/zip; charset=utf-8"} =
+ List.keyfind(conn.resp_headers, "content-type", 0)
+
+ {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])
+
+ assert_csv(
+ zip,
+ 'visitors.csv',
+ "date,visitors,pageviews,bounce_rate,visit_duration\r\n2021-09-20,0,0,,\r\n2021-09-21,0,0,,\r\n2021-09-22,0,0,,\r\n2021-09-23,0,0,,\r\n2021-09-24,0,0,,\r\n2021-09-25,0,0,,\r\n2021-09-26,0,0,,\r\n2021-09-27,0,0,,\r\n2021-09-28,0,0,,\r\n2021-09-29,0,0,,\r\n2021-09-30,0,0,,\r\n2021-10-01,0,0,,\r\n2021-10-02,0,0,,\r\n2021-10-03,0,0,,\r\n2021-10-04,0,0,,\r\n2021-10-05,0,0,,\r\n2021-10-06,0,0,,\r\n2021-10-07,0,0,,\r\n2021-10-08,0,0,,\r\n2021-10-09,0,0,,\r\n2021-10-10,0,0,,\r\n2021-10-11,0,0,,\r\n2021-10-12,0,0,,\r\n2021-10-13,0,0,,\r\n2021-10-14,0,0,,\r\n2021-10-15,0,0,,\r\n2021-10-16,0,0,,\r\n2021-10-17,0,0,,\r\n2021-10-18,0,0,,\r\n2021-10-19,1,1,100,0\r\n2021-10-20,1,1,100,0\r\n"
+ )
+
+ assert_csv(zip, 'sources.csv', "name,count,bounce_rate,visit_duration\r\nGoogle,1,,\r\n")
+ assert_csv(zip, 'utm_mediums.csv', "name,count,bounce_rate,visit_duration\r\n")
+ assert_csv(zip, 'utm_sources.csv', "name,count,bounce_rate,visit_duration\r\n")
- test "exports graph as csv", %{conn: conn, site: site} do
- today = Timex.today() |> Timex.format!("{ISOdate}")
+ assert_csv(
+ zip,
+ 'utm_campaigns.csv',
+ "name,count,bounce_rate,visit_duration\r\nads,1,100,0\r\n"
+ )
- conn = get(conn, "/" <> site.domain <> "/visitors.csv")
- assert response(conn, 200) =~ "visitors,pageviews,bounce_rate,visit_duration"
- assert response(conn, 200) =~ "#{today},3,3,0,0"
+ assert_csv(zip, 'pages.csv', "name,count,bounce_rate,time_on_page\r\n/,2,,\r\n")
+ assert_csv(zip, 'entry_pages.csv', "name,count,entries,visit_duration\r\n/,2,2,0\r\n")
+ assert_csv(zip, 'exit_pages.csv', "name,count,exits,exit_rate\r\n/,2,2,100.0\r\n")
+ assert_csv(zip, 'countries.csv', "name,count\r\nEST,1\r\n")
+ assert_csv(zip, 'browsers.csv', "name,count\r\n,2\r\n")
+ assert_csv(zip, 'operating_systems.csv', "name,count\r\n,2\r\n")
+ assert_csv(zip, 'devices.csv', "name,count\r\n,2\r\n")
+ assert_csv(zip, 'conversions.csv', "name,count,total_count\r\n")
end
end
- describe "GET /:website/visitors.csv - via shared link" do
- test "exports graph as csv", %{conn: conn} do
- site = insert(:site, domain: "test-site.com")
+ describe "GET /:website/export - via shared link" do
+ test "exports data in zipped csvs", %{conn: conn} do
+ site = insert(:site, domain: "new-site.com")
link = insert(:shared_link, site: site)
- today = Timex.today() |> Timex.format!("{ISOdate}")
+ populate_stats(site, [
+ build(:pageview,
+ country_code: "EE",
+ timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -1),
+ referrer_source: "Google"
+ ),
+ build(:pageview,
+ utm_campaign: "ads",
+ timestamp: Timex.shift(~N[2021-10-20 12:00:00], days: -1)
+ )
+ ])
+
+ conn = get(conn, "/" <> site.domain <> "/export?auth=#{link.slug}&date=2021-10-20")
+ assert conn.status == 200
+
+ assert {"content-type", "application/zip; charset=utf-8"} =
+ List.keyfind(conn.resp_headers, "content-type", 0)
+
+ {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])
+
+ assert_csv(
+ zip,
+ 'visitors.csv',
+ "date,visitors,pageviews,bounce_rate,visit_duration\r\n2021-09-20,0,0,,\r\n2021-09-21,0,0,,\r\n2021-09-22,0,0,,\r\n2021-09-23,0,0,,\r\n2021-09-24,0,0,,\r\n2021-09-25,0,0,,\r\n2021-09-26,0,0,,\r\n2021-09-27,0,0,,\r\n2021-09-28,0,0,,\r\n2021-09-29,0,0,,\r\n2021-09-30,0,0,,\r\n2021-10-01,0,0,,\r\n2021-10-02,0,0,,\r\n2021-10-03,0,0,,\r\n2021-10-04,0,0,,\r\n2021-10-05,0,0,,\r\n2021-10-06,0,0,,\r\n2021-10-07,0,0,,\r\n2021-10-08,0,0,,\r\n2021-10-09,0,0,,\r\n2021-10-10,0,0,,\r\n2021-10-11,0,0,,\r\n2021-10-12,0,0,,\r\n2021-10-13,0,0,,\r\n2021-10-14,0,0,,\r\n2021-10-15,0,0,,\r\n2021-10-16,0,0,,\r\n2021-10-17,0,0,,\r\n2021-10-18,0,0,,\r\n2021-10-19,1,1,100,0\r\n2021-10-20,1,1,100,0\r\n"
+ )
+
+ assert_csv(zip, 'sources.csv', "name,count,bounce_rate,visit_duration\r\nGoogle,1,,\r\n")
+ assert_csv(zip, 'utm_mediums.csv', "name,count,bounce_rate,visit_duration\r\n")
+ assert_csv(zip, 'utm_sources.csv', "name,count,bounce_rate,visit_duration\r\n")
+
+ assert_csv(
+ zip,
+ 'utm_campaigns.csv',
+ "name,count,bounce_rate,visit_duration\r\nads,1,100,0\r\n"
+ )
- conn = get(conn, "/" <> site.domain <> "/visitors.csv?auth=#{link.slug}")
- assert response(conn, 200) =~ "visitors,pageviews,bounce_rate,visit_duration"
- assert response(conn, 200) =~ "#{today},3,3,0,0"
+ assert_csv(zip, 'pages.csv', "name,count,bounce_rate,time_on_page\r\n/,2,,\r\n")
+ assert_csv(zip, 'entry_pages.csv', "name,count,entries,visit_duration\r\n/,2,2,0\r\n")
+ assert_csv(zip, 'exit_pages.csv', "name,count,exits,exit_rate\r\n/,2,2,100.0\r\n")
+ assert_csv(zip, 'countries.csv', "name,count\r\nEST,1\r\n")
+ assert_csv(zip, 'browsers.csv', "name,count\r\n,2\r\n")
+ assert_csv(zip, 'operating_systems.csv', "name,count\r\n,2\r\n")
+ assert_csv(zip, 'devices.csv', "name,count\r\n,2\r\n")
+ assert_csv(zip, 'conversions.csv', "name,count,total_count\r\n")
end
end
+ describe "GET /:website/export - for past 6 months" do
+ setup [:create_user, :create_new_site, :log_in]
+
+ test "exports 6 months of data in zipped csvs", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview,
+ timestamp: relative_time(minutes: -1)
+ ),
+ build(:pageview,
+ timestamp: relative_time(months: -1),
+ country_code: "EE",
+ browser: "ABrowserName"
+ ),
+ build(:pageview,
+ timestamp: relative_time(months: -5),
+ utm_campaign: "ads",
+ country_code: "EE",
+ referrer_source: "Google",
+ browser: "ABrowserName"
+ )
+ ])
+
+ conn = get(conn, "/" <> site.domain <> "/export?period=6mo&date=2021-10-20")
+ assert conn.status == 200
+
+ assert {"content-type", "application/zip; charset=utf-8"} =
+ List.keyfind(conn.resp_headers, "content-type", 0)
+
+ {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])
+
+ assert_csv(
+ zip,
+ 'visitors.csv',
+ "date,visitors,pageviews,bounce_rate,visit_duration\r\n2021-05-01,1,1,100,0\r\n2021-06-01,0,0,,\r\n2021-07-01,0,0,,\r\n2021-08-01,0,0,,\r\n2021-09-01,1,1,100,0\r\n2021-10-01,1,1,100,0\r\n"
+ )
+
+ assert_csv(zip, 'sources.csv', "name,count,bounce_rate,visit_duration\r\nGoogle,1,,\r\n")
+ assert_csv(zip, 'utm_mediums.csv', "name,count,bounce_rate,visit_duration\r\n")
+ assert_csv(zip, 'utm_sources.csv', "name,count,bounce_rate,visit_duration\r\n")
+
+ assert_csv(
+ zip,
+ 'utm_campaigns.csv',
+ "name,count,bounce_rate,visit_duration\r\nads,1,100,0\r\n"
+ )
+
+ assert_csv(zip, 'pages.csv', "name,count,bounce_rate,time_on_page\r\n/,3,,\r\n")
+ assert_csv(zip, 'entry_pages.csv', "name,count,entries,visit_duration\r\n/,3,3,0\r\n")
+ assert_csv(zip, 'exit_pages.csv', "name,count,exits,exit_rate\r\n/,3,3,100.0\r\n")
+ assert_csv(zip, 'countries.csv', "name,count\r\nEST,2\r\n")
+ assert_csv(zip, 'browsers.csv', "name,count\r\nABrowserName,2\r\n,1\r\n")
+ assert_csv(zip, 'operating_systems.csv', "name,count\r\n,3\r\n")
+ assert_csv(zip, 'devices.csv', "name,count\r\n,3\r\n")
+ assert_csv(zip, 'conversions.csv', "name,count,total_count\r\n")
+ end
+ end
+
+ defp assert_csv(zip, fileName, string) do
+ {_, contents} = List.keyfind(zip, fileName, 0)
+ assert to_string(contents) == string
+ end
+
describe "GET /share/:slug" do
test "prompts a password for a password-protected link", %{conn: conn} do
site = insert(:site)