Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Props into CSV export (#3167)
RobertJoonas51cad4c0ec4e
diff --git a/lib/plausible/site.ex b/lib/plausible/site.exindex 55d2138088..4195947a84 100644--- a/lib/plausible/site.ex+++ b/lib/plausible/site.ex@@ -166,10 +166,6 @@ defmodule Plausible.Site do)end- def set_allowed_event_props(site, list) do- change(site, allowed_event_props: list)- end-@togglable_features ~w[conversions_enabled funnels_enabled props_enabled]adef feature_toggle_change(site, property, opts \\ [])when property in @togglable_features dodiff --git a/lib/plausible/site/admin.ex b/lib/plausible/site/admin.exindex 19f6994942..e666053f48 100644--- a/lib/plausible/site/admin.ex+++ b/lib/plausible/site/admin.ex@@ -85,8 +85,7 @@ defmodule Plausible.SiteAdmin doprops -> String.split(props, ~r/\s*,\s*/)end- Plausible.Site.set_allowed_event_props(site, props_list)- |> Repo.update!()+ Plausible.Sites.set_allowed_event_props!(site, props_list):okenddiff --git a/lib/plausible/sites.ex b/lib/plausible/sites.exindex 3508e1aacd..7d0c375ea1 100644--- a/lib/plausible/sites.ex+++ b/lib/plausible/sites.ex@@ -177,4 +177,9 @@ defmodule Plausible.Sites dowhere: sm.role == :owner)end++ def set_allowed_event_props!(site, props) do+ Ecto.Changeset.change(site, allowed_event_props: props)+ |> Repo.update!()+ endenddiff --git a/lib/plausible/stats/custom_props.ex b/lib/plausible/stats/custom_props.exindex d37a0dffb5..73e7a402f7 100644--- a/lib/plausible/stats/custom_props.ex+++ b/lib/plausible/stats/custom_props.ex@@ -34,7 +34,7 @@ defmodule Plausible.Stats.CustomProps doendend- defp fetch_prop_names(site, query) do+ def fetch_prop_names(site, query) docase Query.get_filter_by_prefix(query, "event:props:") do{"event:props:" <> key, _} ->[key]diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex a40b90ba4a..7f03947057 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -1166,6 +1166,34 @@ defmodule PlausibleWeb.Api.StatsController dodef custom_prop_values(conn, params) dosite = conn.assigns[:site]+ props = breakdown_custom_prop_values(site, params)+ json(conn, props)+ end++ def all_custom_prop_values(conn, params) do+ site = conn.assigns[:site]+ query = Query.from(site, params) |> Filters.add_prefix()++ prop_names = Plausible.Stats.CustomProps.fetch_prop_names(site, query)++ values =+ prop_names+ |> Enum.map(fn prop_key ->+ breakdown_custom_prop_values(site, Map.put(params, "prop_key", prop_key))+ |> Enum.map(&Map.put(&1, :property, prop_key))+ |> transform_keys(%{:name => :value})+ end)+ |> Enum.concat()++ percent_or_cr =+ if query.filters["event:goal"],+ do: :conversion_rate,+ else: :percentage++ to_csv(values, [:property, :value, :visitors, :events, percent_or_cr])+ end++ defp breakdown_custom_prop_values(site, %{"prop_key" => prop_key} = params) doquery = Query.from(site, params) |> Filters.add_prefix()pagination = parse_pagination(params)@@ -1173,23 +1201,20 @@ defmodule PlausibleWeb.Api.StatsController do%{:visitors => %{value: total_unique_visitors}} = Stats.aggregate(site, total_q, [:visitors])- prefixed_prop = "event:props:" <> params["prop_key"]+ prefixed_prop = "event:props:" <> prop_keyprops =Stats.breakdown(site, query, prefixed_prop, [:visitors, :events], pagination)- |> transform_keys(%{params["prop_key"] => :name})+ |> transform_keys(%{prop_key => :name})|> add_percentages(query)- props =- if Map.has_key?(query.filters, "event:goal") do- Enum.map(props, fn prop ->- Map.put(prop, :conversion_rate, calculate_cr(total_unique_visitors, prop.visitors))- end)- else- props- end-- json(conn, props)+ if Map.has_key?(query.filters, "event:goal") do+ Enum.map(props, fn prop ->+ Map.put(prop, :conversion_rate, calculate_cr(total_unique_visitors, prop.visitors))+ end)+ else+ props+ endenddef prop_breakdown(conn, params) dodiff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.exindex ea4cf908d6..180ac0d3e7 100644--- a/lib/plausible_web/controllers/stats_controller.ex+++ b/lib/plausible_web/controllers/stats_controller.ex@@ -148,6 +148,15 @@ defmodule PlausibleWeb.StatsController do'prop_breakdown.csv' => fn -> Api.StatsController.all_props_breakdown(conn, params) end}+ csvs =+ if FunWithFlags.enabled?(:props) do+ Map.put(csvs, 'custom_props.csv', fn ->+ Api.StatsController.all_custom_prop_values(conn, params)+ end)+ else+ csvs+ end+csv_values =Map.values(csvs)|> Plausible.ClickhouseRepo.parallel_tasks()diff --git a/test/plausible_web/controllers/CSVs/30d-filter-goal/custom_props.csv b/test/plausible_web/controllers/CSVs/30d-filter-goal/custom_props.csvnew file mode 100644index 0000000000..0a7292eeb9--- /dev/null+++ b/test/plausible_web/controllers/CSVs/30d-filter-goal/custom_props.csv@@ -0,0 +1,2 @@+property,value,visitors,events,conversion_rate+variant,A,1,1,25.0diff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/custom_props.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/custom_props.csvnew file mode 100644index 0000000000..ccda364455--- /dev/null+++ b/test/plausible_web/controllers/CSVs/30d-filter-path/custom_props.csv@@ -0,0 +1 @@+property,value,visitors,events,percentagediff --git a/test/plausible_web/controllers/CSVs/30d/custom_props.csv b/test/plausible_web/controllers/CSVs/30d/custom_props.csvnew file mode 100644index 0000000000..22ccc27a40--- /dev/null+++ b/test/plausible_web/controllers/CSVs/30d/custom_props.csv@@ -0,0 +1,3 @@+property,value,visitors,events,percentage+variant,(none),3,4,75.0+variant,A,1,1,25.0diff --git a/test/plausible_web/controllers/CSVs/6m/custom_props.csv b/test/plausible_web/controllers/CSVs/6m/custom_props.csvnew file mode 100644index 0000000000..46a4e35d1e--- /dev/null+++ b/test/plausible_web/controllers/CSVs/6m/custom_props.csv@@ -0,0 +1,3 @@+property,value,visitors,events,percentage+variant,(none),4,5,80.0+variant,A,1,1,20.0diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exsindex 82a9ab20ea..5bef446914 100644--- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs@@ -455,8 +455,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do} dosite =site- |> Plausible.Site.set_allowed_event_props(["author"])- |> Plausible.Repo.update!()+ |> Plausible.Sites.set_allowed_event_props!(["author"])populate_stats(site, [build(:event,@@ -491,8 +490,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do} dosite =site- |> Plausible.Site.set_allowed_event_props(["author", "logged_in"])- |> Plausible.Repo.update!()+ |> Plausible.Sites.set_allowed_event_props!(["author", "logged_in"])populate_stats(site, [build(:event,diff --git a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exsindex 47630c46e9..7fdf2d05fe 100644--- a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs@@ -284,8 +284,7 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do} dosite =site- |> Plausible.Site.set_allowed_event_props(["author"])- |> Plausible.Repo.update!()+ |> Plausible.Sites.set_allowed_event_props!(["author"])populate_stats(site, [build(:pageview,diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exsindex aa014e00b9..82d47099fc 100644--- a/test/plausible_web/controllers/stats_controller_test.exs+++ b/test/plausible_web/controllers/stats_controller_test.exs@@ -124,6 +124,37 @@ defmodule PlausibleWeb.StatsControllerTest dodescribe "GET /:website/export" dosetup [:create_user, :create_new_site, :log_in]+ test "exports all the necessary CSV files", %{conn: conn, site: site} do+ conn = get(conn, "/" <> site.domain <> "/export")++ assert {"content-type", "application/zip; charset=utf-8"} =+ List.keyfind(conn.resp_headers, "content-type", 0)++ {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])++ zip = Enum.map(zip, fn {filename, _} -> filename end)++ assert 'visitors.csv' in zip+ assert 'browsers.csv' in zip+ assert 'cities.csv' in zip+ assert 'conversions.csv' in zip+ assert 'countries.csv' in zip+ assert 'custom_props.csv' in zip+ assert 'devices.csv' in zip+ assert 'entry_pages.csv' in zip+ assert 'exit_pages.csv' in zip+ assert 'operating_systems.csv' in zip+ assert 'pages.csv' in zip+ assert 'prop_breakdown.csv' in zip+ assert 'regions.csv' in zip+ assert 'sources.csv' in zip+ assert 'utm_campaigns.csv' in zip+ assert 'utm_contents.csv' in zip+ assert 'utm_mediums.csv' in zip+ assert 'utm_sources.csv' in zip+ assert 'utm_terms.csv' in zip+ end+test "exports data in zipped csvs", %{conn: conn, site: site} dopopulate_exported_stats(site)conn = get(conn, "/" <> site.domain <> "/export?date=2021-10-20")@@ -139,6 +170,37 @@ defmodule PlausibleWeb.StatsControllerTest do|> response(400)end+ test "exports allowed event props", %{conn: conn, site: site} do+ site = Plausible.Sites.set_allowed_event_props!(site, ["author", "logged_in"])++ populate_stats(site, [+ build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),+ build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),+ build(:event, "meta.key": ["author"], "meta.value": ["marko"], name: "Newsletter Signup"),+ build(:pageview, user_id: 999, "meta.key": ["logged_in"], "meta.value": ["true"]),+ build(:pageview, user_id: 999, "meta.key": ["logged_in"], "meta.value": ["true"]),+ build(:pageview, "meta.key": ["disallowed"], "meta.value": ["whatever"]),+ build(:pageview)+ ])++ conn = get(conn, "/" <> site.domain <> "/export?period=day")+ assert response = response(conn, 200)+ {:ok, zip} = :zip.unzip(response, [:memory])++ {_filename, result} =+ Enum.find(zip, fn {filename, _data} -> filename == 'custom_props.csv' end)++ assert parse_csv(result) == [+ ["property", "value", "visitors", "events", "percentage"],+ ["author", "(none)", "3", "4", "50.0"],+ ["author", "uku", "2", "2", "33.3"],+ ["author", "marko", "1", "1", "16.7"],+ ["logged_in", "(none)", "5", "5", "83.3"],+ ["logged_in", "true", "1", "2", "16.7"],+ [""]+ ]+ end+test "exports data grouped by interval", %{conn: conn, site: site} dopopulate_exported_stats(site)conn = get(conn, "/" <> site.domain <> "/export?date=2021-10-20&period=30d&interval=week")@@ -149,12 +211,7 @@ defmodule PlausibleWeb.StatsControllerTest do{_filename, visitors} =Enum.find(zip, fn {filename, _data} -> filename == 'visitors.csv' end)- parsed_csv =- visitors- |> String.split("\r\n")- |> Enum.map(&String.split(&1, ","))-- assert parsed_csv == [+ assert parse_csv(visitors) == [["date","visitors",@@ -174,6 +231,12 @@ defmodule PlausibleWeb.StatsControllerTest doendend+ defp parse_csv(file_content) when is_binary(file_content) do+ file_content+ |> String.split("\r\n")+ |> Enum.map(&String.split(&1, ","))+ end+describe "GET /:website/export - via shared link" dotest "exports data in zipped csvs", %{conn: conn} dosite = insert(:site, domain: "new-site.com")@@ -207,6 +270,37 @@ defmodule PlausibleWeb.StatsControllerTest doendend+ describe "GET /:website/export - with a custom prop filter" do+ setup [:create_user, :create_new_site, :log_in]++ test "custom-props.csv only returns the prop and its value in filter", %{+ conn: conn,+ site: site+ } do+ site = Plausible.Sites.set_allowed_event_props!(site, ["author", "logged_in"])++ populate_stats(site, [+ build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),+ build(:pageview, "meta.key": ["author"], "meta.value": ["marko"]),+ build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"])+ ])++ filters = Jason.encode!(%{props: %{author: "marko"}})+ conn = get(conn, "/" <> site.domain <> "/export?period=day&filters=#{filters}")++ {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])++ {_filename, result} =+ Enum.find(zip, fn {filename, _data} -> filename == 'custom_props.csv' end)++ assert parse_csv(result) == [+ ["property", "value", "visitors", "events", "percentage"],+ ["author", "marko", "1", "1", "100.0"],+ [""]+ ]+ end+ end+defp assert_zip(conn, folder) doassert conn.status == 200@@ -296,6 +390,35 @@ defmodule PlausibleWeb.StatsControllerTest doconn = get(conn, "/#{site.domain}/export?date=2021-10-20&filters=#{filters}")assert_zip(conn, "30d-filter-goal")end++ test "custom-props.csv only returns the prop names for the goal in filter", %{+ conn: conn,+ site: site+ } do+ site = Plausible.Sites.set_allowed_event_props!(site, ["author", "logged_in"])++ populate_stats(site, [+ build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["uku"]),+ build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["marko"]),+ build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["marko"]),+ build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"])+ ])++ filters = Jason.encode!(%{goal: "Newsletter Signup"})+ conn = get(conn, "/" <> site.domain <> "/export?period=day&filters=#{filters}")++ {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])++ {_filename, result} =+ Enum.find(zip, fn {filename, _data} -> filename == 'custom_props.csv' end)++ assert parse_csv(result) == [+ ["property", "value", "visitors", "events", "conversion_rate"],+ ["author", "marko", "2", "2", "50.0"],+ ["author", "uku", "1", "1", "25.0"],+ [""]+ ]+ endenddescribe "GET /share/:domain?auth=:auth" dodiff --git a/test/test_helper.exs b/test/test_helper.exsindex eac352c8dd..72fb0e84eb 100644--- a/test/test_helper.exs+++ b/test/test_helper.exs@@ -2,5 +2,6 @@Mox.defmock(Plausible.HTTPClient.Mock, for: Plausible.HTTPClient.Interface)Application.ensure_all_started(:double)FunWithFlags.enable(:funnels)+FunWithFlags.enable(:props)Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual)ExUnit.configure(exclude: [:slow])