Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Props into CSV export (#3167)

RobertJoonas51cad4c0ec4e
diff --git a/lib/plausible/site.ex b/lib/plausible/site.ex
index 55d2138088..4195947a84 100644
--- a/lib/plausible/site.ex
+++ b/lib/plausible/site.ex
@@ -166,10 +166,6 @@ defmodule Plausible.Site do
)
end
- def set_allowed_event_props(site, list) do
- change(site, allowed_event_props: list)
- end
-
@togglable_features ~w[conversions_enabled funnels_enabled props_enabled]a
def feature_toggle_change(site, property, opts \\ [])
when property in @togglable_features do
diff --git a/lib/plausible/site/admin.ex b/lib/plausible/site/admin.ex
index 19f6994942..e666053f48 100644
--- a/lib/plausible/site/admin.ex
+++ b/lib/plausible/site/admin.ex
@@ -85,8 +85,7 @@ defmodule Plausible.SiteAdmin do
props -> String.split(props, ~r/\s*,\s*/)
end
- Plausible.Site.set_allowed_event_props(site, props_list)
- |> Repo.update!()
+ Plausible.Sites.set_allowed_event_props!(site, props_list)
:ok
end
diff --git a/lib/plausible/sites.ex b/lib/plausible/sites.ex
index 3508e1aacd..7d0c375ea1 100644
--- a/lib/plausible/sites.ex
+++ b/lib/plausible/sites.ex
@@ -177,4 +177,9 @@ defmodule Plausible.Sites do
where: sm.role == :owner
)
end
+
+ def set_allowed_event_props!(site, props) do
+ Ecto.Changeset.change(site, allowed_event_props: props)
+ |> Repo.update!()
+ end
end
diff --git a/lib/plausible/stats/custom_props.ex b/lib/plausible/stats/custom_props.ex
index d37a0dffb5..73e7a402f7 100644
--- a/lib/plausible/stats/custom_props.ex
+++ b/lib/plausible/stats/custom_props.ex
@@ -34,7 +34,7 @@ defmodule Plausible.Stats.CustomProps do
end
end
- defp fetch_prop_names(site, query) do
+ def fetch_prop_names(site, query) do
case Query.get_filter_by_prefix(query, "event:props:") do
{"event:props:" <> key, _} ->
[key]
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index a40b90ba4a..7f03947057 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -1166,6 +1166,34 @@ defmodule PlausibleWeb.Api.StatsController do
def custom_prop_values(conn, params) do
site = conn.assigns[:site]
+ props = breakdown_custom_prop_values(site, params)
+ json(conn, props)
+ end
+
+ def all_custom_prop_values(conn, params) do
+ site = conn.assigns[:site]
+ query = Query.from(site, params) |> Filters.add_prefix()
+
+ prop_names = Plausible.Stats.CustomProps.fetch_prop_names(site, query)
+
+ values =
+ prop_names
+ |> Enum.map(fn prop_key ->
+ breakdown_custom_prop_values(site, Map.put(params, "prop_key", prop_key))
+ |> Enum.map(&Map.put(&1, :property, prop_key))
+ |> transform_keys(%{:name => :value})
+ end)
+ |> Enum.concat()
+
+ percent_or_cr =
+ if query.filters["event:goal"],
+ do: :conversion_rate,
+ else: :percentage
+
+ to_csv(values, [:property, :value, :visitors, :events, percent_or_cr])
+ end
+
+ defp breakdown_custom_prop_values(site, %{"prop_key" => prop_key} = params) do
query = Query.from(site, params) |> Filters.add_prefix()
pagination = parse_pagination(params)
@@ -1173,23 +1201,20 @@ defmodule PlausibleWeb.Api.StatsController do
%{:visitors => %{value: total_unique_visitors}} = Stats.aggregate(site, total_q, [:visitors])
- prefixed_prop = "event:props:" <> params["prop_key"]
+ prefixed_prop = "event:props:" <> prop_key
props =
Stats.breakdown(site, query, prefixed_prop, [:visitors, :events], pagination)
- |> transform_keys(%{params["prop_key"] => :name})
+ |> transform_keys(%{prop_key => :name})
|> add_percentages(query)
- props =
- if Map.has_key?(query.filters, "event:goal") do
- Enum.map(props, fn prop ->
- Map.put(prop, :conversion_rate, calculate_cr(total_unique_visitors, prop.visitors))
- end)
- else
- props
- end
-
- json(conn, props)
+ if Map.has_key?(query.filters, "event:goal") do
+ Enum.map(props, fn prop ->
+ Map.put(prop, :conversion_rate, calculate_cr(total_unique_visitors, prop.visitors))
+ end)
+ else
+ props
+ end
end
def prop_breakdown(conn, params) do
diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex
index ea4cf908d6..180ac0d3e7 100644
--- a/lib/plausible_web/controllers/stats_controller.ex
+++ b/lib/plausible_web/controllers/stats_controller.ex
@@ -148,6 +148,15 @@ defmodule PlausibleWeb.StatsController do
'prop_breakdown.csv' => fn -> Api.StatsController.all_props_breakdown(conn, params) end
}
+ csvs =
+ if FunWithFlags.enabled?(:props) do
+ Map.put(csvs, 'custom_props.csv', fn ->
+ Api.StatsController.all_custom_prop_values(conn, params)
+ end)
+ else
+ csvs
+ end
+
csv_values =
Map.values(csvs)
|> Plausible.ClickhouseRepo.parallel_tasks()
diff --git a/test/plausible_web/controllers/CSVs/30d-filter-goal/custom_props.csv b/test/plausible_web/controllers/CSVs/30d-filter-goal/custom_props.csv
new file mode 100644
index 0000000000..0a7292eeb9
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/30d-filter-goal/custom_props.csv
@@ -0,0 +1,2 @@
+property,value,visitors,events,conversion_rate
+variant,A,1,1,25.0
diff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/custom_props.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/custom_props.csv
new file mode 100644
index 0000000000..ccda364455
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/30d-filter-path/custom_props.csv
@@ -0,0 +1 @@
+property,value,visitors,events,percentage
diff --git a/test/plausible_web/controllers/CSVs/30d/custom_props.csv b/test/plausible_web/controllers/CSVs/30d/custom_props.csv
new file mode 100644
index 0000000000..22ccc27a40
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/30d/custom_props.csv
@@ -0,0 +1,3 @@
+property,value,visitors,events,percentage
+variant,(none),3,4,75.0
+variant,A,1,1,25.0
diff --git a/test/plausible_web/controllers/CSVs/6m/custom_props.csv b/test/plausible_web/controllers/CSVs/6m/custom_props.csv
new file mode 100644
index 0000000000..46a4e35d1e
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/6m/custom_props.csv
@@ -0,0 +1,3 @@
+property,value,visitors,events,percentage
+variant,(none),4,5,80.0
+variant,A,1,1,20.0
diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
index 82a9ab20ea..5bef446914 100644
--- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
@@ -455,8 +455,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
} do
site =
site
- |> Plausible.Site.set_allowed_event_props(["author"])
- |> Plausible.Repo.update!()
+ |> Plausible.Sites.set_allowed_event_props!(["author"])
populate_stats(site, [
build(:event,
@@ -491,8 +490,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
} do
site =
site
- |> Plausible.Site.set_allowed_event_props(["author", "logged_in"])
- |> Plausible.Repo.update!()
+ |> Plausible.Sites.set_allowed_event_props!(["author", "logged_in"])
populate_stats(site, [
build(:event,
diff --git a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
index 47630c46e9..7fdf2d05fe 100644
--- a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
@@ -284,8 +284,7 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
} do
site =
site
- |> Plausible.Site.set_allowed_event_props(["author"])
- |> Plausible.Repo.update!()
+ |> Plausible.Sites.set_allowed_event_props!(["author"])
populate_stats(site, [
build(:pageview,
diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exs
index aa014e00b9..82d47099fc 100644
--- a/test/plausible_web/controllers/stats_controller_test.exs
+++ b/test/plausible_web/controllers/stats_controller_test.exs
@@ -124,6 +124,37 @@ defmodule PlausibleWeb.StatsControllerTest do
describe "GET /:website/export" do
setup [:create_user, :create_new_site, :log_in]
+ test "exports all the necessary CSV files", %{conn: conn, site: site} do
+ conn = get(conn, "/" <> site.domain <> "/export")
+
+ assert {"content-type", "application/zip; charset=utf-8"} =
+ List.keyfind(conn.resp_headers, "content-type", 0)
+
+ {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])
+
+ zip = Enum.map(zip, fn {filename, _} -> filename end)
+
+ assert 'visitors.csv' in zip
+ assert 'browsers.csv' in zip
+ assert 'cities.csv' in zip
+ assert 'conversions.csv' in zip
+ assert 'countries.csv' in zip
+ assert 'custom_props.csv' in zip
+ assert 'devices.csv' in zip
+ assert 'entry_pages.csv' in zip
+ assert 'exit_pages.csv' in zip
+ assert 'operating_systems.csv' in zip
+ assert 'pages.csv' in zip
+ assert 'prop_breakdown.csv' in zip
+ assert 'regions.csv' in zip
+ assert 'sources.csv' in zip
+ assert 'utm_campaigns.csv' in zip
+ assert 'utm_contents.csv' in zip
+ assert 'utm_mediums.csv' in zip
+ assert 'utm_sources.csv' in zip
+ assert 'utm_terms.csv' in zip
+ end
+
test "exports data in zipped csvs", %{conn: conn, site: site} do
populate_exported_stats(site)
conn = get(conn, "/" <> site.domain <> "/export?date=2021-10-20")
@@ -139,6 +170,37 @@ defmodule PlausibleWeb.StatsControllerTest do
|> response(400)
end
+ test "exports allowed event props", %{conn: conn, site: site} do
+ site = Plausible.Sites.set_allowed_event_props!(site, ["author", "logged_in"])
+
+ populate_stats(site, [
+ build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),
+ build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),
+ build(:event, "meta.key": ["author"], "meta.value": ["marko"], name: "Newsletter Signup"),
+ build(:pageview, user_id: 999, "meta.key": ["logged_in"], "meta.value": ["true"]),
+ build(:pageview, user_id: 999, "meta.key": ["logged_in"], "meta.value": ["true"]),
+ build(:pageview, "meta.key": ["disallowed"], "meta.value": ["whatever"]),
+ build(:pageview)
+ ])
+
+ conn = get(conn, "/" <> site.domain <> "/export?period=day")
+ assert response = response(conn, 200)
+ {:ok, zip} = :zip.unzip(response, [:memory])
+
+ {_filename, result} =
+ Enum.find(zip, fn {filename, _data} -> filename == 'custom_props.csv' end)
+
+ assert parse_csv(result) == [
+ ["property", "value", "visitors", "events", "percentage"],
+ ["author", "(none)", "3", "4", "50.0"],
+ ["author", "uku", "2", "2", "33.3"],
+ ["author", "marko", "1", "1", "16.7"],
+ ["logged_in", "(none)", "5", "5", "83.3"],
+ ["logged_in", "true", "1", "2", "16.7"],
+ [""]
+ ]
+ end
+
test "exports data grouped by interval", %{conn: conn, site: site} do
populate_exported_stats(site)
conn = get(conn, "/" <> site.domain <> "/export?date=2021-10-20&period=30d&interval=week")
@@ -149,12 +211,7 @@ defmodule PlausibleWeb.StatsControllerTest do
{_filename, visitors} =
Enum.find(zip, fn {filename, _data} -> filename == 'visitors.csv' end)
- parsed_csv =
- visitors
- |> String.split("\r\n")
- |> Enum.map(&String.split(&1, ","))
-
- assert parsed_csv == [
+ assert parse_csv(visitors) == [
[
"date",
"visitors",
@@ -174,6 +231,12 @@ defmodule PlausibleWeb.StatsControllerTest do
end
end
+ defp parse_csv(file_content) when is_binary(file_content) do
+ file_content
+ |> String.split("\r\n")
+ |> Enum.map(&String.split(&1, ","))
+ end
+
describe "GET /:website/export - via shared link" do
test "exports data in zipped csvs", %{conn: conn} do
site = insert(:site, domain: "new-site.com")
@@ -207,6 +270,37 @@ defmodule PlausibleWeb.StatsControllerTest do
end
end
+ describe "GET /:website/export - with a custom prop filter" do
+ setup [:create_user, :create_new_site, :log_in]
+
+ test "custom-props.csv only returns the prop and its value in filter", %{
+ conn: conn,
+ site: site
+ } do
+ site = Plausible.Sites.set_allowed_event_props!(site, ["author", "logged_in"])
+
+ populate_stats(site, [
+ build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),
+ build(:pageview, "meta.key": ["author"], "meta.value": ["marko"]),
+ build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"])
+ ])
+
+ filters = Jason.encode!(%{props: %{author: "marko"}})
+ conn = get(conn, "/" <> site.domain <> "/export?period=day&filters=#{filters}")
+
+ {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])
+
+ {_filename, result} =
+ Enum.find(zip, fn {filename, _data} -> filename == 'custom_props.csv' end)
+
+ assert parse_csv(result) == [
+ ["property", "value", "visitors", "events", "percentage"],
+ ["author", "marko", "1", "1", "100.0"],
+ [""]
+ ]
+ end
+ end
+
defp assert_zip(conn, folder) do
assert conn.status == 200
@@ -296,6 +390,35 @@ defmodule PlausibleWeb.StatsControllerTest do
conn = get(conn, "/#{site.domain}/export?date=2021-10-20&filters=#{filters}")
assert_zip(conn, "30d-filter-goal")
end
+
+ test "custom-props.csv only returns the prop names for the goal in filter", %{
+ conn: conn,
+ site: site
+ } do
+ site = Plausible.Sites.set_allowed_event_props!(site, ["author", "logged_in"])
+
+ populate_stats(site, [
+ build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["uku"]),
+ build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["marko"]),
+ build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["marko"]),
+ build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"])
+ ])
+
+ filters = Jason.encode!(%{goal: "Newsletter Signup"})
+ conn = get(conn, "/" <> site.domain <> "/export?period=day&filters=#{filters}")
+
+ {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])
+
+ {_filename, result} =
+ Enum.find(zip, fn {filename, _data} -> filename == 'custom_props.csv' end)
+
+ assert parse_csv(result) == [
+ ["property", "value", "visitors", "events", "conversion_rate"],
+ ["author", "marko", "2", "2", "50.0"],
+ ["author", "uku", "1", "1", "25.0"],
+ [""]
+ ]
+ end
end
describe "GET /share/:domain?auth=:auth" do
diff --git a/test/test_helper.exs b/test/test_helper.exs
index eac352c8dd..72fb0e84eb 100644
--- a/test/test_helper.exs
+++ b/test/test_helper.exs
@@ -2,5 +2,6 @@
Mox.defmock(Plausible.HTTPClient.Mock, for: Plausible.HTTPClient.Interface)
Application.ensure_all_started(:double)
FunWithFlags.enable(:funnels)
+FunWithFlags.enable(:props)
Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual)
ExUnit.configure(exclude: [:slow])