Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Dashboard csv export v2 continued (#6455)

RobertJoonase20b0c2c0a51
diff --git a/assets/js/dashboard/stats/csv-export/csv-export-body.ts b/assets/js/dashboard/stats/csv-export/csv-export-body.ts
index 860ee609e0..b03731702b 100644
--- a/assets/js/dashboard/stats/csv-export/csv-export-body.ts
+++ b/assets/js/dashboard/stats/csv-export/csv-export-body.ts
@@ -19,7 +19,8 @@ import { Metric } from '../metrics'
import {
BREAKDOWN_REPORTS,
BreakdownReportConfig,
- BreakdownReportKey
+ BreakdownReportKey,
+ getCustomPropsMetrics
} from '../reports/reports-config'
const BREAKDOWN_CSV_REPORTS = {
@@ -41,9 +42,13 @@ const BREAKDOWN_CSV_REPORTS = {
'utm_terms.csv': BreakdownReportKey.utmTerms,
'countries.csv': BreakdownReportKey.countries,
'regions.csv': BreakdownReportKey.regions,
- 'cities.csv': BreakdownReportKey.cities
+ 'cities.csv': BreakdownReportKey.cities,
+ 'conversions.csv': BreakdownReportKey.goals
}
-type CsvFilename = 'visitors.csv' | keyof typeof BREAKDOWN_CSV_REPORTS
+type CsvFilename =
+ | 'visitors.csv'
+ | 'custom_props.csv'
+ | keyof typeof BREAKDOWN_CSV_REPORTS
type CsvReportParams = {
dimensions: [TimeDimension] | BreakdownReportConfig['dimensions']
@@ -114,6 +119,13 @@ export function createCsvExportRequestBody(
? PAGE_FILTERED_VISITORS_CSV_METRICS
: DEFAULT_VISITORS_CSV_METRICS
},
+ 'custom_props.csv': {
+ dimensions: ['event:props:*'],
+ metrics: getCustomPropsMetrics({
+ hasConversionGoalFilter: isGoalFilter,
+ isRevenueAvailable: false
+ })
+ },
...Object.entries(BREAKDOWN_CSV_REPORTS).reduce(
(acc, [filename, reportKey]) => {
const config = BREAKDOWN_REPORTS[reportKey]
@@ -128,7 +140,7 @@ export function createCsvExportRequestBody(
}
return acc
},
- {} as Omit<CsvReportsConfig, 'visitors.csv'>
+ {} as Omit<CsvReportsConfig, 'visitors.csv' | 'custom_props.csv'>
)
}
diff --git a/assets/js/dashboard/stats/reports/reports-config.ts b/assets/js/dashboard/stats/reports/reports-config.ts
index 7ef2be6c27..8f0fbb823b 100644
--- a/assets/js/dashboard/stats/reports/reports-config.ts
+++ b/assets/js/dashboard/stats/reports/reports-config.ts
@@ -408,6 +408,9 @@ export const BREAKDOWN_REPORTS: Record<
[BreakdownReportKey.goals]: {
dimensions: ['event:goal'],
getMetrics: (ctx: MetricContext) => {
+ if (ctx.isCsv) {
+ return ['visitors', 'events']
+ }
if (ctx.isRevenueAvailable) {
return [
'visitors',
@@ -430,23 +433,25 @@ export function customPropsReportConfig(
): BreakdownReportConfig {
return {
dimensions: [`event:props:${propKey}` as NonTimeDimension],
- getMetrics: (ctx: MetricContext) => {
- if (ctx.hasConversionGoalFilter && ctx.isRevenueAvailable) {
- return [
- 'visitors',
- 'events',
- 'conversion_rate',
- 'total_revenue',
- 'average_revenue'
- ]
- }
- if (ctx.hasConversionGoalFilter) {
- return ['visitors', 'events', 'conversion_rate']
- }
- return ['visitors', 'events', 'percentage']
- },
+ getMetrics: getCustomPropsMetrics,
detailsTitle: 'Custom property breakdown',
detailsPath: `custom-prop-values/${propKey}`,
dimensionLabel: propKey
}
}
+
+export function getCustomPropsMetrics(ctx: MetricContext): Metric[] {
+ if (ctx.hasConversionGoalFilter && ctx.isRevenueAvailable) {
+ return [
+ 'visitors',
+ 'events',
+ 'conversion_rate',
+ 'total_revenue',
+ 'average_revenue'
+ ]
+ }
+ if (ctx.hasConversionGoalFilter) {
+ return ['visitors', 'events', 'conversion_rate']
+ }
+ return ['visitors', 'events', 'percentage']
+}
diff --git a/e2e/tests/dashboard/csv-export.spec.ts b/e2e/tests/dashboard/csv-export.spec.ts
index dc87e89344..a7d742acdc 100644
--- a/e2e/tests/dashboard/csv-export.spec.ts
+++ b/e2e/tests/dashboard/csv-export.spec.ts
@@ -57,7 +57,21 @@ const EXPECTED_HEADERS = {
commonGoalFilteredBreakdown: ['name', 'conversions', 'conversion_rate'],
nameAndVisitors: ['name', 'visitors'],
nameVersionAndVisitors: ['name', 'version', 'visitors'],
- nameVersionGoalFiltered: ['name', 'version', 'conversions', 'conversion_rate']
+ nameVersionGoalFiltered: [
+ 'name',
+ 'version',
+ 'conversions',
+ 'conversion_rate'
+ ],
+ customPropsDefault: ['property', 'value', 'visitors', 'events', 'percentage'],
+ customPropsGoalFiltered: [
+ 'property',
+ 'value',
+ 'visitors',
+ 'events',
+ 'conversion_rate'
+ ],
+ conversionsBreakdown: ['name', 'unique_conversions', 'total_conversions']
}
const UTM_AND_SOURCE_REPORTS = [
@@ -119,11 +133,14 @@ test('csv export column headers match expected metrics for each report', async (
await populateStats({
request,
domain,
- events: [{ name: 'pageview', timestamp: '2021-01-01 12:00:00' }]
+ events: [
+ { name: 'pageview', 'meta.key': ['author'], 'meta.value': ['john'] },
+ { name: 'Signup', 'meta.key': ['author'], 'meta.value': ['john'] }
+ ]
})
await test.step('without any filters', async () => {
- await page.goto(`/${domain}`, {
+ await page.goto(`/${domain}?period=all`, {
waitUntil: 'commit'
})
@@ -157,13 +174,23 @@ test('csv export column headers match expected metrics for each report', async (
EXPECTED_HEADERS.nameVersionAndVisitors
)
}
+ expect(getCsv(csvs, 'custom_props.csv')[0]).toEqual(
+ EXPECTED_HEADERS.customPropsDefault
+ )
+ expect(getCsv(csvs, 'conversions.csv')[0]).toEqual(
+ EXPECTED_HEADERS.conversionsBreakdown
+ )
})
await test.step('with a page filter', async () => {
- await page.goto(`/${domain}?f=is,page,/blog`, {
+ await page.goto(`/${domain}?period=all&f=is,page,/`, {
waitUntil: 'commit'
})
+ await expect(
+ page.getByRole('button', { name: 'Remove filter: Page is /' })
+ ).toBeVisible()
+
const csvs = parseAllCsvs(await triggerExportAndAwaitDownload(page))
expect(getCsv(csvs, 'visitors.csv')[0]).toEqual(
@@ -194,15 +221,25 @@ test('csv export column headers match expected metrics for each report', async (
EXPECTED_HEADERS.nameVersionAndVisitors
)
}
+ expect(getCsv(csvs, 'custom_props.csv')[0]).toEqual(
+ EXPECTED_HEADERS.customPropsDefault
+ )
+ expect(getCsv(csvs, 'conversions.csv')[0]).toEqual(
+ EXPECTED_HEADERS.conversionsBreakdown
+ )
})
await test.step('with a goal filter', async () => {
await addGoal({ request, domain, params: { event_name: 'Signup' } })
- await page.goto(`/${domain}?f=is,goal,Signup`, {
+ await page.goto(`/${domain}?period=all&f=is,goal,Signup`, {
waitUntil: 'commit'
})
+ await expect(
+ page.getByRole('button', { name: 'Remove filter: Goal is Signup' })
+ ).toBeVisible()
+
const csvs = parseAllCsvs(await triggerExportAndAwaitDownload(page))
expect(getCsv(csvs, 'visitors.csv')[0]).toEqual(
@@ -225,13 +262,25 @@ test('csv export column headers match expected metrics for each report', async (
EXPECTED_HEADERS.nameVersionGoalFiltered
)
}
+ expect(getCsv(csvs, 'custom_props.csv')[0]).toEqual(
+ EXPECTED_HEADERS.customPropsGoalFiltered
+ )
+ expect(getCsv(csvs, 'conversions.csv')[0]).toEqual(
+ EXPECTED_HEADERS.conversionsBreakdown
+ )
})
await test.step('with a custom prop filter', async () => {
- await page.goto(`/${domain}?f=is,props:author,john`, {
+ await page.goto(`/${domain}?period=all&f=is,props:author,john`, {
waitUntil: 'commit'
})
+ await expect(
+ page.getByRole('button', {
+ name: 'Remove filter: Property author is john'
+ })
+ ).toBeVisible()
+
const csvs = parseAllCsvs(await triggerExportAndAwaitDownload(page))
expect(getCsv(csvs, 'visitors.csv')[0]).toEqual(
@@ -262,6 +311,12 @@ test('csv export column headers match expected metrics for each report', async (
EXPECTED_HEADERS.nameVersionAndVisitors
)
}
+ expect(getCsv(csvs, 'custom_props.csv')[0]).toEqual(
+ EXPECTED_HEADERS.customPropsDefault
+ )
+ expect(getCsv(csvs, 'conversions.csv')[0]).toEqual(
+ EXPECTED_HEADERS.conversionsBreakdown
+ )
})
})
@@ -309,7 +364,7 @@ test('filters out empty visit:* dimension values', async ({
]
})
- await page.goto(`/${domain}?period=day&date=2021-01-01`, {
+ await page.goto(`/${domain}?period=all`, {
waitUntil: 'commit'
})
diff --git a/lib/plausible/stats/api_query_parser.ex b/lib/plausible/stats/api_query_parser.ex
index 35837f1eba..c8d16cc1f7 100644
--- a/lib/plausible/stats/api_query_parser.ex
+++ b/lib/plausible/stats/api_query_parser.ex
@@ -250,8 +250,8 @@ defmodule Plausible.Stats.ApiQueryParser do
defp parse_dimension_entry(key, error_message) do
case {
- parse_time(key),
- parse_filter_dimension_string(key)
+ parse_time_dimension(key),
+ parse_non_time_dimension(key)
} do
{{:ok, time}, _} -> {:ok, time}
{_, {:ok, dimension}} -> {:ok, dimension}
@@ -261,18 +261,18 @@ defmodule Plausible.Stats.ApiQueryParser do
defp parse_metric_or_dimension([value, _] = entry) do
case {
- parse_time(value),
+ parse_time_dimension(value),
parse_metric(value),
- parse_filter_dimension_string(value)
+ parse_non_time_dimension(value)
} do
- {{:ok, time}, _, _} ->
- {:ok, time}
+ {{:ok, time_dimension}, _, _} ->
+ {:ok, time_dimension}
{_, {:ok, metric}, _} ->
{:ok, metric}
- {_, _, {:ok, dimension}} ->
- {:ok, dimension}
+ {_, _, {:ok, non_time_dimension}} ->
+ {:ok, non_time_dimension}
_ ->
{:error,
@@ -280,13 +280,23 @@ defmodule Plausible.Stats.ApiQueryParser do
end
end
- defp parse_time("time"), do: {:ok, "time"}
- defp parse_time("time:minute"), do: {:ok, "time:minute"}
- defp parse_time("time:hour"), do: {:ok, "time:hour"}
- defp parse_time("time:day"), do: {:ok, "time:day"}
- defp parse_time("time:week"), do: {:ok, "time:week"}
- defp parse_time("time:month"), do: {:ok, "time:month"}
- defp parse_time(_), do: :error
+ defp parse_time_dimension("time"), do: {:ok, "time"}
+ defp parse_time_dimension("time:minute"), do: {:ok, "time:minute"}
+ defp parse_time_dimension("time:hour"), do: {:ok, "time:hour"}
+ defp parse_time_dimension("time:day"), do: {:ok, "time:day"}
+ defp parse_time_dimension("time:week"), do: {:ok, "time:week"}
+ defp parse_time_dimension("time:month"), do: {:ok, "time:month"}
+ defp parse_time_dimension(_), do: :error
+
+ @breakdown_only_non_time_dimensions ["event:prop_key"]
+
+ defp parse_non_time_dimension(dimension) do
+ if dimension in @breakdown_only_non_time_dimensions do
+ {:ok, dimension}
+ else
+ parse_filter_dimension_string(dimension)
+ end
+ end
defp parse_order_direction([_, "asc"]), do: {:ok, :asc}
defp parse_order_direction([_, "desc"]), do: {:ok, :desc}
diff --git a/lib/plausible/stats/dashboard/csv_export.ex b/lib/plausible/stats/dashboard/csv_export.ex
index af78a18e6d..869b6e8cf1 100644
--- a/lib/plausible/stats/dashboard/csv_export.ex
+++ b/lib/plausible/stats/dashboard/csv_export.ex
@@ -24,39 +24,55 @@ defmodule Plausible.Stats.Dashboard.CsvExport do
"utm_terms.csv",
"countries.csv",
"regions.csv",
- "cities.csv"
+ "cities.csv",
+ "custom_props.csv",
+ "conversions.csv"
]
def get_csvs(site, params, debug_metadata) do
- with {:ok, queries_by_filename} <- create_queries_by_filename(site, params, debug_metadata) do
+ with {:ok, tasks_by_filename} <- create_tasks_by_filename(site, params, debug_metadata) do
csv_values =
- queries_by_filename
- |> Enum.map(fn {filename, query} ->
- fn -> run_query_into_csv(site, filename, query) end
- end)
+ tasks_by_filename
+ |> Enum.map(fn {_filename, task} -> task end)
|> Plausible.ClickhouseRepo.parallel_tasks()
csvs =
- queries_by_filename
- |> Enum.map(fn {filename, _query} -> filename end)
+ tasks_by_filename
+ |> Enum.map(fn {filename, _task} -> filename end)
|> Enum.zip(csv_values)
+ |> Enum.reject(fn {_filename, csv_values} -> is_nil(csv_values) end)
|> Enum.map(fn {k, v} -> {String.to_charlist(k), IO.iodata_to_binary(v)} end)
{:ok, csvs}
end
end
- defp create_queries_by_filename(site, params, debug_metadata) do
+ defp create_tasks_by_filename(site, params, debug_metadata) do
requested = Map.keys(params["reports"] || %{}) |> Enum.filter(&(&1 in @csv_filenames))
Enum.reduce_while(requested, {:ok, []}, fn filename, acc ->
- case parse_csv_query(site, filename, params, debug_metadata) do
- {:ok, query} -> {:cont, {:ok, elem(acc, 1) ++ [{filename, query}]}}
+ case create_task(site, filename, params, debug_metadata) do
+ {:ok, task} -> {:cont, {:ok, elem(acc, 1) ++ [{filename, task}]}}
error_tuple -> {:halt, error_tuple}
end
end)
end
+ defp create_task(site, "custom_props.csv", params, debug_metadata) do
+ with {:ok, prop_keys_query} <- get_custom_prop_key_query(site, params, debug_metadata),
+ prop_keys = get_custom_prop_keys(site, prop_keys_query),
+ {:ok, prop_value_queries_by_prop_key} <-
+ get_custom_prop_queries(site, params, debug_metadata, prop_keys) do
+ {:ok, fn -> get_custom_props_csv(site, prop_value_queries_by_prop_key) end}
+ end
+ end
+
+ defp create_task(site, filename, params, debug_metadata) do
+ with {:ok, query} <- parse_csv_query(site, filename, params, debug_metadata) do
+ {:ok, fn -> run_query_into_csv(site, filename, query) end}
+ end
+ end
+
defp parse_csv_query(site, filename, params, debug_metadata) do
params = construct_raw_query_params(filename, params)
@@ -100,11 +116,10 @@ defmodule Plausible.Stats.Dashboard.CsvExport do
# data like time on page or bounce_rate for pages in a separate query using
# the IN filter, it causes the requests to balloon in payload size.
limit = if filename in ["pages.csv", "exit_pages.csv"], do: 100, else: 300
- order_by = [{:visitors, :desc} | Enum.map(query.dimensions, &{&1, :asc})]
query =
query
- |> Query.set(order_by: order_by)
+ |> Query.set(order_by: get_order_by(query.dimensions))
|> Query.set(pagination: %{limit: limit, offset: 0})
QueryRunner.run(site, query) |> query_result_to_csv()
@@ -136,6 +151,12 @@ defmodule Plausible.Stats.Dashboard.CsvExport do
|> Map.get(timelabel)
end
+ defp get_order_by(["event:goal"]), do: [{:visitors, :desc}]
+
+ defp get_order_by(dimensions) do
+ [{:visitors, :desc} | Enum.map(dimensions, &{&1, :asc})]
+ end
+
defp csv_first_row(query) do
csv_dimension_labels(query[:dimensions]) ++ csv_metric_labels(query)
end
@@ -165,6 +186,12 @@ defmodule Plausible.Stats.Dashboard.CsvExport do
group_conversion_rate: :conversion_rate
}
+ ["event:goal"] ->
+ %{
+ visitors: :unique_conversions,
+ events: :total_conversions
+ }
+
_ ->
%{
visitors: :conversions,
@@ -181,6 +208,10 @@ defmodule Plausible.Stats.Dashboard.CsvExport do
["visit:exit_page"] => %{
visitors: :unique_exits,
visits: :total_exits
+ },
+ ["event:goal"] => %{
+ visitors: :unique_conversions,
+ events: :total_conversions
}
}
@@ -211,4 +242,88 @@ defmodule Plausible.Stats.Dashboard.CsvExport do
defp csv_dimension_values([_dimension], row) do
row[:dimensions]
end
+
+ defp get_custom_prop_key_query(site, params, debug_metadata) do
+ raw_params =
+ construct_raw_query_params("custom_props.csv", params)
+ |> Map.merge(%{
+ "dimensions" => ["event:prop_key"],
+ "metrics" => ["visitors"],
+ "pagination" => %{"limit" => 25}
+ })
+
+ with {:ok, parsed} <- QueryParser.parse(raw_params),
+ {:ok, query} <- QueryBuilder.build(site, parsed, debug_metadata) do
+ {:ok, Query.set(query, order_by: [{:visitors, :desc}, {"event:prop_key", :asc}])}
+ end
+ end
+
+ defp get_custom_prop_keys(site, query) do
+ case Plausible.Stats.Filters.get_toplevel_filter(query, "event:props:") do
+ [_op, "event:props:" <> key | _rest] ->
+ [key]
+
+ _ ->
+ %QueryResult{results: results} = QueryRunner.run(site, query)
+
+ Enum.map(results, fn r -> hd(r.dimensions) end)
+ |> maybe_allowed_props_only(site)
+ end
+ end
+
+ defp get_custom_prop_queries(site, params, debug_metadata, prop_keys) do
+ Enum.reduce_while(prop_keys, {:ok, []}, fn prop_key, acc ->
+ dimension = "event:props:#{prop_key}"
+
+ raw_params =
+ construct_raw_query_params("custom_props.csv", params)
+ |> Map.put("dimensions", [dimension])
+
+ with {:ok, parsed} <- QueryParser.parse(raw_params),
+ {:ok, query} <- QueryBuilder.build(site, parsed, debug_metadata) do
+ query =
+ query
+ |> Query.set(order_by: [{:visitors, :desc}, {dimension, :asc}])
+ |> Query.set(pagination: %{limit: 300, offset: 0})
+
+ {:cont, {:ok, elem(acc, 1) ++ [{prop_key, query}]}}
+ else
+ error_tuple -> {:halt, error_tuple}
+ end
+ end)
+ end
+
+ defp get_custom_props_csv(_site, []), do: nil
+
+ defp get_custom_props_csv(site, prop_value_queries_by_prop_key) do
+ [{_prop_key, %Query{metrics: metrics}} | _] = prop_value_queries_by_prop_key
+ header_row = format_custom_props_header_row(metrics)
+ data_rows = custom_prop_queries_into_data_rows(site, prop_value_queries_by_prop_key)
+
+ NimbleCSV.RFC4180.dump_to_iodata([header_row] ++ data_rows)
+ end
+
+ defp custom_prop_queries_into_data_rows(site, prop_value_queries_by_prop_key) do
+ prop_value_queries_by_prop_key
+ |> Enum.map(fn {prop_key, query} ->
+ fn ->
+ %QueryResult{results: results} = QueryRunner.run(site, query)
+ Enum.map(results, &format_custom_props_data_row(prop_key, &1))
+ end
+ end)
+ |> Plausible.ClickhouseRepo.parallel_tasks()
+ |> Enum.concat()
+ end
+
+ defp format_custom_props_header_row(metrics), do: [:property, :value] ++ metrics
+
+ defp format_custom_props_data_row(prop_key, row),
+ do: [prop_key | row.dimensions] ++ row.metrics
+
+ defp maybe_allowed_props_only(prop_keys, site) do
+ case Plausible.Props.allowed_for(site) do
+ :all -> prop_keys
+ allowed -> Enum.filter(prop_keys, &(&1 in allowed))
+ end
+ end
end
diff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.ex
index 91caed4c96..8e5b4132c4 100644
--- a/lib/plausible/stats/sql/query_builder.ex
+++ b/lib/plausible/stats/sql/query_builder.ex
@@ -170,6 +170,16 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
|> group_by([], selected_as(^key))
end
+ defp dimension_group_by(q, :events, query, "event:prop_key" = dimension) do
+ from(e in q,
+ join: meta in fragment("meta"),
+ hints: "ARRAY",
+ on: true,
+ group_by: meta.key
+ )
+ |> select_merge_as([_e, meta], %{shortname(query, dimension) => meta.key})
+ end
+
defp dimension_group_by(q, :events, query, "event:goal" = dimension) do
goal_join_data = Plausible.Stats.Goals.goal_join_data(query)
diff --git a/test/plausible/stats/dashboard/query_parser_test.exs b/test/plausible/stats/dashboard/query_parser_test.exs
index 32b08eaa8d..8e855caad2 100644
--- a/test/plausible/stats/dashboard/query_parser_test.exs
+++ b/test/plausible/stats/dashboard/query_parser_test.exs
@@ -193,6 +193,27 @@ defmodule Plausible.Stats.Dashboard.QueryParserTest do
end
end
+ describe "dimensions" do
+ test "parses breakdown-only dimensions" do
+ params = Map.merge(@base_params, %{"dimensions" => ["event:prop_key"]})
+ {:ok, parsed} = parse(params)
+ assert parsed.dimensions == ["event:prop_key"]
+ end
+ end
+
+ describe "order_by" do
+ test "breakdown-only dimensions can be used in order_by" do
+ params =
+ Map.merge(@base_params, %{
+ "dimensions" => ["event:prop_key"],
+ "order_by" => [["event:prop_key", "asc"]]
+ })
+
+ {:ok, parsed} = parse(params)
+ assert parsed.order_by == [{"event:prop_key", :asc}]
+ end
+ end
+
describe "filters" do
test "parses valid filters" do
params =
@@ -215,6 +236,15 @@ defmodule Plausible.Stats.Dashboard.QueryParserTest do
]
end
+ test "cannot filter by breakdown-only dimensions" do
+ params =
+ Map.merge(@base_params, %{
+ "filters" => [["is", "event:prop_key", ["author"]]]
+ })
+
+ {:error, %QueryError{code: :invalid_filters}} = parse(params)
+ end
+
test "parses city filter with multiple clauses" do
params =
Map.merge(@base_params, %{
diff --git a/test/plausible/stats/query/query_test.exs b/test/plausible/stats/query/query_test.exs
index 4b513f4bfb..1616571d52 100644
--- a/test/plausible/stats/query/query_test.exs
+++ b/test/plausible/stats/query/query_test.exs
@@ -601,4 +601,73 @@ defmodule Plausible.Stats.QueryTest do
assert is_nil(meta[:empty_metrics])
end
end
+
+ describe "event:prop_key dimension" do
+ test "returns distinct prop keys with visitor counts", %{site: site} do
+ populate_stats(site, [
+ build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),
+ build(:pageview, "meta.key": ["author"], "meta.value": ["marko"]),
+ build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"]),
+ build(:pageview)
+ ])
+
+ {:ok, query} =
+ QueryBuilder.build(site, %ParsedQueryParams{
+ metrics: [:visitors],
+ input_date_range: :all,
+ dimensions: ["event:prop_key"]
+ })
+
+ %Stats.QueryResult{results: results} = Stats.query(site, query)
+
+ assert [
+ %{dimensions: ["author"], metrics: [2]},
+ %{dimensions: ["logged_in"], metrics: [1]}
+ ] = Enum.sort_by(results, &hd(&1.dimensions))
+ end
+
+ test "supports order_by", %{site: site} do
+ populate_stats(site, [
+ build(:pageview, "meta.key": ["author"], "meta.value": ["a"]),
+ build(:pageview, "meta.key": ["author"], "meta.value": ["b"]),
+ build(:pageview, "meta.key": ["author"], "meta.value": ["b"]),
+ build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"])
+ ])
+
+ {:ok, query} =
+ QueryBuilder.build(site, %ParsedQueryParams{
+ metrics: [:visitors],
+ input_date_range: :all,
+ dimensions: ["event:prop_key"],
+ order_by: [{"event:prop_key", :desc}]
+ })
+
+ %Stats.QueryResult{results: results} = Stats.query(site, query)
+
+ assert results == [
+ %{dimensions: ["logged_in"], metrics: [1]},
+ %{dimensions: ["author"], metrics: [3]}
+ ]
+ end
+
+ test "works combined with a filter", %{site: site} do
+ populate_stats(site, [
+ build(:event, name: "purchase", "meta.key": ["author"], "meta.value": ["uku"]),
+ build(:pageview, "meta.key": ["author"], "meta.value": ["marko"]),
+ build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"])
+ ])
+
+ {:ok, query} =
+ QueryBuilder.build(site, %ParsedQueryParams{
+ metrics: [:visitors],
+ input_date_range: :all,
+ dimensions: ["event:prop_key"],
+ filters: [[:is, "event:name", ["purchase"]]]
+ })
+
+ %Stats.QueryResult{results: results} = Stats.query(site, query)
+
+ assert results == [%{dimensions: ["author"], metrics: [1]}]
+ end
+ end
end
diff --git a/test/plausible_web/controllers/api/stats_controller/dashboard_csv_export_test.exs b/test/plausible_web/controllers/api/stats_controller/dashboard_csv_export_test.exs
index ecc13363e7..fee06be0c2 100644
--- a/test/plausible_web/controllers/api/stats_controller/dashboard_csv_export_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/dashboard_csv_export_test.exs
@@ -99,6 +99,14 @@ defmodule PlausibleWeb.Api.StatsController.DashboardCsvExportTest do
dimensions: ["visit:city_name"],
always_on_filters: [["is_not", "visit:city", [0]]],
metrics: ["visitors"]
+ },
+ "custom_props.csv" => %{
+ dimensions: ["event:props:*"],
+ metrics: ["visitors", "events", "percentage"]
+ },
+ "conversions.csv" => %{
+ dimensions: ["event:goal"],
+ metrics: ["visitors", "events"]
}
}
@@ -114,12 +122,24 @@ defmodule PlausibleWeb.Api.StatsController.DashboardCsvExportTest do
["exit_pages.csv", :metrics],
get_in(@default_reports, ["exit_pages.csv", :metrics]) -- ["exit_rate"]
)
+ @prop_filtered_reports put_in(
+ @default_reports,
+ ["exit_pages.csv", :metrics],
+ get_in(@default_reports, ["exit_pages.csv", :metrics]) -- ["exit_rate"]
+ )
@goal_filtered_reports Map.new(@default_reports, fn
{"visitors.csv" = filename, params} ->
{filename,
%{params | metrics: ["visitors", "events", "group_conversion_rate"]}}
+ {"custom_props.csv" = filename, params} ->
+ {filename,
+ %{params | metrics: ["visitors", "events", "conversion_rate"]}}
+
+ {"conversions.csv" = filename, params} ->
+ {filename, params}
+
{filename, params} ->
{filename,
%{params | metrics: ["visitors", "group_conversion_rate"]}}
@@ -150,7 +170,7 @@ defmodule PlausibleWeb.Api.StatsController.DashboardCsvExportTest do
assert ~c"browsers.csv" in zip
assert ~c"browser_versions.csv" in zip
assert ~c"cities.csv" in zip
- # assert ~c"conversions.csv" in zip
+ assert ~c"conversions.csv" in zip
assert ~c"countries.csv" in zip
assert ~c"devices.csv" in zip
assert ~c"entry_pages.csv" in zip
@@ -185,6 +205,33 @@ defmodule PlausibleWeb.Api.StatsController.DashboardCsvExportTest do
assert length(exit_pages) == 102
end
+ test "limits custom_props.csv to 25 prop keys", %{conn: conn, site: site} do
+ keys = for i <- 1..26, do: "key_#{i}"
+ {:ok, site} = Plausible.Props.allow(site, keys)
+
+ populate_stats(site, [
+ build(:pageview,
+ "meta.key": keys,
+ "meta.value": List.duplicate("v", 26)
+ )
+ ])
+
+ custom_props =
+ conn
+ |> do_export(site, %{@base_params | date_range: "day"})
+ |> response(200)
+ |> unzip_and_parse_csv(~c"custom_props.csv")
+
+ prop_keys =
+ custom_props
+ |> Enum.drop(1)
+ |> Enum.reject(&(&1 == [""]))
+ |> Enum.map(&hd/1)
+ |> Enum.uniq()
+
+ assert length(prop_keys) == 25
+ end
+
test "limits other breakdown reports to 300 rows", %{conn: conn, site: site} do
events = for i <- 1..301, do: build(:pageview, referrer_source: "Source #{i}")
populate_stats(site, events)
@@ -268,57 +315,57 @@ defmodule PlausibleWeb.Api.StatsController.DashboardCsvExportTest do
]
end
- # test "exports only internally used props in custom_props.csv for a growth plan", %{
- # conn: conn,
- # site: site
- # } do
- # {:ok, site} = Plausible.Props.allow(site, ["author"])
-
- # [owner | _] = Repo.preload(site, :owners).owners
- # subscribe_to_growth_plan(owner)
-
- # populate_stats(site, [
- # build(:pageview, "meta.key": ["author"], "meta.value": ["a"]),
- # build(:event, name: "File Download", "meta.key": ["url"], "meta.value": ["b"])
- # ])
-
- # result =
- # conn
- # |> do_export(site, %{@base_params | date_range: "day"})
- # |> response(200)
- # |> unzip_and_parse_csv(~c"custom_props.csv")
-
- # assert result == [
- # ["property", "value", "visitors", "events", "percentage"],
- # ["url", "(none)", "1", "1", "50.0"],
- # ["url", "b", "1", "1", "50.0"],
- # [""]
- # ]
- # end
-
- # test "does not include custom_props.csv for a growth plan if no internal props used", %{
- # conn: conn,
- # site: site
- # } do
- # {:ok, site} = Plausible.Props.allow(site, ["author"])
-
- # [owner | _] = Repo.preload(site, :owners).owners
- # subscribe_to_growth_plan(owner)
-
- # populate_stats(site, [
- # build(:pageview, "meta.key": ["author"], "meta.value": ["a"])
- # ])
-
- # {:ok, zip} =
- # conn
- # |> do_export(site, %{@base_params | date_range: "day"})
- # |> response(200)
- # |> :zip.unzip([:memory])
-
- # files = Map.new(zip)
-
- # refute Map.has_key?(files, ~c"custom_props.csv")
- # end
+ test "exports only internally used props in custom_props.csv for a growth plan", %{
+ conn: conn,
+ site: site
+ } do
+ {:ok, site} = Plausible.Props.allow(site, ["author"])
+
+ [owner | _] = Repo.preload(site, :owners).owners
+ subscribe_to_growth_plan(owner)
+
+ populate_stats(site, [
+ build(:pageview, "meta.key": ["author"], "meta.value": ["a"]),
+ build(:event, name: "File Download", "meta.key": ["url"], "meta.value": ["b"])
+ ])
+
+ result =
+ conn
+ |> do_export(site, %{@base_params | date_range: "day"})
+ |> response(200)
+ |> unzip_and_parse_csv(~c"custom_props.csv")
+
+ assert result == [
+ ["property", "value", "visitors", "events", "percentage"],
+ ["url", "(none)", "1", "1", "50.0"],
+ ["url", "b", "1", "1", "50.0"],
+ [""]
+ ]
+ end
+
+ test "does not include custom_props.csv for a growth plan if no internal props used", %{
+ conn: conn,
+ site: site
+ } do
+ {:ok, site} = Plausible.Props.allow(site, ["author"])
+
+ [owner | _] = Repo.preload(site, :owners).owners
+ subscribe_to_growth_plan(owner)
+
+ populate_stats(site, [
+ build(:pageview, "meta.key": ["author"], "meta.value": ["a"])
+ ])
+
+ {:ok, zip} =
+ conn
+ |> do_export(site, %{@base_params | date_range: "day"})
+ |> response(200)
+ |> :zip.unzip([:memory])
+
+ files = Map.new(zip)
+
+ refute Map.has_key?(files, ~c"custom_props.csv")
+ end
test "exports data in zipped csvs", %{conn: conn, site: site} do
populate_exported_stats(site)
@@ -329,35 +376,35 @@ defmodule PlausibleWeb.Api.StatsController.DashboardCsvExportTest do
assert_zip(conn, "30d")
end
- # test "exports allowed event props for a trial account", %{conn: conn, site: site} do
- # {:ok, site} = Plausible.Props.allow(site, ["author", "logged_in"])
-
- # populate_stats(site, [
- # build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),
- # build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),
- # build(:event, "meta.key": ["author"], "meta.value": ["marko"], name: "Newsletter Signup"),
- # build(:pageview, user_id: 999, "meta.key": ["logged_in"], "meta.value": ["true"]),
- # build(:pageview, user_id: 999, "meta.key": ["logged_in"], "meta.value": ["true"]),
- # build(:pageview, "meta.key": ["disallowed"], "meta.value": ["whatever"]),
- # build(:pageview)
- # ])
-
- # result =
- # conn
- # |> do_export(site, %{@base_params | date_range: "day"})
- # |> response(200)
- # |> unzip_and_parse_csv(~c"custom_props.csv")
-
- # assert result == [
- # ["property", "value", "visitors", "events", "percentage"],
- # ["author", "(none)", "3", "4", "50.0"],
- # ["author", "uku", "2", "2", "33.33"],
- # ["author", "marko", "1", "1", "16.67"],
- # ["logged_in", "(none)", "5", "5", "83.33"],
- # ["logged_in", "true", "1", "2", "16.67"],
- # [""]
- # ]
- # end
+ test "exports allowed event props for a trial account", %{conn: conn, site: site} do
+ {:ok, site} = Plausible.Props.allow(site, ["author", "logged_in"])
+
+ populate_stats(site, [
+ build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),
+ build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),
+ build(:event, "meta.key": ["author"], "meta.value": ["marko"], name: "Newsletter Signup"),
+ build(:pageview, user_id: 999, "meta.key": ["logged_in"], "meta.value": ["true"]),
+ build(:pageview, user_id: 999, "meta.key": ["logged_in"], "meta.value": ["true"]),
+ build(:pageview, "meta.key": ["disallowed"], "meta.value": ["whatever"]),
+ build(:pageview)
+ ])
+
+ result =
+ conn
+ |> do_export(site, %{@base_params | date_range: "day"})
+ |> response(200)
+ |> unzip_and_parse_csv(~c"custom_props.csv")
+
+ assert result == [
+ ["property", "value", "visitors", "events", "percentage"],
+ ["author", "(none)", "3", "4", "50.0"],
+ ["author", "uku", "2", "2", "33.33"],
+ ["author", "marko", "1", "1", "16.67"],
+ ["logged_in", "(none)", "5", "5", "83.33"],
+ ["logged_in", "true", "1", "2", "16.67"],
+ [""]
+ ]
+ end
test "exports data grouped by interval", %{conn: conn, site: site} do
populate_exported_stats(site)
@@ -479,7 +526,7 @@ defmodule PlausibleWeb.Api.StatsController.DashboardCsvExportTest do
"operating_systems.csv",
"operating_system_versions.csv",
"devices.csv",
- # "conversions.csv",
+ "conversions.csv",
"referrers.csv"
]
@@ -630,6 +677,9 @@ defmodule PlausibleWeb.Api.StatsController.DashboardCsvExportTest do
["Direct / None", "1", "0.0", "10.0"],
[""]
]
+
+ {~c"custom_props.csv", _data} ->
+ :ok
end)
end
end
@@ -752,38 +802,39 @@ defmodule PlausibleWeb.Api.StatsController.DashboardCsvExportTest do
end
end
- # describe "POST /api/stats/:domain/export - with a custom prop filter" do
- # setup [:create_user, :create_site, :log_in]
-
- # test "custom-props.csv only returns the prop and its value in filter", %{
- # conn: conn,
- # site: site
- # } do
- # {:ok, site} = Plausible.Props.allow(site, ["author", "logged_in"])
-
- # populate_stats(site, [
- # build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),
- # build(:pageview, "meta.key": ["author"], "meta.value": ["marko"]),
- # build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"])
- # ])
-
- # result =
- # conn
- # |> do_export(site, %{
- # @base_params
- # | date_range: "day",
- # filters: [["is", "event:props:author", ["marko"]]]
- # })
- # |> response(200)
- # |> unzip_and_parse_csv(~c"custom_props.csv")
-
- # assert result == [
- # ["property", "value", "visitors", "events", "percentage"],
- # ["author", "marko", "1", "1", "100.0"],
- # [""]
- # ]
- # end
- # end
+ describe "POST /api/stats/:domain/export - with a custom prop filter" do
+ setup [:create_user, :create_site, :log_in]
+
+ test "custom-props.csv only returns the prop and its value in filter", %{
+ conn: conn,
+ site: site
+ } do
+ {:ok, site} = Plausible.Props.allow(site, ["author", "logged_in"])
+
+ populate_stats(site, [
+ build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),
+ build(:pageview, "meta.key": ["author"], "meta.value": ["marko"]),
+ build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"])
+ ])
+
+ result =
+ conn
+ |> do_export(site, %{
+ @base_params
+ | date_range: "day",
+ filters: [["is", "event:props:author", ["marko"]]],
+ reports: @prop_filtered_reports
+ })
+ |> response(200)
+ |> unzip_and_parse_csv(~c"custom_props.csv")
+
+ assert result == [
+ ["property", "value", "visitors", "events", "percentage"],
+ ["author", "marko", "1", "1", "100.0"],
+ [""]
+ ]
+ end
+ end
defp unzip_and_parse_csv(archive, filename) do
{:ok, zip} = :zip.unzip(archive, [:memory])
@@ -998,39 +1049,39 @@ defmodule PlausibleWeb.Api.StatsController.DashboardCsvExportTest do
assert_zip(conn, "30d-filter-goal")
end
- # test "custom-props.csv only returns the prop names for the goal in filter", %{
- # conn: conn,
- # site: site
- # } do
- # {:ok, site} = Plausible.Props.allow(site, ["author", "logged_in"])
-
- # populate_stats(site, [
- # build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["uku"]),
- # build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["marko"]),
- # build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["marko"]),
- # build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"])
- # ])
-
- # insert(:goal, site: site, event_name: "Newsletter Signup")
-
- # result =
- # conn
- # |> do_export(site, %{
- # @base_params
- # | date_range: "day",
- # filters: [["is", "event:goal", ["Newsletter Signup"]]],
- # reports: @goal_filtered_reports
- # })
- # |> response(200)
- # |> unzip_and_parse_csv(~c"custom_props.csv")
-
- # assert result == [
- # ["property", "value", "visitors", "events", "conversion_rate"],
- # ["author", "marko", "2", "2", "50.0"],
- # ["author", "uku", "1", "1", "25.0"],
- # [""]
- # ]
- # end
+ test "custom-props.csv only returns the prop names for the goal in filter", %{
+ conn: conn,
+ site: site
+ } do
+ {:ok, site} = Plausible.Props.allow(site, ["author", "logged_in"])
+
+ populate_stats(site, [
+ build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["uku"]),
+ build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["marko"]),
+ build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["marko"]),
+ build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"])
+ ])
+
+ insert(:goal, site: site, event_name: "Newsletter Signup")
+
+ result =
+ conn
+ |> do_export(site, %{
+ @base_params
+ | date_range: "day",
+ filters: [["is", "event:goal", ["Newsletter Signup"]]],
+ reports: @goal_filtered_reports
+ })
+ |> response(200)
+ |> unzip_and_parse_csv(~c"custom_props.csv")
+
+ assert result == [
+ ["property", "value", "visitors", "events", "conversion_rate"],
+ ["author", "marko", "2", "2", "50.0"],
+ ["author", "uku", "1", "1", "25.0"],
+ [""]
+ ]
+ end
test "exports conversions and conversion rate for operating system versions", %{
conn: conn,