Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Dashboard csv export v2 (#6436)

RobertJoonasba238ad3c4e7
diff --git a/assets/js/dashboard/api.ts b/assets/js/dashboard/api.ts
index 1eb66db7f9..c2e748b8e9 100644
--- a/assets/js/dashboard/api.ts
+++ b/assets/js/dashboard/api.ts
@@ -6,6 +6,7 @@ import { formatISO } from './util/date'
import { serializeApiFilters } from './util/filters'
import * as url from './util/url'
import { MainGraphResponse } from './stats/graph/fetch-main-graph'
+import { CsvExportRequestBody } from './stats/csv-export/csv-export-body'
let abortController = new AbortController()
let SHARED_LINK_AUTH: null | string = null
@@ -140,13 +141,16 @@ function getHeaders(): Record<string, string> {
return SHARED_LINK_AUTH ? { 'X-Shared-Link-Auth': SHARED_LINK_AUTH } : {}
}
-async function handleApiResponse(response: Response) {
- const payload = await response.json()
+async function throwApiErrorIfNotOk(response: Response) {
if (!response.ok) {
+ const payload = await response.json()
throw new ApiError(payload.error, payload, response.status)
}
+}
- return payload
+async function handleApiResponse(response: Response) {
+ await throwApiErrorIfNotOk(response)
+ return response.json()
}
function getSharedLinkSearchParams(): Record<string, string> {
@@ -175,6 +179,24 @@ export async function stats<
return (await handleApiResponse(response)) as TResponse
}
+export async function csvExport(
+ site: PlausibleSite,
+ body: CsvExportRequestBody
+): Promise<Blob> {
+ const sharedLinkParams = getSharedLinkSearchParams()
+ const queryString = sharedLinkParams.auth
+ ? new URLSearchParams(sharedLinkParams).toString()
+ : ''
+ const path = url.apiPath(site, '/export')
+ const response = await fetch(queryString ? `${path}?${queryString}` : path, {
+ method: 'POST',
+ headers: { ...getHeaders(), 'Content-Type': 'application/json' },
+ body: JSON.stringify(body)
+ })
+ await throwApiErrorIfNotOk(response)
+ return response.blob()
+}
+
export async function get(
url: string,
dashboardState?: DashboardState,
diff --git a/assets/js/dashboard/nav-menu/dashboard-options-menu.tsx b/assets/js/dashboard/nav-menu/dashboard-options-menu.tsx
index 88567690af..ac2041a93a 100644
--- a/assets/js/dashboard/nav-menu/dashboard-options-menu.tsx
+++ b/assets/js/dashboard/nav-menu/dashboard-options-menu.tsx
@@ -1,4 +1,4 @@
-import React, { useRef, useState } from 'react'
+import React, { useEffect, useRef, useState } from 'react'
import { Popover, Transition } from '@headlessui/react'
import {
EllipsisVerticalIcon,
@@ -20,6 +20,7 @@ import { Notice } from '../components/notice'
import { isModifierPressed, isTyping, Keybind } from '../keybinding'
import { useMatch } from 'react-router-dom'
import { rootRoute } from '../router'
+import { CsvExportV2, ExportStatus } from '../stats/csv-export/csv-export'
function ExportItem({ selectedInterval }: { selectedInterval: string }) {
const site = useSiteContext()
@@ -91,10 +92,21 @@ function ImportedSwitchItem({ disabled }: { disabled: boolean }) {
}
function DashboardOptionsMenuItems() {
+ const site = useSiteContext()
+ const { dashboardState } = useDashboardStateContext()
const { selectedInterval, onIntervalClick, availableIntervals } =
useGraphIntervalContext()
const imports = useImportsIncludedContext()
const buttonRef = useRef<HTMLButtonElement>(null)
+ const [exportStatus, setExportStatus] = useState<ExportStatus>(
+ ExportStatus.idle
+ )
+
+ useEffect(() => {
+ setExportStatus((prev) =>
+ prev === ExportStatus.error ? ExportStatus.idle : prev
+ )
+ }, [dashboardState])
const showIntervalSection = availableIntervals.length > 1
@@ -146,7 +158,14 @@ function DashboardOptionsMenuItems() {
options={availableIntervals}
/>
)}
- <ExportItem selectedInterval={selectedInterval} />
+ {site.flags.dashboard_csv_export_v2 ? (
+ <CsvExportV2
+ exportStatus={exportStatus}
+ setExportStatus={setExportStatus}
+ />
+ ) : (
+ <ExportItem selectedInterval={selectedInterval} />
+ )}
{imports.status === 'visible' && (
<>
<ImportedSwitchItem disabled={imports.disabled} />
diff --git a/assets/js/dashboard/site-context.test.tsx b/assets/js/dashboard/site-context.test.tsx
index 13ddfea577..5822549247 100644
--- a/assets/js/dashboard/site-context.test.tsx
+++ b/assets/js/dashboard/site-context.test.tsx
@@ -32,7 +32,7 @@ describe('parseSiteFromDataset', () => {
data-is-dbip="false"
data-current-user-role="owner"
data-current-user-id="1"
- data-flags="{}"
+ data-flags='{"dashboard_csv_export_v2": false}'
data-valid-intervals-by-period='{"12mo":["day","week","month"],"7d":["hour","day"],"28d":["day","week"],"30d":["day","week"],"91d":["day","week","month"],"6mo":["day","week","month"],"all":["week","month"],"custom":["day","week","month"],"day":["minute","hour"],"month":["day","week"],"realtime":["minute"],"year":["day","week","month"]}'
{...attrs}
/>
@@ -58,7 +58,7 @@ describe('parseSiteFromDataset', () => {
embedded: false,
background: undefined,
isDbip: false,
- flags: {},
+ flags: { dashboard_csv_export_v2: false },
shared: false,
isConsolidatedView: false
}
diff --git a/assets/js/dashboard/site-context.tsx b/assets/js/dashboard/site-context.tsx
index a7ea4d3fc8..2e61d5e9be 100644
--- a/assets/js/dashboard/site-context.tsx
+++ b/assets/js/dashboard/site-context.tsx
@@ -32,7 +32,7 @@ export function parseSiteFromDataset(dataset: DOMStringMap): PlausibleSite {
}
// Update this object when new feature flags are added to the frontend.
-type FeatureFlags = Record<never, boolean>
+type FeatureFlags = Record<'dashboard_csv_export_v2', boolean>
export const siteContextDefaultValue = {
domain: '',
diff --git a/assets/js/dashboard/stats-query.ts b/assets/js/dashboard/stats-query.ts
index 2e49f3b1f7..234a593d8b 100644
--- a/assets/js/dashboard/stats-query.ts
+++ b/assets/js/dashboard/stats-query.ts
@@ -7,13 +7,7 @@ import {
} from './dashboard-state'
import { ComparisonMode, DashboardPeriod } from './dashboard-time-periods'
import { formatISO } from './util/date'
-import {
- Pagination,
- SimpleFilterDimensions,
- CustomPropertyFilterDimensions,
- TimeDimensions,
- SortDirection
-} from '../types/query-api'
+import * as apiTypes from '../types/query-api'
import { remapToApiFilters } from './util/filters'
export type FilterModifiers = { case_sensitive?: boolean }
@@ -23,22 +17,26 @@ export type ApiFilter =
| [FilterOperator, FilterKey, FilterClause[], FilterModifiers]
export type NonTimeDimension =
- | SimpleFilterDimensions
- | CustomPropertyFilterDimensions
+ | apiTypes.SimpleFilterDimensions
+ | apiTypes.CustomPropertyFilterDimensions
+
+export type TimeDimension = apiTypes.TimeDimensions | 'time:minute'
-export type Dimension = NonTimeDimension | TimeDimensions | 'time:minute'
+export type Dimension = NonTimeDimension | TimeDimension
-export type OrderByEntry = [Metric | NonTimeDimension, SortDirection]
+export type OrderByEntry = [Metric | NonTimeDimension, apiTypes.SortDirection]
export type OrderBy = OrderByEntry[]
-type DateRange = DashboardPeriod | [string, string]
+export type DateRange = DashboardPeriod | [string, string]
+export type RelativeDate = string | null
+
type IncludeCompare =
| ComparisonMode.previous_period
| ComparisonMode.year_over_year
| string[]
| null
-type QueryInclude = {
+export type QueryInclude = {
imports: boolean
imports_meta: boolean
time_labels: boolean
@@ -54,19 +52,19 @@ export type ReportParams = {
dimensions: Dimension[]
include?: Partial<QueryInclude>
order_by?: OrderBy
- pagination?: Pagination
+ pagination?: apiTypes.Pagination
alwaysOnFilters?: ApiFilter[]
}
export type StatsQuery = {
date_range: DateRange
- relative_date: string | null
+ relative_date: RelativeDate
filters: ApiFilter[]
dimensions: Dimension[]
metrics: Metric[]
include: QueryInclude
order_by?: OrderBy | null
- pagination?: Pagination | null
+ pagination?: apiTypes.Pagination | null
}
export function addFilter(
@@ -104,7 +102,7 @@ export function createStatsQuery(
}
}
-function createDateRange(dashboardState: DashboardState): DateRange {
+export function createDateRange(dashboardState: DashboardState): DateRange {
if (dashboardState.period === DashboardPeriod.custom) {
return [formatISO(dashboardState.from), formatISO(dashboardState.to)]
} else {
diff --git a/assets/js/dashboard/stats/breakdowns.tsx b/assets/js/dashboard/stats/breakdowns.tsx
index 249d7785f7..ac01583061 100644
--- a/assets/js/dashboard/stats/breakdowns.tsx
+++ b/assets/js/dashboard/stats/breakdowns.tsx
@@ -13,7 +13,6 @@ import {
StatsQuery
} from '../stats-query'
import { Filter } from '../dashboard-state'
-import { MetricsByContext } from './reports/reports-config'
import classNames from 'classnames'
import { DIRECT_NONE } from './sources'
@@ -214,46 +213,6 @@ export function extractMetricValue(
return { metricIndex, value, comparison }
}
-type MetricContext = {
- isRealtime: boolean
- isDetailed: boolean
- hasConversionGoalFilter: boolean
- isRevenueAvailable: boolean
-}
-
-export const chooseBreakdownMetricsByContext = (
- metricsByContext: MetricsByContext,
- context: MetricContext
-): Metric[] => {
- const {
- isRealtime,
- isDetailed,
- hasConversionGoalFilter,
- isRevenueAvailable
- } = context
-
- if (hasConversionGoalFilter && isDetailed && isRevenueAvailable) {
- return [
- ...metricsByContext.goalFilterDetailedMetrics,
- 'total_revenue',
- 'average_revenue'
- ]
- }
- if (hasConversionGoalFilter && isDetailed) {
- return metricsByContext.goalFilterDetailedMetrics
- }
- if (hasConversionGoalFilter) {
- return metricsByContext.goalFilterIndexMetrics
- }
- if (isRealtime) {
- return metricsByContext.realtimeMetrics
- }
- if (isDetailed) {
- return metricsByContext.defaultDetailedMetrics
- }
- return metricsByContext.defaultIndexMetrics
-}
-
export function addDimensionSearchFilter(
statsQuery: StatsQuery,
dimension: string,
diff --git a/assets/js/dashboard/stats/csv-export/csv-export-body.ts b/assets/js/dashboard/stats/csv-export/csv-export-body.ts
new file mode 100644
index 0000000000..860ee609e0
--- /dev/null
+++ b/assets/js/dashboard/stats/csv-export/csv-export-body.ts
@@ -0,0 +1,142 @@
+import { DashboardState } from '../../dashboard-state'
+import {
+ ApiFilter,
+ createDateRange,
+ DateRange,
+ TimeDimension,
+ QueryInclude,
+ RelativeDate
+} from '../../stats-query'
+import { formatISO } from '../../util/date'
+import {
+ hasConversionGoalFilter,
+ hasEventFilters,
+ hasPageFilter,
+ remapToApiFilters
+} from '../../util/filters'
+import { Interval } from '../graph/intervals'
+import { Metric } from '../metrics'
+import {
+ BREAKDOWN_REPORTS,
+ BreakdownReportConfig,
+ BreakdownReportKey
+} from '../reports/reports-config'
+
+const BREAKDOWN_CSV_REPORTS = {
+ 'pages.csv': BreakdownReportKey.pages,
+ 'entry_pages.csv': BreakdownReportKey.entryPages,
+ 'exit_pages.csv': BreakdownReportKey.exitPages,
+ 'browsers.csv': BreakdownReportKey.browsers,
+ 'browser_versions.csv': BreakdownReportKey.browserVersions,
+ 'operating_systems.csv': BreakdownReportKey.operatingSystems,
+ 'operating_system_versions.csv': BreakdownReportKey.operatingSystemVersions,
+ 'devices.csv': BreakdownReportKey.screenSizes,
+ 'channels.csv': BreakdownReportKey.channels,
+ 'sources.csv': BreakdownReportKey.sources,
+ 'referrers.csv': BreakdownReportKey.referrers,
+ 'utm_mediums.csv': BreakdownReportKey.utmMediums,
+ 'utm_sources.csv': BreakdownReportKey.utmSources,
+ 'utm_campaigns.csv': BreakdownReportKey.utmCampaigns,
+ 'utm_contents.csv': BreakdownReportKey.utmContents,
+ 'utm_terms.csv': BreakdownReportKey.utmTerms,
+ 'countries.csv': BreakdownReportKey.countries,
+ 'regions.csv': BreakdownReportKey.regions,
+ 'cities.csv': BreakdownReportKey.cities
+}
+type CsvFilename = 'visitors.csv' | keyof typeof BREAKDOWN_CSV_REPORTS
+
+type CsvReportParams = {
+ dimensions: [TimeDimension] | BreakdownReportConfig['dimensions']
+ metrics: Metric[]
+ always_on_filters?: BreakdownReportConfig['alwaysOnFilters']
+}
+type CsvReportsConfig = Record<CsvFilename, CsvReportParams>
+
+export type CsvExportRequestBody = {
+ date_range: DateRange
+ relative_date: RelativeDate
+ filters: ApiFilter[]
+ include: Pick<QueryInclude, 'imports'>
+ reports: CsvReportsConfig
+}
+
+const DEFAULT_VISITORS_CSV_METRICS: Metric[] = [
+ 'visitors',
+ 'pageviews',
+ 'visits',
+ 'views_per_visit',
+ 'bounce_rate',
+ 'visit_duration'
+]
+const PAGE_FILTERED_VISITORS_CSV_METRICS: Metric[] = [
+ 'visitors',
+ 'pageviews',
+ 'visits',
+ 'bounce_rate',
+ 'time_on_page',
+ 'scroll_depth'
+]
+const GOAL_FILTERED_VISITORS_CSV_METRICS: Metric[] = [
+ 'visitors',
+ 'events',
+ 'conversion_rate'
+]
+
+function csvDimensions(
+ reportKey: BreakdownReportKey
+): CsvReportParams['dimensions'] {
+ switch (reportKey) {
+ case BreakdownReportKey.countries:
+ return ['visit:country_name']
+ case BreakdownReportKey.regions:
+ return ['visit:region_name']
+ case BreakdownReportKey.cities:
+ return ['visit:city_name']
+ default:
+ return BREAKDOWN_REPORTS[reportKey].dimensions
+ }
+}
+
+export function createCsvExportRequestBody(
+ dashboardState: DashboardState,
+ graphInterval: Interval
+): CsvExportRequestBody {
+ const isGoalFilter = hasConversionGoalFilter(dashboardState)
+ const isPageFilter = hasPageFilter(dashboardState)
+ const isEventFilter = hasEventFilters(dashboardState)
+
+ const reports: CsvReportsConfig = {
+ 'visitors.csv': {
+ dimensions: [`time:${graphInterval}`],
+ metrics: isGoalFilter
+ ? GOAL_FILTERED_VISITORS_CSV_METRICS
+ : isPageFilter
+ ? PAGE_FILTERED_VISITORS_CSV_METRICS
+ : DEFAULT_VISITORS_CSV_METRICS
+ },
+ ...Object.entries(BREAKDOWN_CSV_REPORTS).reduce(
+ (acc, [filename, reportKey]) => {
+ const config = BREAKDOWN_REPORTS[reportKey]
+ acc[filename as keyof typeof BREAKDOWN_CSV_REPORTS] = {
+ dimensions: csvDimensions(reportKey),
+ always_on_filters: config.alwaysOnFilters,
+ metrics: config.getMetrics({
+ isCsv: true,
+ hasConversionGoalFilter: isGoalFilter,
+ hasEventFilters: isEventFilter
+ })
+ }
+ return acc
+ },
+ {} as Omit<CsvReportsConfig, 'visitors.csv'>
+ )
+ }
+
+ return {
+ date_range: createDateRange(dashboardState),
+ relative_date: dashboardState.date ? formatISO(dashboardState.date) : null,
+ filters: remapToApiFilters(dashboardState.filters),
+ include: { imports: dashboardState.with_imported },
+ reports: reports
+ }
+}
diff --git a/assets/js/dashboard/stats/csv-export/csv-export.test.tsx b/assets/js/dashboard/stats/csv-export/csv-export.test.tsx
new file mode 100644
index 0000000000..5f601a0c6b
--- /dev/null
+++ b/assets/js/dashboard/stats/csv-export/csv-export.test.tsx
@@ -0,0 +1,56 @@
+import React from 'react'
+import { render, screen } from '@testing-library/react'
+import userEvent from '@testing-library/user-event'
+import { TestContextProviders } from '../../../../test-utils/app-context-providers'
+import { CsvExportV2, ExportStatus } from './csv-export'
+
+const exportButton = () => screen.getByRole('button', { name: /export stats/i })
+
+function renderCsvExport({
+ exportStatus,
+ setExportStatus
+}: {
+ exportStatus: ExportStatus
+ setExportStatus: () => void
+}) {
+ render(
+ <CsvExportV2
+ exportStatus={exportStatus}
+ setExportStatus={setExportStatus}
+ />,
+ {
+ wrapper: (props) => (
+ <TestContextProviders
+ {...props}
+ siteOptions={{ flags: { dashboard_csv_export_v2: true } }}
+ />
+ )
+ }
+ )
+}
+
+test('shows spinner while exporting', () => {
+ renderCsvExport({
+ exportStatus: ExportStatus.exporting,
+ setExportStatus: jest.fn()
+ })
+ expect(exportButton().querySelector('.animate-spin')).toBeInTheDocument()
+})
+
+test('does not start a second export while one is in progress', async () => {
+ const setExportStatus = jest.fn()
+ renderCsvExport({ exportStatus: ExportStatus.exporting, setExportStatus })
+
+ await userEvent.click(exportButton())
+
+ expect(setExportStatus).not.toHaveBeenCalled()
+})
+
+test('shows error icon when export fails', () => {
+ renderCsvExport({
+ exportStatus: ExportStatus.error,
+ setExportStatus: jest.fn()
+ })
+
+ expect(screen.getByTestId('export-error-icon')).toBeInTheDocument()
+})
diff --git a/assets/js/dashboard/stats/csv-export/csv-export.tsx b/assets/js/dashboard/stats/csv-export/csv-export.tsx
new file mode 100644
index 0000000000..05f52534f0
--- /dev/null
+++ b/assets/js/dashboard/stats/csv-export/csv-export.tsx
@@ -0,0 +1,96 @@
+import React from 'react'
+import { useSiteContext } from '../../site-context'
+import { popover } from '../../components/popover'
+import classNames from 'classnames'
+import { Spinner } from '../../components/icons'
+import {
+ ArrowDownTrayIcon,
+ ExclamationCircleIcon
+} from '@heroicons/react/24/outline'
+import { useDashboardStateContext } from '../../dashboard-state-context'
+import { useGraphIntervalContext } from '../graph/graph-interval-context'
+import { createCsvExportRequestBody } from './csv-export-body'
+import * as api from '../../api'
+import { DateRange } from '../../stats-query'
+import { Tooltip } from '../../util/tooltip'
+import { useBodyPortalRef } from '../breakdowns'
+
+export enum ExportStatus {
+ idle = 'idle',
+ exporting = 'exporting',
+ error = 'error'
+}
+
+export function CsvExportV2({
+ exportStatus,
+ setExportStatus
+}: {
+ exportStatus: ExportStatus
+ setExportStatus: (v: ExportStatus) => void
+}) {
+ const site = useSiteContext()
+ const { dashboardState } = useDashboardStateContext()
+ const { selectedInterval } = useGraphIntervalContext()
+
+ const canStartExport = exportStatus === ExportStatus.idle
+
+ const startExport = async () => {
+ if (!canStartExport) return
+ setExportStatus(ExportStatus.exporting)
+
+ try {
+ const body = createCsvExportRequestBody(dashboardState, selectedInterval)
+ const blob = await api.csvExport(site, body)
+ const url = URL.createObjectURL(blob)
+ const a = document.createElement('a')
+ a.href = url
+ a.download = constructFilename(site.domain, body.date_range)
+ document.body.appendChild(a)
+ a.click()
+ document.body.removeChild(a)
+ URL.revokeObjectURL(url)
+ setExportStatus(ExportStatus.idle)
+ } catch {
+ setExportStatus(ExportStatus.error)
+ }
+ }
+
+ return (
+ <button
+ onClick={startExport}
+ className={classNames(
+ popover.items.classNames.navigationLink,
+ popover.items.classNames.hoverLink,
+ !canStartExport && '!cursor-default'
+ )}
+ >
+ <span className="text-sm">Export stats</span>
+ {exportStatus === ExportStatus.exporting ? (
+ <Spinner className="animate-spin size-4 text-indigo-500" />
+ ) : exportStatus === ExportStatus.error ? (
+ <ExportFailedBubble />
+ ) : (
+ <ArrowDownTrayIcon className="size-4 stroke-2" />
+ )}
+ </button>
+ )
+}
+
+function ExportFailedBubble() {
+ const portalRef = useBodyPortalRef()
+ return (
+ <Tooltip info={'Export failed'} containerRef={portalRef}>
+ <ExclamationCircleIcon
+ data-testid="export-error-icon"
+ className="size-4 text-gray-500 dark:text-gray-400"
+ />
+ </Tooltip>
+ )
+}
+
+function constructFilename(domain: string, dateRange: DateRange) {
+ const dateRangeText = Array.isArray(dateRange)
+ ? `${dateRange[0]} to ${dateRange[1]}`
+ : dateRange
+ return `Plausible export ${domain} ${dateRangeText} .zip`
+}
diff --git a/assets/js/dashboard/stats/devices/details.tsx b/assets/js/dashboard/stats/devices/details.tsx
index cb01df4eae..506324c0e4 100644
--- a/assets/js/dashboard/stats/devices/details.tsx
+++ b/assets/js/dashboard/stats/devices/details.tsx
@@ -6,11 +6,7 @@ import {
hasConversionGoalFilter,
isRealTimeDashboard
} from '../../util/filters'
-import {
- chooseBreakdownMetricsByContext,
- defaultGetFilterInfo,
- getScreenFilterInfo
-} from '../breakdowns'
+import { defaultGetFilterInfo, getScreenFilterInfo } from '../breakdowns'
import {
BREAKDOWN_REPORTS,
BreakdownReportKey
@@ -45,15 +41,12 @@ export function DevicesDetails({
const isRevenueAvailable =
BUILD_EXTRA && revenueAvailable(dashboardState, site)
- const metrics = chooseBreakdownMetricsByContext(
- reportConfig.metricsByContext,
- {
- hasConversionGoalFilter: hasConversionGoalFilter(dashboardState),
- isRealtime: isRealTimeDashboard(dashboardState),
- isDetailed: true,
- isRevenueAvailable: isRevenueAvailable
- }
- )
+ const metrics = reportConfig.getMetrics({
+ hasConversionGoalFilter: hasConversionGoalFilter(dashboardState),
+ isRealtime: isRealTimeDashboard(dashboardState),
+ isDetailed: true,
+ isRevenueAvailable: isRevenueAvailable
+ })
const DimensionElement = {
[BreakdownReportKey.browsers]: BrowsersDimensionCell,
diff --git a/assets/js/dashboard/stats/devices/index.tsx b/assets/js/dashboard/stats/devices/index.tsx
index 7cc262c174..6c985079d6 100644
--- a/assets/js/dashboard/stats/devices/index.tsx
+++ b/assets/js/dashboard/stats/devices/index.tsx
@@ -24,11 +24,7 @@ import {
DimensionCellWithBarProps,
IndexBreakdown
} from '../reports/index-breakdown'
-import {
- chooseBreakdownMetricsByContext,
- defaultGetFilterInfo,
- getScreenFilterInfo
-} from '../breakdowns'
+import { defaultGetFilterInfo, getScreenFilterInfo } from '../breakdowns'
import { DashboardState } from '../../dashboard-state'
import { BrowserIcon, OsIcon, ScreenSizeIcon } from './icons'
@@ -53,15 +49,10 @@ export function Devices() {
[BreakdownReportKey.screenSizes]: ScreenSizesDimensionCell
}[reportKey]
- const metrics = chooseBreakdownMetricsByContext(
- reportConfig.metricsByContext,
- {
- isRealtime: isRealTimeDashboard(dashboardState),
- isDetailed: false,
- hasConversionGoalFilter: hasConversionGoalFilter(dashboardState),
- isRevenueAvailable: false
- }
- )
+ const metrics = reportConfig.getMetrics({
+ isRealtime: isRealTimeDashboard(dashboardState),
+ hasConversionGoalFilter: hasConversionGoalFilter(dashboardState)
+ })
function switchTab(tab: TabKey) {
storage.setItem(storageKey, tab)
diff --git a/assets/js/dashboard/stats/locations/details.tsx b/assets/js/dashboard/stats/locations/details.tsx
index 7d2353bf71..6ef1dc44b0 100644
--- a/assets/js/dashboard/stats/locations/details.tsx
+++ b/assets/js/dashboard/stats/locations/details.tsx
@@ -6,7 +6,6 @@ import {
hasConversionGoalFilter,
isRealTimeDashboard
} from '../../util/filters'
-import { chooseBreakdownMetricsByContext } from '../breakdowns'
import {
BREAKDOWN_REPORTS,
BreakdownReportKey
@@ -38,15 +37,12 @@ export function LocationsDetails({
const isRevenueAvailable =
BUILD_EXTRA && revenueAvailable(dashboardState, site)
- const metrics = chooseBreakdownMetricsByContext(
- reportConfig.metricsByContext,
- {
- hasConversionGoalFilter: hasConversionGoalFilter(dashboardState),
- isRealtime: isRealTimeDashboard(dashboardState),
- isDetailed: true,
- isRevenueAvailable: isRevenueAvailable
- }
- )
+ const metrics = reportConfig.getMetrics({
+ hasConversionGoalFilter: hasConversionGoalFilter(dashboardState),
+ isRealtime: isRealTimeDashboard(dashboardState),
+ isDetailed: true,
+ isRevenueAvailable: isRevenueAvailable
+ })
const DimensionElement = DIMENSION_ELEMENTS[reportKey]
diff --git a/assets/js/dashboard/stats/locations/index.tsx b/assets/js/dashboard/stats/locations/index.tsx
index a899dcf7cd..501ebf00db 100644
--- a/assets/js/dashboard/stats/locations/index.tsx
+++ b/assets/js/dashboard/stats/locations/index.tsx
@@ -26,7 +26,6 @@ import {
DimensionCellWithBarProps,
IndexBreakdown
} from '../reports/index-breakdown'
-import { chooseBreakdownMetricsByContext } from '../breakdowns'
import { FilterInfo } from '../../components/drilldown-link'
import { NonTimeDimension } from '../../stats-query'
import { FlagEmoji } from './flag-emoji'
@@ -124,15 +123,10 @@ export function Locations() {
tab === 'map' ? BreakdownReportKey.countries : tab
const reportConfig = BREAKDOWN_REPORTS[selectedListKey]
- const metrics = chooseBreakdownMetricsByContext(
- reportConfig.metricsByContext,
- {
- isRealtime: isRealTimeDashboard(dashboardState),
- isDetailed: false,
- hasConversionGoalFilter: hasConversionGoalFilter(dashboardState),
- isRevenueAvailable: false
- }
- )
+ const metrics = reportConfig.getMetrics({
+ isRealtime: isRealTimeDashboard(dashboardState),
+ hasConversionGoalFilter: hasConversionGoalFilter(dashboardState)
+ })
const moreLinkState = currentData
? currentData.results.length > 0
diff --git a/assets/js/dashboard/stats/modals/details-breakdown.test.tsx b/assets/js/dashboard/stats/modals/details-breakdown.test.tsx
index 6083bd6eb3..37dc53e256 100644
--- a/assets/js/dashboard/stats/modals/details-breakdown.test.tsx
+++ b/assets/js/dashboard/stats/modals/details-breakdown.test.tsx
@@ -17,8 +17,12 @@ const queryPath = `/api/stats/${domain}/query/`
const MOCK_RESPONSE_DELAY_MS = 50
const SMALL_LOADING_SPINNER_TEST_ID = 'small-loading-spinner'
-const PAGES_DETAILED_METRICS =
- BREAKDOWN_REPORTS.pages.metricsByContext.defaultDetailedMetrics
+const PAGES_DETAILED_METRICS = BREAKDOWN_REPORTS.pages.getMetrics({
+ isDetailed: true,
+ isRealtime: false,
+ hasConversionGoalFilter: false,
+ isRevenueAvailable: false
+})
function buildResponse(
results: { dimensions: string[]; metrics: number[] }[] = []
diff --git a/assets/js/dashboard/stats/pages/details.tsx b/assets/js/dashboard/stats/pages/details.tsx
index 0616043edf..0852e325fe 100644
--- a/assets/js/dashboard/stats/pages/details.tsx
+++ b/assets/js/dashboard/stats/pages/details.tsx
@@ -7,10 +7,7 @@ import {
hasEventFilters,
isRealTimeDashboard
} from '../../util/filters'
-import {
- chooseBreakdownMetricsByContext,
- defaultGetFilterInfo
-} from '../breakdowns'
+import { defaultGetFilterInfo } from '../breakdowns'
import {
BREAKDOWN_REPORTS,
BreakdownReportKey
@@ -38,17 +35,14 @@ export function PagesDetails({
const isRevenueAvailable =
BUILD_EXTRA && revenueAvailable(dashboardState, site)
- let metrics = chooseBreakdownMetricsByContext(reportConfig.metricsByContext, {
+ const metrics = reportConfig.getMetrics({
hasConversionGoalFilter: hasConversionGoalFilter(dashboardState),
isRealtime: isRealTimeDashboard(dashboardState),
isDetailed: true,
- isRevenueAvailable: isRevenueAvailable
+ isRevenueAvailable: isRevenueAvailable,
+ hasEventFilters: hasEventFilters(dashboardState)
})
- if (metrics.includes('exit_rate') && hasEventFilters(dashboardState)) {
- metrics = metrics.filter((m) => m !== 'exit_rate')
- }
-
return (
<Modal>
<DetailsBreakdown
diff --git a/assets/js/dashboard/stats/pages/index.tsx b/assets/js/dashboard/stats/pages/index.tsx
index 980a2f3949..e099f84554 100644
--- a/assets/js/dashboard/stats/pages/index.tsx
+++ b/assets/js/dashboard/stats/pages/index.tsx
@@ -23,10 +23,7 @@ import {
IndexBreakdown,
DimensionCellWithBarProps
} from '../reports/index-breakdown'
-import {
- chooseBreakdownMetricsByContext,
- defaultGetFilterInfo
-} from '../breakdowns'
+import { defaultGetFilterInfo } from '../breakdowns'
import { externalLinkForPage, trimURL } from '../../util/url'
import { IndexExternalLink } from './external-link'
@@ -44,15 +41,10 @@ export default function Pages() {
const reportKey = getReportKey(tab)
const reportConfig = BREAKDOWN_REPORTS[reportKey]
- const metrics = chooseBreakdownMetricsByContext(
- reportConfig.metricsByContext,
- {
- isRealtime: isRealTimeDashboard(dashboardState),
- isDetailed: false,
- hasConversionGoalFilter: hasConversionGoalFilter(dashboardState),
- isRevenueAvailable: false
- }
- )
+ const metrics = reportConfig.getMetrics({
+ isRealtime: isRealTimeDashboard(dashboardState),
+ hasConversionGoalFilter: hasConversionGoalFilter(dashboardState)
+ })
function switchTab(tab: TabKey) {
storage.setItem(storageKey, tab)
diff --git a/assets/js/dashboard/stats/reports/reports-config.ts b/assets/js/dashboard/stats/reports/reports-config.ts
index 3d37e6ad3b..d89ac4d9f0 100644
--- a/assets/js/dashboard/stats/reports/reports-config.ts
+++ b/assets/js/dashboard/stats/reports/reports-config.ts
@@ -1,24 +1,35 @@
import { ApiFilter, NonTimeDimension } from '../../stats-query'
import { Metric } from '../metrics'
+export type MetricContext = {
+ hasConversionGoalFilter: boolean
+ isRealtime?: boolean
+ isCsv?: boolean
+ isDetailed?: boolean
+ isRevenueAvailable?: boolean
+ hasEventFilters?: boolean
+}
+
export type MetricsByContext = {
realtimeMetrics: Metric[]
defaultIndexMetrics: Metric[]
defaultDetailedMetrics: Metric[]
+ defaultCsvMetrics: Metric[]
goalFilterIndexMetrics: Metric[]
goalFilterDetailedMetrics: Metric[]
+ goalFilterCsvMetrics: Metric[]
}
export type BreakdownReportConfig = {
dimensions: [NonTimeDimension, ...NonTimeDimension[]]
- metricsByContext: MetricsByContext
+ getMetrics: (context: MetricContext) => Metric[]
detailsTitle: string
detailsPath: string
dimensionLabel: string
alwaysOnFilters?: ApiFilter[]
}
-const COMMON_METRICS_BY_CONTEXT: MetricsByContext = {
+export const COMMON_BREAKDOWN_METRICS_BY_CONTEXT: MetricsByContext = {
realtimeMetrics: ['visitors', 'percentage'],
defaultIndexMetrics: ['visitors', 'percentage'],
defaultDetailedMetrics: [
@@ -27,12 +38,56 @@ const COMMON_METRICS_BY_CONTEXT: MetricsByContext = {
'bounce_rate',
'visit_duration'
],
+ defaultCsvMetrics: ['visitors', 'bounce_rate', 'visit_duration'],
goalFilterIndexMetrics: ['visitors', 'group_conversion_rate'],
goalFilterDetailedMetrics: [
'total_visitors',
'visitors',
'group_conversion_rate'
- ]
+ ],
+ goalFilterCsvMetrics: ['visitors', 'group_conversion_rate']
+}
+
+function chooseMetrics(mbc: MetricsByContext, ctx: MetricContext): Metric[] {
+ const {
+ isRealtime,
+ isCsv,
+ isDetailed,
+ hasConversionGoalFilter,
+ isRevenueAvailable
+ } = ctx
+ if (isCsv && hasConversionGoalFilter) {
+ return mbc.goalFilterCsvMetrics
+ }
+ if (isCsv) {
+ return mbc.defaultCsvMetrics
+ }
+ if (hasConversionGoalFilter && isDetailed && isRevenueAvailable) {
+ return [
+ ...mbc.goalFilterDetailedMetrics,
+ 'total_revenue',
+ 'average_revenue'
+ ]
+ }
+ if (hasConversionGoalFilter && isDetailed) {
+ return mbc.goalFilterDetailedMetrics
+ }
+ if (hasConversionGoalFilter) {
+ return mbc.goalFilterIndexMetrics
+ }
+ if (isRealtime) {
+ return mbc.realtimeMetrics
+ }
+ if (isDetailed) {
+ return mbc.defaultDetailedMetrics
+ }
+ return mbc.defaultIndexMetrics
+}
+
+function createGetMetricsFn(
+ mbc: MetricsByContext
+): (context: MetricContext) => Metric[] {
+ return (ctx) => chooseMetrics(mbc, ctx)
}
export enum BreakdownReportKey {
@@ -63,8 +118,8 @@ export const BREAKDOWN_REPORTS: Record<
> = {
[BreakdownReportKey.pages]: {
dimensions: ['event:page'],
- metricsByContext: {
- ...COMMON_METRICS_BY_CONTEXT,
+ getMetrics: createGetMetricsFn({
+ ...COMMON_BREAKDOWN_METRICS_BY_CONTEXT,
defaultDetailedMetrics: [
'visitors',
'percentage',
@@ -72,24 +127,32 @@ export const BREAKDOWN_REPORTS: Record<
'bounce_rate',
'time_on_page',
'scroll_depth'
+ ],
+ defaultCsvMetrics: [
+ 'visitors',
+ 'pageviews',
+ 'bounce_rate',
+ 'time_on_page',
+ 'scroll_depth'
]
- },
+ }),
detailsTitle: 'Top pages',
detailsPath: 'pages',
dimensionLabel: 'Page'
},
[BreakdownReportKey.entryPages]: {
dimensions: ['visit:entry_page'],
- metricsByContext: {
- ...COMMON_METRICS_BY_CONTEXT,
+ getMetrics: createGetMetricsFn({
+ ...COMMON_BREAKDOWN_METRICS_BY_CONTEXT,
defaultDetailedMetrics: [
'visitors',
'percentage',
'visits',
'bounce_rate',
'visit_duration'
- ]
- },
+ ],
+ defaultCsvMetrics: ['visitors', 'visits', 'bounce_rate', 'visit_duration']
+ }),
detailsTitle: 'Entry pages',
detailsPath: 'entry-pages',
dimensionLabel: 'Entry page',
@@ -97,9 +160,22 @@ export const BREAKDOWN_REPORTS: Record<
},
[BreakdownReportKey.exitPages]: {
dimensions: ['visit:exit_page'],
- metricsByContext: {
- ...COMMON_METRICS_BY_CONTEXT,
- defaultDetailedMetrics: ['visitors', 'percentage', 'visits', 'exit_rate']
+ getMetrics: (ctx) => {
+ const base = chooseMetrics(
+ {
+ ...COMMON_BREAKDOWN_METRICS_BY_CONTEXT,
+ defaultDetailedMetrics: [
+ 'visitors',
+ 'percentage',
+ 'visits',
+ 'exit_rate'
+ ],
+ defaultCsvMetrics: ['visitors', 'visits', 'exit_rate']
+ },
+ ctx
+ )
+
+ return ctx.hasEventFilters ? base.filter((m) => m !== 'exit_rate') : base
},
detailsTitle: 'Exit pages',
detailsPath: 'exit-pages',
@@ -108,63 +184,78 @@ export const BREAKDOWN_REPORTS: Record<
},
[BreakdownReportKey.browsers]: {
dimensions: ['visit:browser'],
- metricsByContext: COMMON_METRICS_BY_CONTEXT,
+ getMetrics: createGetMetricsFn({
+ ...COMMON_BREAKDOWN_METRICS_BY_CONTEXT,
+ defaultCsvMetrics: ['visitors']
+ }),
detailsTitle: 'Browsers',
detailsPath: 'browsers',
dimensionLabel: 'Browser'
},
[BreakdownReportKey.browserVersions]: {
dimensions: ['visit:browser_version', 'visit:browser'],
- metricsByContext: COMMON_METRICS_BY_CONTEXT,
+ getMetrics: createGetMetricsFn({
+ ...COMMON_BREAKDOWN_METRICS_BY_CONTEXT,
+ defaultCsvMetrics: ['visitors']
+ }),
detailsTitle: 'Browser versions',
detailsPath: 'browser-versions',
dimensionLabel: 'Browser version'
},
[BreakdownReportKey.operatingSystems]: {
dimensions: ['visit:os'],
- metricsByContext: COMMON_METRICS_BY_CONTEXT,
+ getMetrics: createGetMetricsFn({
+ ...COMMON_BREAKDOWN_METRICS_BY_CONTEXT,
+ defaultCsvMetrics: ['visitors']
+ }),
detailsTitle: 'Operating systems',
detailsPath: 'operating-systems',
dimensionLabel: 'Operating system'
},
[BreakdownReportKey.operatingSystemVersions]: {
dimensions: ['visit:os_version', 'visit:os'],
- metricsByContext: COMMON_METRICS_BY_CONTEXT,
+ getMetrics: createGetMetricsFn({
+ ...COMMON_BREAKDOWN_METRICS_BY_CONTEXT,
+ defaultCsvMetrics: ['visitors']
+ }),
detailsTitle: 'Operating system versions',
detailsPath: 'operating-system-versions',
dimensionLabel: 'Operating system version'
},
[BreakdownReportKey.screenSizes]: {
dimensions: ['visit:device'],
- metricsByContext: COMMON_METRICS_BY_CONTEXT,
+ getMetrics: createGetMetricsFn({
+ ...COMMON_BREAKDOWN_METRICS_BY_CONTEXT,
+ defaultCsvMetrics: ['visitors']
+ }),
detailsTitle: 'Devices',
detailsPath: 'screen-sizes',
dimensionLabel: 'Device'
},
[BreakdownReportKey.channels]: {
dimensions: ['visit:channel'],
- metricsByContext: COMMON_METRICS_BY_CONTEXT,
+ getMetrics: createGetMetricsFn(COMMON_BREAKDOWN_METRICS_BY_CONTEXT),
detailsTitle: 'Top acquisition channels',
detailsPath: 'channels',
dimensionLabel: 'Channel'
},
[BreakdownReportKey.sources]: {
dimensions: ['visit:source'],
- metricsByContext: COMMON_METRICS_BY_CONTEXT,
+ getMetrics: createGetMetricsFn(COMMON_BREAKDOWN_METRICS_BY_CONTEXT),
detailsTitle: 'Top sources',
detailsPath: 'sources',
dimensionLabel: 'Source'
},
[BreakdownReportKey.referrers]: {
dimensions: ['visit:referrer'],
- metricsByContext: COMMON_METRICS_BY_CONTEXT,
+ getMetrics: createGetMetricsFn(COMMON_BREAKDOWN_METRICS_BY_CONTEXT),
detailsTitle: 'Referrer drilldown',
detailsPath: 'referrers/:referrer',
dimensionLabel: 'Referrer'
},
[BreakdownReportKey.utmMediums]: {
dimensions: ['visit:utm_medium'],
- metricsByContext: COMMON_METRICS_BY_CONTEXT,
+ getMetrics: createGetMetricsFn(COMMON_BREAKDOWN_METRICS_BY_CONTEXT),
detailsTitle: 'UTM mediums',
detailsPath: 'utm_mediums',
dimensionLabel: 'UTM medium',
@@ -172,7 +263,7 @@ export const BREAKDOWN_REPORTS: Record<
},
[BreakdownReportKey.utmSources]: {
dimensions: ['visit:utm_source'],
- metricsByContext: COMMON_METRICS_BY_CONTEXT,
+ getMetrics: createGetMetricsFn(COMMON_BREAKDOWN_METRICS_BY_CONTEXT),
detailsTitle: 'UTM sources',
detailsPath: 'utm_sources',
dimensionLabel: 'UTM source',
@@ -180,7 +271,7 @@ export const BREAKDOWN_REPORTS: Record<
},
[BreakdownReportKey.utmCampaigns]: {
dimensions: ['visit:utm_campaign'],
- metricsByContext: COMMON_METRICS_BY_CONTEXT,
+ getMetrics: createGetMetricsFn(COMMON_BREAKDOWN_METRICS_BY_CONTEXT),
detailsTitle: 'UTM campaigns',
detailsPath: 'utm_campaigns',
dimensionLabel: 'UTM campaign',
@@ -188,7 +279,7 @@ export const BREAKDOWN_REPORTS: Record<
},
[BreakdownReportKey.utmContents]: {
dimensions: ['visit:utm_content'],
- metricsByContext: COMMON_METRICS_BY_CONTEXT,
+ getMetrics: createGetMetricsFn(COMMON_BREAKDOWN_METRICS_BY_CONTEXT),
detailsTitle: 'UTM contents',
detailsPath: 'utm_contents',
dimensionLabel: 'UTM content',
@@ -196,7 +287,7 @@ export const BREAKDOWN_REPORTS: Record<
},
[BreakdownReportKey.utmTerms]: {
dimensions: ['visit:utm_term'],
- metricsByContext: COMMON_METRICS_BY_CONTEXT,
+ getMetrics: createGetMetricsFn(COMMON_BREAKDOWN_METRICS_BY_CONTEXT),
detailsTitle: 'UTM terms',
detailsPath: 'utm_terms',
dimensionLabel: 'UTM term',
@@ -204,10 +295,11 @@ export const BREAKDOWN_REPORTS: Record<
},
[BreakdownReportKey.countries]: {
dimensions: ['visit:country_name', 'visit:country'],
- metricsByContext: {
- ...COMMON_METRICS_BY_CONTEXT,
- defaultDetailedMetrics: ['visitors', 'percentage']
- },
+ getMetrics: createGetMetricsFn({
+ ...COMMON_BREAKDOWN_METRICS_BY_CONTEXT,
+ defaultDetailedMetrics: ['visitors', 'percentage'],
+ defaultCsvMetrics: ['visitors']
+ }),
detailsTitle: 'Top countries',
detailsPath: 'countries',
dimensionLabel: 'Country',
@@ -216,10 +308,11 @@ export const BREAKDOWN_REPORTS: Record<
[BreakdownReportKey.regions]: {
// the 3rd dimension "visit:country" is needed to render the country flag
dimensions: ['visit:region_name', 'visit:region', 'visit:country'],
- metricsByContext: {
- ...COMMON_METRICS_BY_CONTEXT,
- defaultDetailedMetrics: ['visitors', 'percentage']
- },
+ getMetrics: createGetMetricsFn({
+ ...COMMON_BREAKDOWN_METRICS_BY_CONTEXT,
+ defaultDetailedMetrics: ['visitors', 'percentage'],
+ defaultCsvMetrics: ['visitors']
+ }),
detailsTitle: 'Top regions',
detailsPath: 'regions',
dimensionLabel: 'Region',
@@ -228,10 +321,11 @@ export const BREAKDOWN_REPORTS: Record<
[BreakdownReportKey.cities]: {
// the 3rd dimension "visit:country" is needed to render the country flag
dimensions: ['visit:city_name', 'visit:city', 'visit:country'],
- metricsByContext: {
- ...COMMON_METRICS_BY_CONTEXT,
- defaultDetailedMetrics: ['visitors', 'percentage']
- },
+ getMetrics: createGetMetricsFn({
+ ...COMMON_BREAKDOWN_METRICS_BY_CONTEXT,
+ defaultDetailedMetrics: ['visitors', 'percentage'],
+ defaultCsvMetrics: ['visitors']
+ }),
detailsTitle: 'Top cities',
detailsPath: 'cities',
dimensionLabel: 'City',
diff --git a/assets/js/dashboard/stats/sources/details.tsx b/assets/js/dashboard/stats/sources/details.tsx
index 25447b485e..741e9d3cdb 100644
--- a/assets/js/dashboard/stats/sources/details.tsx
+++ b/assets/js/dashboard/stats/sources/details.tsx
@@ -6,11 +6,7 @@ import {
hasConversionGoalFilter,
isRealTimeDashboard
} from '../../util/filters'
-import {
- chooseBreakdownMetricsByContext,
- defaultGetFilterInfo,
- getReferrerUrlFilterInfo
-} from '../breakdowns'
+import { defaultGetFilterInfo, getReferrerUrlFilterInfo } from '../breakdowns'
import {
BREAKDOWN_REPORTS,
BreakdownReportKey
@@ -45,15 +41,12 @@ export function SourcesDetails({ reportKey }: { reportKey: SourcesReportKey }) {
const isRevenueAvailable =
BUILD_EXTRA && revenueAvailable(dashboardState, site)
- const metrics = chooseBreakdownMetricsByContext(
- reportConfig.metricsByContext,
- {
- hasConversionGoalFilter: hasConversionGoalFilter(dashboardState),
- isRealtime: isRealTimeDashboard(dashboardState),
- isDetailed: true,
- isRevenueAvailable: isRevenueAvailable
- }
- )
+ const metrics = reportConfig.getMetrics({
+ hasConversionGoalFilter: hasConversionGoalFilter(dashboardState),
+ isRealtime: isRealTimeDashboard(dashboardState),
+ isDetailed: true,
+ isRevenueAvailable: isRevenueAvailable
+ })
const DimensionElement =
reportKey === BreakdownReportKey.sources
diff --git a/assets/js/dashboard/stats/sources/index.tsx b/assets/js/dashboard/stats/sources/index.tsx
index b50c34baf3..087e0fe546 100644
--- a/assets/js/dashboard/stats/sources/index.tsx
+++ b/assets/js/dashboard/stats/sources/index.tsx
@@ -28,10 +28,7 @@ import {
DimensionCellWithBarProps,
IndexBreakdown
} from '../reports/index-breakdown'
-import {
- chooseBreakdownMetricsByContext,
- defaultGetFilterInfo
-} from '../breakdowns'
+import { defaultGetFilterInfo } from '../breakdowns'
import ImportedWarningBubble from '../imported-warning-bubble'
import { IndexExternalLink } from '../pages/external-link'
import { SearchTerms } from './search-terms'
@@ -193,15 +190,10 @@ export default function Sources() {
const reportConfig = BREAKDOWN_REPORTS[currentReportKey]
- const metrics = chooseBreakdownMetricsByContext(
- reportConfig.metricsByContext,
- {
- isRealtime: isRealTimeDashboard(dashboardState),
- isDetailed: false,
- hasConversionGoalFilter: hasConversionGoalFilter(dashboardState),
- isRevenueAvailable: false
- }
- )
+ const metrics = reportConfig.getMetrics({
+ isRealtime: isRealTimeDashboard(dashboardState),
+ hasConversionGoalFilter: hasConversionGoalFilter(dashboardState)
+ })
const DimensionElement = {
[BreakdownReportKey.channels]: ChannelsDimensionCell,
diff --git a/assets/test-utils/app-context-providers.tsx b/assets/test-utils/app-context-providers.tsx
index e0fcd5588b..f0b2f59270 100644
--- a/assets/test-utils/app-context-providers.tsx
+++ b/assets/test-utils/app-context-providers.tsx
@@ -47,7 +47,7 @@ export const DEFAULT_SITE: PlausibleSite = {
embedded: false,
background: '',
isDbip: false,
- flags: {},
+ flags: { dashboard_csv_export_v2: false },
shared: false,
isConsolidatedView: false
}
diff --git a/e2e/package-lock.json b/e2e/package-lock.json
index 4cbd79a532..e8d62c33f0 100644
--- a/e2e/package-lock.json
+++ b/e2e/package-lock.json
@@ -15,6 +15,7 @@
"eslint": "^9.39.2",
"eslint-config-prettier": "^10.1.8",
"eslint-plugin-playwright": "^2.5.1",
+ "fflate": "^0.8.0",
"prettier": "^3.8.1",
"typescript": "^5.9.3",
"typescript-eslint": "^8.55.0"
@@ -1007,6 +1008,13 @@
}
}
},
+ "node_modules/fflate": {
+ "version": "0.8.3",
+ "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.3.tgz",
+ "integrity": "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==",
+ "dev": true,
+ "license": "MIT"
+ },
"node_modules/file-entry-cache": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz",
diff --git a/e2e/package.json b/e2e/package.json
index c02121e332..04b77d6e55 100644
--- a/e2e/package.json
+++ b/e2e/package.json
@@ -10,6 +10,7 @@
"license": "MIT",
"devDependencies": {
"@eslint/js": "^9.39.2",
+ "fflate": "^0.8.0",
"@js-joda/core": "^5.7.0",
"@js-joda/locale_en-us": "^4.15.3",
"@playwright/test": "^1.58.0",
diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts
index d9e9e062ae..84408c4989 100644
--- a/e2e/playwright.config.ts
+++ b/e2e/playwright.config.ts
@@ -23,6 +23,8 @@ export default defineConfig({
/* Base URL to use in actions like `await page.goto('')`. */
baseURL,
+ // launchOptions: {slowMo: 500},
+
/* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
trace: 'on-first-retry'
},
diff --git a/e2e/tests/dashboard/csv-export.spec.ts b/e2e/tests/dashboard/csv-export.spec.ts
new file mode 100644
index 0000000000..dc87e89344
--- /dev/null
+++ b/e2e/tests/dashboard/csv-export.spec.ts
@@ -0,0 +1,369 @@
+import { test, expect } from '@playwright/test'
+import type { Page } from '@playwright/test'
+import { unzipSync } from 'fflate'
+import * as fs from 'fs'
+import {
+ setupSite,
+ populateStats,
+ enableDashboardCsvExportV2,
+ addGoal,
+ addCustomProp,
+ StatsEntry
+} from '../fixtures'
+
+const EXPECTED_HEADERS = {
+ defaultVisitorsCsv: [
+ 'date',
+ 'visitors',
+ 'pageviews',
+ 'visits',
+ 'views_per_visit',
+ 'bounce_rate',
+ 'visit_duration'
+ ],
+ pageFilteredVisitorsCsv: [
+ 'date',
+ 'visitors',
+ 'pageviews',
+ 'visits',
+ 'bounce_rate',
+ 'time_on_page',
+ 'scroll_depth'
+ ],
+ goalFilteredVisitorsCsv: [
+ 'date',
+ 'unique_conversions',
+ 'total_conversions',
+ 'conversion_rate'
+ ],
+ pagesBreakdown: [
+ 'name',
+ 'visitors',
+ 'pageviews',
+ 'bounce_rate',
+ 'time_on_page',
+ 'scroll_depth'
+ ],
+ entryPagesBreakdown: [
+ 'name',
+ 'unique_entrances',
+ 'total_entrances',
+ 'bounce_rate',
+ 'visit_duration'
+ ],
+ exitPagesBreakdown: ['name', 'unique_exits', 'total_exits', 'exit_rate'],
+ exitPagesNoExitRate: ['name', 'unique_exits', 'total_exits'],
+ commonBreakdown: ['name', 'visitors', 'bounce_rate', 'visit_duration'],
+ commonGoalFilteredBreakdown: ['name', 'conversions', 'conversion_rate'],
+ nameAndVisitors: ['name', 'visitors'],
+ nameVersionAndVisitors: ['name', 'version', 'visitors'],
+ nameVersionGoalFiltered: ['name', 'version', 'conversions', 'conversion_rate']
+}
+
+const UTM_AND_SOURCE_REPORTS = [
+ 'sources.csv',
+ 'channels.csv',
+ 'referrers.csv',
+ 'utm_mediums.csv',
+ 'utm_sources.csv',
+ 'utm_campaigns.csv',
+ 'utm_contents.csv',
+ 'utm_terms.csv'
+]
+
+const SIMPLE_LOCATION_AND_DEVICE_REPORTS = [
+ 'countries.csv',
+ 'regions.csv',
+ 'cities.csv',
+ 'browsers.csv',
+ 'operating_systems.csv',
+ 'devices.csv'
+]
+
+const VERSION_REPORTS = [
+ 'browser_versions.csv',
+ 'operating_system_versions.csv'
+]
+
+function parseAllCsvs(
+ files: Record<string, string>
+): Record<string, string[][]> {
+ return Object.fromEntries(
+ Object.entries(files).map(([filename, content]) => [
+ filename,
+ content
+ .split(/\r?\n/)
+ .filter((row) => !!row.length)
+ .map((line) => line.split(','))
+ ])
+ )
+}
+
+function getCsv(
+ csvs: Record<string, string[][]>,
+ filename: string
+): string[][] {
+ const rows = csvs[filename]
+ if (rows === undefined) throw new Error(`Expected ${filename} in CSV export`)
+ return rows
+}
+
+test('csv export column headers match expected metrics for each report', async ({
+ page,
+ request
+}) => {
+ const { domain } = await setupSite({ page, request })
+ await enableDashboardCsvExportV2({ request, domain })
+ await addCustomProp({ page, domain, name: 'author' })
+
+ await populateStats({
+ request,
+ domain,
+ events: [{ name: 'pageview', timestamp: '2021-01-01 12:00:00' }]
+ })
+
+ await test.step('without any filters', async () => {
+ await page.goto(`/${domain}`, {
+ waitUntil: 'commit'
+ })
+
+ const csvs = parseAllCsvs(await triggerExportAndAwaitDownload(page))
+
+ expect(getCsv(csvs, 'visitors.csv')[0]).toEqual(
+ EXPECTED_HEADERS.defaultVisitorsCsv
+ )
+ expect(getCsv(csvs, 'pages.csv')[0]).toEqual(
+ EXPECTED_HEADERS.pagesBreakdown
+ )
+ expect(getCsv(csvs, 'entry_pages.csv')[0]).toEqual(
+ EXPECTED_HEADERS.entryPagesBreakdown
+ )
+ expect(getCsv(csvs, 'exit_pages.csv')[0]).toEqual(
+ EXPECTED_HEADERS.exitPagesBreakdown
+ )
+
+ for (const filename of UTM_AND_SOURCE_REPORTS) {
+ expect(getCsv(csvs, filename)[0]).toEqual(
+ EXPECTED_HEADERS.commonBreakdown
+ )
+ }
+ for (const filename of SIMPLE_LOCATION_AND_DEVICE_REPORTS) {
+ expect(getCsv(csvs, filename)[0]).toEqual(
+ EXPECTED_HEADERS.nameAndVisitors
+ )
+ }
+ for (const filename of VERSION_REPORTS) {
+ expect(getCsv(csvs, filename)[0]).toEqual(
+ EXPECTED_HEADERS.nameVersionAndVisitors
+ )
+ }
+ })
+
+ await test.step('with a page filter', async () => {
+ await page.goto(`/${domain}?f=is,page,/blog`, {
+ waitUntil: 'commit'
+ })
+
+ const csvs = parseAllCsvs(await triggerExportAndAwaitDownload(page))
+
+ expect(getCsv(csvs, 'visitors.csv')[0]).toEqual(
+ EXPECTED_HEADERS.pageFilteredVisitorsCsv
+ )
+ expect(getCsv(csvs, 'pages.csv')[0]).toEqual(
+ EXPECTED_HEADERS.pagesBreakdown
+ )
+ expect(getCsv(csvs, 'entry_pages.csv')[0]).toEqual(
+ EXPECTED_HEADERS.entryPagesBreakdown
+ )
+ expect(getCsv(csvs, 'exit_pages.csv')[0]).toEqual(
+ EXPECTED_HEADERS.exitPagesNoExitRate
+ )
+
+ for (const filename of UTM_AND_SOURCE_REPORTS) {
+ expect(getCsv(csvs, filename)[0]).toEqual(
+ EXPECTED_HEADERS.commonBreakdown
+ )
+ }
+ for (const filename of SIMPLE_LOCATION_AND_DEVICE_REPORTS) {
+ expect(getCsv(csvs, filename)[0]).toEqual(
+ EXPECTED_HEADERS.nameAndVisitors
+ )
+ }
+ for (const filename of VERSION_REPORTS) {
+ expect(getCsv(csvs, filename)[0]).toEqual(
+ EXPECTED_HEADERS.nameVersionAndVisitors
+ )
+ }
+ })
+
+ await test.step('with a goal filter', async () => {
+ await addGoal({ request, domain, params: { event_name: 'Signup' } })
+
+ await page.goto(`/${domain}?f=is,goal,Signup`, {
+ waitUntil: 'commit'
+ })
+
+ const csvs = parseAllCsvs(await triggerExportAndAwaitDownload(page))
+
+ expect(getCsv(csvs, 'visitors.csv')[0]).toEqual(
+ EXPECTED_HEADERS.goalFilteredVisitorsCsv
+ )
+
+ for (const filename of [
+ 'pages.csv',
+ 'entry_pages.csv',
+ 'exit_pages.csv',
+ ...UTM_AND_SOURCE_REPORTS,
+ ...SIMPLE_LOCATION_AND_DEVICE_REPORTS
+ ]) {
+ expect(getCsv(csvs, filename)[0]).toEqual(
+ EXPECTED_HEADERS.commonGoalFilteredBreakdown
+ )
+ }
+ for (const filename of VERSION_REPORTS) {
+ expect(getCsv(csvs, filename)[0]).toEqual(
+ EXPECTED_HEADERS.nameVersionGoalFiltered
+ )
+ }
+ })
+
+ await test.step('with a custom prop filter', async () => {
+ await page.goto(`/${domain}?f=is,props:author,john`, {
+ waitUntil: 'commit'
+ })
+
+ const csvs = parseAllCsvs(await triggerExportAndAwaitDownload(page))
+
+ expect(getCsv(csvs, 'visitors.csv')[0]).toEqual(
+ EXPECTED_HEADERS.defaultVisitorsCsv
+ )
+ expect(getCsv(csvs, 'pages.csv')[0]).toEqual(
+ EXPECTED_HEADERS.pagesBreakdown
+ )
+ expect(getCsv(csvs, 'entry_pages.csv')[0]).toEqual(
+ EXPECTED_HEADERS.entryPagesBreakdown
+ )
+ expect(getCsv(csvs, 'exit_pages.csv')[0]).toEqual(
+ EXPECTED_HEADERS.exitPagesNoExitRate
+ )
+
+ for (const filename of UTM_AND_SOURCE_REPORTS) {
+ expect(getCsv(csvs, filename)[0]).toEqual(
+ EXPECTED_HEADERS.commonBreakdown
+ )
+ }
+ for (const filename of SIMPLE_LOCATION_AND_DEVICE_REPORTS) {
+ expect(getCsv(csvs, filename)[0]).toEqual(
+ EXPECTED_HEADERS.nameAndVisitors
+ )
+ }
+ for (const filename of VERSION_REPORTS) {
+ expect(getCsv(csvs, filename)[0]).toEqual(
+ EXPECTED_HEADERS.nameVersionAndVisitors
+ )
+ }
+ })
+})
+
+test('filters out empty visit:* dimension values', async ({
+ page,
+ request
+}) => {
+ const { domain } = await setupSite({ page, request })
+ await enableDashboardCsvExportV2({ request, domain })
+
+ const base: StatsEntry = {
+ name: 'pageview',
+ pathname: '/',
+ timestamp: '2021-01-01 12:00:00',
+ utm_medium: 't',
+ utm_source: 't',
+ utm_campaign: 't',
+ utm_content: 't',
+ utm_term: 't',
+ country_code: 'EE',
+ subdivision1_code: 'EE-37',
+ city_geoname_id: 588_409
+ }
+
+ await populateStats({
+ request,
+ domain,
+ events: [
+ // session with empty entry page — excluded from entry_pages.csv
+ { ...base, user_id: 1, pathname: '' },
+ { ...base, user_id: 1, pathname: '/', timestamp: '2021-01-01 12:01:00' },
+ // session with empty exit page — excluded from exit_pages.csv
+ { ...base, user_id: 2, pathname: '/' },
+ { ...base, user_id: 2, pathname: '', timestamp: '2021-01-01 12:01:00' },
+ // one bad utm_* per session — each excluded from its own report
+ { ...base, user_id: 3, utm_medium: '' },
+ { ...base, user_id: 4, utm_source: '' },
+ { ...base, user_id: 5, utm_campaign: '' },
+ { ...base, user_id: 6, utm_content: '' },
+ { ...base, user_id: 7, utm_term: '' },
+ // one bad location per session — each excluded from its own report
+ { ...base, user_id: 8, country_code: 'ZZ' },
+ { ...base, user_id: 9, subdivision1_code: '' },
+ { ...base, user_id: 10, city_geoname_id: 0 }
+ ]
+ })
+
+ await page.goto(`/${domain}?period=day&date=2021-01-01`, {
+ waitUntil: 'commit'
+ })
+
+ const csvs = parseAllCsvs(await triggerExportAndAwaitDownload(page))
+
+ // assert that each
+ for (const filename of [
+ 'entry_pages.csv',
+ 'exit_pages.csv',
+ 'utm_mediums.csv',
+ 'utm_sources.csv',
+ 'utm_campaigns.csv',
+ 'utm_contents.csv',
+ 'utm_terms.csv',
+ 'countries.csv',
+ 'regions.csv',
+ 'cities.csv'
+ ]) {
+ await test.step(filename, async () => {
+ expect(getCsv(csvs, filename).slice(1)).toHaveLength(1)
+ })
+ }
+})
+
+async function triggerExportAndAwaitDownload(
+ page: Page
+): Promise<Record<string, string>> {
+ await page.getByTestId('dashboard-options-menu').click()
+ await page.getByText('Export stats').waitFor({ state: 'visible' })
+
+ const [result] = await Promise.all([
+ Promise.race([
+ page
+ .waitForEvent('download')
+ .then((d) => ({ ok: true as const, download: d })),
+ page
+ .waitForEvent('pageerror')
+ .then((e) => ({ ok: false as const, error: e }))
+ ]),
+ page.getByText('Export stats').click()
+ ])
+
+ if (!result.ok) throw result.error
+ const { download } = result
+
+ const zipPath = await download.path()
+ if (!zipPath) throw new Error('Download path unavailable')
+
+ const files = unzipSync(new Uint8Array(fs.readFileSync(zipPath)))
+
+ return Object.fromEntries(
+ Object.entries(files).map(([filename, data]) => [
+ filename,
+ new TextDecoder().decode(data)
+ ])
+ )
+}
diff --git a/e2e/tests/fixtures.ts b/e2e/tests/fixtures.ts
index 0d15f21b93..7537118e71 100644
--- a/e2e/tests/fixtures.ts
+++ b/e2e/tests/fixtures.ts
@@ -477,6 +477,31 @@ export async function addAllCustomProps({
)
}
+export async function enableDashboardCsvExportV2({
+ request,
+ domain
+}: {
+ request: APIRequestContext
+ domain: string
+}) {
+ const response = await request.post(
+ '/e2e-tests/enable-dashboard-csv-export-v2',
+ {
+ headers: {
+ 'Content-Type': 'application/json',
+ Accept: 'application/json'
+ },
+ data: { domain }
+ }
+ )
+ if (!response.ok()) {
+ const body = await response.text()
+ throw new Error(
+ `enableDashboardCsvExportV2 failed (${response.status()}): ${body}`
+ )
+ }
+}
+
export async function addFunnel({
request,
domain,
diff --git a/lib/plausible/stats/dashboard/csv_export.ex b/lib/plausible/stats/dashboard/csv_export.ex
new file mode 100644
index 0000000000..af78a18e6d
--- /dev/null
+++ b/lib/plausible/stats/dashboard/csv_export.ex
@@ -0,0 +1,214 @@
+defmodule Plausible.Stats.Dashboard.CsvExport do
+ @moduledoc false
+
+ alias Plausible.Stats.Dashboard.QueryParser
+ alias Plausible.Stats.{Time, ParsedQueryParams, Query, QueryBuilder, QueryRunner, QueryResult}
+
+ @csv_filenames [
+ "visitors.csv",
+ "pages.csv",
+ "entry_pages.csv",
+ "exit_pages.csv",
+ "browsers.csv",
+ "browser_versions.csv",
+ "operating_systems.csv",
+ "operating_system_versions.csv",
+ "devices.csv",
+ "channels.csv",
+ "sources.csv",
+ "referrers.csv",
+ "utm_mediums.csv",
+ "utm_sources.csv",
+ "utm_campaigns.csv",
+ "utm_contents.csv",
+ "utm_terms.csv",
+ "countries.csv",
+ "regions.csv",
+ "cities.csv"
+ ]
+
+ def get_csvs(site, params, debug_metadata) do
+ with {:ok, queries_by_filename} <- create_queries_by_filename(site, params, debug_metadata) do
+ csv_values =
+ queries_by_filename
+ |> Enum.map(fn {filename, query} ->
+ fn -> run_query_into_csv(site, filename, query) end
+ end)
+ |> Plausible.ClickhouseRepo.parallel_tasks()
+
+ csvs =
+ queries_by_filename
+ |> Enum.map(fn {filename, _query} -> filename end)
+ |> Enum.zip(csv_values)
+ |> Enum.map(fn {k, v} -> {String.to_charlist(k), IO.iodata_to_binary(v)} end)
+
+ {:ok, csvs}
+ end
+ end
+
+ defp create_queries_by_filename(site, params, debug_metadata) do
+ requested = Map.keys(params["reports"] || %{}) |> Enum.filter(&(&1 in @csv_filenames))
+
+ Enum.reduce_while(requested, {:ok, []}, fn filename, acc ->
+ case parse_csv_query(site, filename, params, debug_metadata) do
+ {:ok, query} -> {:cont, {:ok, elem(acc, 1) ++ [{filename, query}]}}
+ error_tuple -> {:halt, error_tuple}
+ end
+ end)
+ end
+
+ defp parse_csv_query(site, filename, params, debug_metadata) do
+ params = construct_raw_query_params(filename, params)
+
+ with {:ok, %ParsedQueryParams{} = params} <- QueryParser.parse(params) do
+ QueryBuilder.build(site, params, debug_metadata)
+ end
+ end
+
+ defp construct_raw_query_params(filename, params) do
+ report_params = params["reports"][filename]
+
+ filters =
+ case report_params["always_on_filters"] do
+ always_on when is_list(always_on) -> params["filters"] ++ always_on
+ _ -> params["filters"]
+ end
+
+ %{
+ "dimensions" => report_params["dimensions"],
+ "metrics" => report_params["metrics"],
+ "date_range" => params["date_range"],
+ "relative_date" => params["relative_date"],
+ "filters" => filters,
+ "include" => params["include"]
+ }
+ end
+
+ defp run_query_into_csv(site, "visitors.csv", query) do
+ query =
+ query
+ |> Query.set(order_by: [{Time.time_dimension(query), :asc}])
+ |> Query.set_include(:time_labels, true)
+ |> Query.set_include(:empty_metrics, true)
+
+ QueryRunner.run(site, query) |> timeseries_query_result_to_csv()
+ end
+
+ defp run_query_into_csv(site, filename, query) do
+ # The export is limited to 300 entries for other reports and 100 entries for
+ # pages because bigger result sets start causing failures. Since we request
+ # data like time on page or bounce_rate for pages in a separate query using
+ # the IN filter, it causes the requests to balloon in payload size.
+ limit = if filename in ["pages.csv", "exit_pages.csv"], do: 100, else: 300
+ order_by = [{:visitors, :desc} | Enum.map(query.dimensions, &{&1, :asc})]
+
+ query =
+ query
+ |> Query.set(order_by: order_by)
+ |> Query.set(pagination: %{limit: limit, offset: 0})
+
+ QueryRunner.run(site, query) |> query_result_to_csv()
+ end
+
+ defp query_result_to_csv(%QueryResult{results: results, query: query}) do
+ results
+ |> Enum.reduce([csv_first_row(query)], fn row, acc ->
+ acc ++ [csv_dimension_values(query[:dimensions], row) ++ row.metrics]
+ end)
+ |> NimbleCSV.RFC4180.dump_to_iodata()
+ end
+
+ defp timeseries_query_result_to_csv(%QueryResult{results: results, query: query, meta: meta}) do
+ meta[:time_labels]
+ |> Enum.reduce([csv_first_row(query)], fn timelabel, acc ->
+ if result_item = find_result_item_by_timelabel(results, timelabel) do
+ acc ++ [[timelabel | result_item.metrics]]
+ else
+ acc ++ [[timelabel | meta[:empty_metrics]]]
+ end
+ end)
+ |> NimbleCSV.RFC4180.dump_to_iodata()
+ end
+
+ defp find_result_item_by_timelabel(results, timelabel) do
+ results
+ |> Map.new(fn entry -> {Enum.at(entry.dimensions, 0), entry} end)
+ |> Map.get(timelabel)
+ end
+
+ defp csv_first_row(query) do
+ csv_dimension_labels(query[:dimensions]) ++ csv_metric_labels(query)
+ end
+
+ defp csv_dimension_labels(["time:" <> _]) do
+ [:date]
+ end
+
+ defp csv_dimension_labels(["visit:browser_version", "visit:browser"]) do
+ [:name, :version]
+ end
+
+ defp csv_dimension_labels(["visit:os_version", "visit:os"]) do
+ [:name, :version]
+ end
+
+ defp csv_dimension_labels([_dimension]) do
+ [:name]
+ end
+
+ defp get_goal_filtered_metric_label_overrides(query) do
+ case query[:dimensions] do
+ ["time:" <> _] ->
+ %{
+ visitors: :unique_conversions,
+ events: :total_conversions,
+ group_conversion_rate: :conversion_rate
+ }
+
+ _ ->
+ %{
+ visitors: :conversions,
+ group_conversion_rate: :conversion_rate
+ }
+ end
+ end
+
+ @dimension_specific_metric_label_overrides %{
+ ["visit:entry_page"] => %{
+ visitors: :unique_entrances,
+ visits: :total_entrances
+ },
+ ["visit:exit_page"] => %{
+ visitors: :unique_exits,
+ visits: :total_exits
+ }
+ }
+
+ defp csv_metric_labels(query) do
+ goal_filter? =
+ Enum.any?(query[:filters], fn [_operator, key | _rest] ->
+ key == "event:goal"
+ end)
+
+ metric_label_overrides =
+ if goal_filter? do
+ get_goal_filtered_metric_label_overrides(query)
+ else
+ Map.get(@dimension_specific_metric_label_overrides, query[:dimensions], %{})
+ end
+
+ Enum.map(query[:metrics], &(metric_label_overrides[&1] || &1))
+ end
+
+ defp csv_dimension_values(["visit:browser_version", "visit:browser"], row) do
+ row[:dimensions] |> Enum.reverse()
+ end
+
+ defp csv_dimension_values(["visit:os_version", "visit:os"], row) do
+ row[:dimensions] |> Enum.reverse()
+ end
+
+ defp csv_dimension_values([_dimension], row) do
+ row[:dimensions]
+ end
+end
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index 945f2e23d3..30e205525c 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -17,6 +17,7 @@ defmodule PlausibleWeb.Api.StatsController do
QueryError
}
+ alias Plausible.Stats.Dashboard.CsvExport
alias PlausibleWeb.Api.Helpers, as: H
require Logger
@@ -24,7 +25,7 @@ defmodule PlausibleWeb.Api.StatsController do
@revenue_metrics on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])
@not_set "(not set)"
- plug(:date_validation_plug when action not in [:query])
+ plug(:date_validation_plug when action not in [:query, :csv_export_v2])
plug(:validate_required_filters_plug when action not in [:current_visitors])
def query(conn, params) do
@@ -46,6 +47,26 @@ defmodule PlausibleWeb.Api.StatsController do
end
end
+ def csv_export_v2(conn, params) do
+ site = conn.assigns.site
+
+ case CsvExport.get_csvs(site, params, debug_metadata(conn)) do
+ {:ok, csvs} ->
+ {:ok, {_, zip_content}} = :zip.create(~c"export.zip", csvs, [:memory])
+
+ conn
+ |> put_resp_content_type("application/zip")
+ |> put_resp_header(
+ "content-disposition",
+ Plausible.Exports.content_disposition("export.zip")
+ )
+ |> send_resp(200, zip_content)
+
+ {:error, %QueryError{message: message}} ->
+ H.bad_request(conn, message)
+ end
+ end
+
def sources(conn, params) do
site = conn.assigns[:site]
params = Map.put(params, "property", "visit:source")
diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex
index a627ca3f2c..7e563d44df 100644
--- a/lib/plausible_web/controllers/stats_controller.ex
+++ b/lib/plausible_web/controllers/stats_controller.ex
@@ -550,7 +550,7 @@ defmodule PlausibleWeb.StatsController do
defp get_flags(user, site),
do:
- []
+ [:dashboard_csv_export_v2]
|> Enum.map(fn flag ->
{flag, FunWithFlags.enabled?(flag, for: user) || FunWithFlags.enabled?(flag, for: site)}
end)
diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex
index b1a6b609b3..71f4db39e9 100644
--- a/lib/plausible_web/router.ex
+++ b/lib/plausible_web/router.ex
@@ -175,6 +175,7 @@ defmodule PlausibleWeb.Router do
post "/stats", E2EController, :populate_stats
post "/funnel", E2EController, :create_funnel
post "/goal", E2EController, :create_goal
+ post "/enable-dashboard-csv-export-v2", E2EController, :enable_dashboard_csv_export_v2
end
end
end
@@ -284,6 +285,7 @@ defmodule PlausibleWeb.Router do
scope private: %{allow_consolidated_views: true} do
post "/:domain/query", StatsController, :query
+ post "/:domain/export", StatsController, :csv_export_v2
get "/:domain/google-search-terms", StatsController, :google_search_terms
get "/:domain/current-visitors", StatsController, :current_visitors
get "/:domain/sources", StatsController, :sources
diff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/exit_pages.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/exit_pages.csv
index 736ba6d58f..fd838ca9ed 100644
--- a/test/plausible_web/controllers/CSVs/30d-filter-path/exit_pages.csv
+++ b/test/plausible_web/controllers/CSVs/30d-filter-path/exit_pages.csv
@@ -1,2 +1,2 @@
-name,unique_exits,total_exits,exit_rate
-/some-other-page,1,1,
+name,unique_exits,total_exits
+/some-other-page,1,1
diff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/exit_pages_legacy.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/exit_pages_legacy.csv
new file mode 100644
index 0000000000..736ba6d58f
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/30d-filter-path/exit_pages_legacy.csv
@@ -0,0 +1,2 @@
+name,unique_exits,total_exits,exit_rate
+/some-other-page,1,1,
diff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/visitors.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/visitors.csv
index c1e30fab06..e345da62c5 100644
--- a/test/plausible_web/controllers/CSVs/30d-filter-path/visitors.csv
+++ b/test/plausible_web/controllers/CSVs/30d-filter-path/visitors.csv
@@ -1,32 +1,32 @@
-date,visitors,pageviews,visits,views_per_visit,bounce_rate,visit_duration,scroll_depth
-2021-09-20,0,0,0,0.0,0.0,,
-2021-09-21,0,0,0,0.0,0.0,,
-2021-09-22,0,0,0,0.0,0.0,,
-2021-09-23,0,0,0,0.0,0.0,,
-2021-09-24,0,0,0,0.0,0.0,,
-2021-09-25,0,0,0,0.0,0.0,,
-2021-09-26,0,0,0,0.0,0.0,,
-2021-09-27,0,0,0,0.0,0.0,,
-2021-09-28,0,0,0,0.0,0.0,,
-2021-09-29,0,0,0,0.0,0.0,,
-2021-09-30,0,0,0,0.0,0.0,,
-2021-10-01,0,0,0,0.0,0.0,,
-2021-10-02,0,0,0,0.0,0.0,,
-2021-10-03,0,0,0,0.0,0.0,,
-2021-10-04,0,0,0,0.0,0.0,,
-2021-10-05,0,0,0,0.0,0.0,,
-2021-10-06,0,0,0,0.0,0.0,,
-2021-10-07,0,0,0,0.0,0.0,,
-2021-10-08,0,0,0,0.0,0.0,,
-2021-10-09,0,0,0,0.0,0.0,,
-2021-10-10,0,0,0,0.0,0.0,,
-2021-10-11,0,0,0,0.0,0.0,,
-2021-10-12,0,0,0,0.0,0.0,,
-2021-10-13,0,0,0,0.0,0.0,,
-2021-10-14,0,0,0,0.0,0.0,,
-2021-10-15,0,0,0,0.0,0.0,,
-2021-10-16,0,0,0,0.0,0.0,,
-2021-10-17,0,0,0,0.0,0.0,,
-2021-10-18,0,0,0,0.0,0.0,,
-2021-10-19,0,0,0,0.0,0.0,,
-2021-10-20,1,1,1,2.0,0,60,30
+date,visitors,pageviews,visits,bounce_rate,time_on_page,scroll_depth
+2021-09-20,0,0,0,0.0,,
+2021-09-21,0,0,0,0.0,,
+2021-09-22,0,0,0,0.0,,
+2021-09-23,0,0,0,0.0,,
+2021-09-24,0,0,0,0.0,,
+2021-09-25,0,0,0,0.0,,
+2021-09-26,0,0,0,0.0,,
+2021-09-27,0,0,0,0.0,,
+2021-09-28,0,0,0,0.0,,
+2021-09-29,0,0,0,0.0,,
+2021-09-30,0,0,0,0.0,,
+2021-10-01,0,0,0,0.0,,
+2021-10-02,0,0,0,0.0,,
+2021-10-03,0,0,0,0.0,,
+2021-10-04,0,0,0,0.0,,
+2021-10-05,0,0,0,0.0,,
+2021-10-06,0,0,0,0.0,,
+2021-10-07,0,0,0,0.0,,
+2021-10-08,0,0,0,0.0,,
+2021-10-09,0,0,0,0.0,,
+2021-10-10,0,0,0,0.0,,
+2021-10-11,0,0,0,0.0,,
+2021-10-12,0,0,0,0.0,,
+2021-10-13,0,0,0,0.0,,
+2021-10-14,0,0,0,0.0,,
+2021-10-15,0,0,0,0.0,,
+2021-10-16,0,0,0,0.0,,
+2021-10-17,0,0,0,0.0,,
+2021-10-18,0,0,0,0.0,,
+2021-10-19,0,0,0,0.0,,
+2021-10-20,1,1,1,0,60,30
diff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/visitors_legacy.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/visitors_legacy.csv
new file mode 100644
index 0000000000..c1e30fab06
--- /dev/null
+++ b/test/plausible_web/controllers/CSVs/30d-filter-path/visitors_legacy.csv
@@ -0,0 +1,32 @@
+date,visitors,pageviews,visits,views_per_visit,bounce_rate,visit_duration,scroll_depth
+2021-09-20,0,0,0,0.0,0.0,,
+2021-09-21,0,0,0,0.0,0.0,,
+2021-09-22,0,0,0,0.0,0.0,,
+2021-09-23,0,0,0,0.0,0.0,,
+2021-09-24,0,0,0,0.0,0.0,,
+2021-09-25,0,0,0,0.0,0.0,,
+2021-09-26,0,0,0,0.0,0.0,,
+2021-09-27,0,0,0,0.0,0.0,,
+2021-09-28,0,0,0,0.0,0.0,,
+2021-09-29,0,0,0,0.0,0.0,,
+2021-09-30,0,0,0,0.0,0.0,,
+2021-10-01,0,0,0,0.0,0.0,,
+2021-10-02,0,0,0,0.0,0.0,,
+2021-10-03,0,0,0,0.0,0.0,,
+2021-10-04,0,0,0,0.0,0.0,,
+2021-10-05,0,0,0,0.0,0.0,,
+2021-10-06,0,0,0,0.0,0.0,,
+2021-10-07,0,0,0,0.0,0.0,,
+2021-10-08,0,0,0,0.0,0.0,,
+2021-10-09,0,0,0,0.0,0.0,,
+2021-10-10,0,0,0,0.0,0.0,,
+2021-10-11,0,0,0,0.0,0.0,,
+2021-10-12,0,0,0,0.0,0.0,,
+2021-10-13,0,0,0,0.0,0.0,,
+2021-10-14,0,0,0,0.0,0.0,,
+2021-10-15,0,0,0,0.0,0.0,,
+2021-10-16,0,0,0,0.0,0.0,,
+2021-10-17,0,0,0,0.0,0.0,,
+2021-10-18,0,0,0,0.0,0.0,,
+2021-10-19,0,0,0,0.0,0.0,,
+2021-10-20,1,1,1,2.0,0,60,30
diff --git a/test/plausible_web/controllers/api/stats_controller/dashboard_csv_export_test.exs b/test/plausible_web/controllers/api/stats_controller/dashboard_csv_export_test.exs
new file mode 100644
index 0000000000..ecc13363e7
--- /dev/null
+++ b/test/plausible_web/controllers/api/stats_controller/dashboard_csv_export_test.exs
@@ -0,0 +1,1083 @@
+defmodule PlausibleWeb.Api.StatsController.DashboardCsvExportTest do
+ use PlausibleWeb.ConnCase, async: false
+ use Plausible.Repo
+
+ @default_reports %{
+ "visitors.csv" => %{
+ dimensions: ["time:day"],
+ metrics: [
+ "visitors",
+ "pageviews",
+ "visits",
+ "views_per_visit",
+ "bounce_rate",
+ "visit_duration"
+ ]
+ },
+ "sources.csv" => %{
+ dimensions: ["visit:source"],
+ metrics: ["visitors", "bounce_rate", "visit_duration"]
+ },
+ "channels.csv" => %{
+ dimensions: ["visit:channel"],
+ metrics: ["visitors", "bounce_rate", "visit_duration"]
+ },
+ "referrers.csv" => %{
+ dimensions: ["visit:referrer"],
+ metrics: ["visitors", "bounce_rate", "visit_duration"]
+ },
+ "utm_mediums.csv" => %{
+ dimensions: ["visit:utm_medium"],
+ always_on_filters: [["is_not", "visit:utm_medium", [""]]],
+ metrics: ["visitors", "bounce_rate", "visit_duration"]
+ },
+ "utm_sources.csv" => %{
+ dimensions: ["visit:utm_source"],
+ always_on_filters: [["is_not", "visit:utm_source", [""]]],
+ metrics: ["visitors", "bounce_rate", "visit_duration"]
+ },
+ "utm_campaigns.csv" => %{
+ dimensions: ["visit:utm_campaign"],
+ always_on_filters: [["is_not", "visit:utm_campaign", [""]]],
+ metrics: ["visitors", "bounce_rate", "visit_duration"]
+ },
+ "utm_contents.csv" => %{
+ dimensions: ["visit:utm_content"],
+ always_on_filters: [["is_not", "visit:utm_content", [""]]],
+ metrics: ["visitors", "bounce_rate", "visit_duration"]
+ },
+ "utm_terms.csv" => %{
+ dimensions: ["visit:utm_term"],
+ always_on_filters: [["is_not", "visit:utm_term", [""]]],
+ metrics: ["visitors", "bounce_rate", "visit_duration"]
+ },
+ "pages.csv" => %{
+ dimensions: ["event:page"],
+ metrics: ["visitors", "pageviews", "bounce_rate", "time_on_page", "scroll_depth"]
+ },
+ "entry_pages.csv" => %{
+ dimensions: ["visit:entry_page"],
+ always_on_filters: [["is_not", "visit:entry_page", [""]]],
+ metrics: ["visitors", "visits", "bounce_rate", "visit_duration"]
+ },
+ "exit_pages.csv" => %{
+ dimensions: ["visit:exit_page"],
+ always_on_filters: [["is_not", "visit:exit_page", [""]]],
+ metrics: ["visitors", "visits", "exit_rate"]
+ },
+ "browsers.csv" => %{
+ dimensions: ["visit:browser"],
+ metrics: ["visitors"]
+ },
+ "browser_versions.csv" => %{
+ dimensions: ["visit:browser_version", "visit:browser"],
+ metrics: ["visitors"]
+ },
+ "operating_systems.csv" => %{
+ dimensions: ["visit:os"],
+ metrics: ["visitors"]
+ },
+ "operating_system_versions.csv" => %{
+ dimensions: ["visit:os_version", "visit:os"],
+ metrics: ["visitors"]
+ },
+ "devices.csv" => %{
+ dimensions: ["visit:device"],
+ metrics: ["visitors"]
+ },
+ "countries.csv" => %{
+ dimensions: ["visit:country_name"],
+ always_on_filters: [["is_not", "visit:country", ["\0\0", "ZZ"]]],
+ metrics: ["visitors"]
+ },
+ "regions.csv" => %{
+ dimensions: ["visit:region_name"],
+ always_on_filters: [["is_not", "visit:region", [""]]],
+ metrics: ["visitors"]
+ },
+ "cities.csv" => %{
+ dimensions: ["visit:city_name"],
+ always_on_filters: [["is_not", "visit:city", [0]]],
+ metrics: ["visitors"]
+ }
+ }
+
+ @page_filtered_reports put_in(@default_reports, ["visitors.csv", :metrics], [
+ "visitors",
+ "pageviews",
+ "visits",
+ "bounce_rate",
+ "time_on_page",
+ "scroll_depth"
+ ])
+ |> put_in(
+ ["exit_pages.csv", :metrics],
+ get_in(@default_reports, ["exit_pages.csv", :metrics]) -- ["exit_rate"]
+ )
+
+ @goal_filtered_reports Map.new(@default_reports, fn
+ {"visitors.csv" = filename, params} ->
+ {filename,
+ %{params | metrics: ["visitors", "events", "group_conversion_rate"]}}
+
+ {filename, params} ->
+ {filename,
+ %{params | metrics: ["visitors", "group_conversion_rate"]}}
+ end)
+
+ @base_params %{
+ date_range: "30d",
+ relative_date: nil,
+ filters: [],
+ include: %{imports: false},
+ reports: @default_reports
+ }
+
+ describe "POST /api/stats/:domain/export" do
+ setup [:create_user, :create_site, :log_in]
+
+ test "exports all the necessary CSV files", %{conn: conn, site: site} do
+ conn = do_export(conn, site, @base_params)
+
+ assert {"content-type", "application/zip; charset=utf-8"} =
+ List.keyfind(conn.resp_headers, "content-type", 0)
+
+ {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])
+
+ zip = Enum.map(zip, fn {filename, _} -> filename end)
+
+ assert ~c"visitors.csv" in zip
+ assert ~c"browsers.csv" in zip
+ assert ~c"browser_versions.csv" in zip
+ assert ~c"cities.csv" in zip
+ # assert ~c"conversions.csv" in zip
+ assert ~c"countries.csv" in zip
+ assert ~c"devices.csv" in zip
+ assert ~c"entry_pages.csv" in zip
+ assert ~c"exit_pages.csv" in zip
+ assert ~c"operating_systems.csv" in zip
+ assert ~c"operating_system_versions.csv" in zip
+ assert ~c"pages.csv" in zip
+ assert ~c"regions.csv" in zip
+ assert ~c"sources.csv" in zip
+ assert ~c"channels.csv" in zip
+ assert ~c"utm_campaigns.csv" in zip
+ assert ~c"utm_contents.csv" in zip
+ assert ~c"utm_mediums.csv" in zip
+ assert ~c"utm_sources.csv" in zip
+ assert ~c"utm_terms.csv" in zip
+ end
+
+ test "limits pages.csv and exit_pages.csv to 100 rows", %{conn: conn, site: site} do
+ events = for i <- 1..101, do: build(:pageview, pathname: "/page-#{i}")
+ populate_stats(site, events)
+
+ zip_content =
+ conn
+ |> do_export(site, %{@base_params | date_range: "day"})
+ |> response(200)
+
+ pages = unzip_and_parse_csv(zip_content, ~c"pages.csv")
+ # header + 100 data rows + trailing empty row
+ assert length(pages) == 102
+
+ exit_pages = unzip_and_parse_csv(zip_content, ~c"exit_pages.csv")
+ assert length(exit_pages) == 102
+ end
+
+ test "limits other breakdown reports to 300 rows", %{conn: conn, site: site} do
+ events = for i <- 1..301, do: build(:pageview, referrer_source: "Source #{i}")
+ populate_stats(site, events)
+
+ sources =
+ conn
+ |> do_export(site, %{@base_params | date_range: "day"})
+ |> response(200)
+ |> unzip_and_parse_csv(~c"sources.csv")
+
+ # header + 300 data rows + trailing empty row
+ assert length(sources) == 302
+ end
+
+ test "exports scroll depth metric in pages.csv", %{conn: conn, site: site} do
+ t0 = ~N[2020-01-01 00:00:00]
+ [t1, t2, t3] = for i <- 1..3, do: NaiveDateTime.add(t0, i, :minute)
+
+ populate_stats(site, [
+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: t0),
+ build(:engagement,
+ user_id: 12,
+ pathname: "/blog",
+ timestamp: t1,
+ scroll_depth: 20,
+ engagement_time: 60_000
+ ),
+ build(:pageview, user_id: 12, pathname: "/another", timestamp: t1),
+ build(:engagement,
+ user_id: 12,
+ pathname: "/another",
+ timestamp: t2,
+ scroll_depth: 24,
+ engagement_time: 60_000
+ ),
+ build(:pageview, user_id: 34, pathname: "/blog", timestamp: t0),
+ build(:engagement,
+ user_id: 34,
+ pathname: "/blog",
+ timestamp: t1,
+ scroll_depth: 17,
+ engagement_time: 60_000
+ ),
+ build(:pageview, user_id: 34, pathname: "/another", timestamp: t1),
+ build(:engagement,
+ user_id: 34,
+ pathname: "/another",
+ timestamp: t2,
+ scroll_depth: 26,
+ engagement_time: 60_000
+ ),
+ build(:pageview, user_id: 34, pathname: "/blog", timestamp: t2),
+ build(:engagement,
+ user_id: 34,
+ pathname: "/blog",
+ timestamp: t3,
+ scroll_depth: 60,
+ engagement_time: 60_000
+ ),
+ build(:pageview, user_id: 56, pathname: "/blog", timestamp: t0),
+ build(:engagement,
+ user_id: 56,
+ pathname: "/blog",
+ timestamp: t1,
+ scroll_depth: 100,
+ engagement_time: 60_000
+ )
+ ])
+
+ pages =
+ conn
+ |> do_export(site, %{@base_params | date_range: "day", relative_date: "2020-01-01"})
+ |> response(200)
+ |> unzip_and_parse_csv(~c"pages.csv")
+
+ assert pages == [
+ ["name", "visitors", "pageviews", "bounce_rate", "time_on_page", "scroll_depth"],
+ ["/blog", "3", "4", "33", "80", "60"],
+ ["/another", "2", "2", "0", "60", "25"],
+ [""]
+ ]
+ end
+
+ # test "exports only internally used props in custom_props.csv for a growth plan", %{
+ # conn: conn,
+ # site: site
+ # } do
+ # {:ok, site} = Plausible.Props.allow(site, ["author"])
+
+ # [owner | _] = Repo.preload(site, :owners).owners
+ # subscribe_to_growth_plan(owner)
+
+ # populate_stats(site, [
+ # build(:pageview, "meta.key": ["author"], "meta.value": ["a"]),
+ # build(:event, name: "File Download", "meta.key": ["url"], "meta.value": ["b"])
+ # ])
+
+ # result =
+ # conn
+ # |> do_export(site, %{@base_params | date_range: "day"})
+ # |> response(200)
+ # |> unzip_and_parse_csv(~c"custom_props.csv")
+
+ # assert result == [
+ # ["property", "value", "visitors", "events", "percentage"],
+ # ["url", "(none)", "1", "1", "50.0"],
+ # ["url", "b", "1", "1", "50.0"],
+ # [""]
+ # ]
+ # end
+
+ # test "does not include custom_props.csv for a growth plan if no internal props used", %{
+ # conn: conn,
+ # site: site
+ # } do
+ # {:ok, site} = Plausible.Props.allow(site, ["author"])
+
+ # [owner | _] = Repo.preload(site, :owners).owners
+ # subscribe_to_growth_plan(owner)
+
+ # populate_stats(site, [
+ # build(:pageview, "meta.key": ["author"], "meta.value": ["a"])
+ # ])
+
+ # {:ok, zip} =
+ # conn
+ # |> do_export(site, %{@base_params | date_range: "day"})
+ # |> response(200)
+ # |> :zip.unzip([:memory])
+
+ # files = Map.new(zip)
+
+ # refute Map.has_key?(files, ~c"custom_props.csv")
+ # end
+
+ test "exports data in zipped csvs", %{conn: conn, site: site} do
+ populate_exported_stats(site)
+
+ conn =
+ do_export(conn, site, %{@base_params | date_range: ["2021-09-20", "2021-10-20"]})
+
+ assert_zip(conn, "30d")
+ end
+
+ # test "exports allowed event props for a trial account", %{conn: conn, site: site} do
+ # {:ok, site} = Plausible.Props.allow(site, ["author", "logged_in"])
+
+ # populate_stats(site, [
+ # build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),
+ # build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),
+ # build(:event, "meta.key": ["author"], "meta.value": ["marko"], name: "Newsletter Signup"),
+ # build(:pageview, user_id: 999, "meta.key": ["logged_in"], "meta.value": ["true"]),
+ # build(:pageview, user_id: 999, "meta.key": ["logged_in"], "meta.value": ["true"]),
+ # build(:pageview, "meta.key": ["disallowed"], "meta.value": ["whatever"]),
+ # build(:pageview)
+ # ])
+
+ # result =
+ # conn
+ # |> do_export(site, %{@base_params | date_range: "day"})
+ # |> response(200)
+ # |> unzip_and_parse_csv(~c"custom_props.csv")
+
+ # assert result == [
+ # ["property", "value", "visitors", "events", "percentage"],
+ # ["author", "(none)", "3", "4", "50.0"],
+ # ["author", "uku", "2", "2", "33.33"],
+ # ["author", "marko", "1", "1", "16.67"],
+ # ["logged_in", "(none)", "5", "5", "83.33"],
+ # ["logged_in", "true", "1", "2", "16.67"],
+ # [""]
+ # ]
+ # end
+
+ test "exports data grouped by interval", %{conn: conn, site: site} do
+ populate_exported_stats(site)
+
+ params =
+ @base_params
+ |> Map.put(:date_range, ["2021-09-20", "2021-10-20"])
+ |> put_in([:reports, "visitors.csv", :dimensions], ["time:week"])
+
+ visitors =
+ conn
+ |> do_export(site, params)
+ |> response(200)
+ |> unzip_and_parse_csv(~c"visitors.csv")
+
+ assert visitors == [
+ [
+ "date",
+ "visitors",
+ "pageviews",
+ "visits",
+ "views_per_visit",
+ "bounce_rate",
+ "visit_duration"
+ ],
+ ["2021-09-20", "1", "1", "1", "1.0", "100", "0"],
+ ["2021-09-27", "0", "0", "0", "0.0", "0.0", ""],
+ ["2021-10-04", "0", "0", "0", "0.0", "0.0", ""],
+ ["2021-10-11", "0", "0", "0", "0.0", "0.0", ""],
+ ["2021-10-18", "3", "4", "3", "1.33", "33", "40"],
+ [""]
+ ]
+ end
+
+ test "exports operating system versions", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, operating_system: "Mac", operating_system_version: "14"),
+ build(:pageview, operating_system: "Mac", operating_system_version: "14"),
+ build(:pageview, operating_system: "Mac", operating_system_version: "14"),
+ build(:pageview, operating_system: "Ubuntu", operating_system_version: "20.04"),
+ build(:pageview, operating_system: "Ubuntu", operating_system_version: "20.04"),
+ build(:pageview, operating_system: "Mac", operating_system_version: "13")
+ ])
+
+ os_versions =
+ conn
+ |> do_export(site, %{@base_params | date_range: "day"})
+ |> response(200)
+ |> unzip_and_parse_csv(~c"operating_system_versions.csv")
+
+ assert os_versions == [
+ ["name", "version", "visitors"],
+ ["Mac", "14", "3"],
+ ["Ubuntu", "20.04", "2"],
+ ["Mac", "13", "1"],
+ [""]
+ ]
+ end
+
+ test "exports imported data when requested", %{conn: conn, site: site} do
+ site_import = insert(:site_import, site: site)
+
+ insert(:goal, site: site, event_name: "Outbound Link: Click")
+
+ populate_stats(site, site_import.id, [
+ build(:imported_visitors, visitors: 9),
+ build(:imported_browsers, browser: "Chrome", pageviews: 1),
+ build(:imported_devices, device: "Desktop", pageviews: 1),
+ build(:imported_entry_pages, entry_page: "/test", pageviews: 1),
+ build(:imported_exit_pages, exit_page: "/test", pageviews: 1),
+ build(:imported_locations, country: "PL", region: "PL-22", city: 3_099_434, pageviews: 1),
+ build(:imported_operating_systems, operating_system: "Mac", pageviews: 1),
+ build(:imported_pages, page: "/test", pageviews: 1),
+ build(:imported_sources,
+ source: "Google",
+ channel: "Paid Search",
+ utm_medium: "search",
+ utm_campaign: "ads",
+ utm_source: "google",
+ utm_content: "content",
+ utm_term: "term",
+ pageviews: 1
+ ),
+ build(:imported_custom_events,
+ name: "Outbound Link: Click",
+ link_url: "https://example.com",
+ visitors: 5,
+ events: 10
+ )
+ ])
+
+ tomorrow = Date.utc_today() |> Date.add(1) |> Date.to_iso8601()
+
+ params = %{@base_params | relative_date: tomorrow, include: %{imports: true}}
+
+ assert response = do_export(conn, site, params) |> response(200)
+ {:ok, zip} = :zip.unzip(response, [:memory])
+
+ filenames = zip |> Enum.map(fn {filename, _} -> to_string(filename) end)
+
+ # NOTE: currently, custom_props.csv is not populated from imported data
+ expected_filenames = [
+ "visitors.csv",
+ "sources.csv",
+ "channels.csv",
+ "utm_mediums.csv",
+ "utm_sources.csv",
+ "utm_campaigns.csv",
+ "utm_contents.csv",
+ "utm_terms.csv",
+ "pages.csv",
+ "entry_pages.csv",
+ "exit_pages.csv",
+ "countries.csv",
+ "regions.csv",
+ "cities.csv",
+ "browsers.csv",
+ "browser_versions.csv",
+ "operating_systems.csv",
+ "operating_system_versions.csv",
+ "devices.csv",
+ # "conversions.csv",
+ "referrers.csv"
+ ]
+
+ Enum.each(expected_filenames, fn expected -> assert expected in filenames end)
+
+ Enum.each(zip, fn
+ {~c"visitors.csv", data} ->
+ csv = parse_csv(data)
+
+ assert List.first(csv) == [
+ "date",
+ "visitors",
+ "pageviews",
+ "visits",
+ "views_per_visit",
+ "bounce_rate",
+ "visit_duration"
+ ]
+
+ assert Enum.at(csv, -2) ==
+ [Date.to_iso8601(Date.utc_today()), "9", "1", "1", "1.0", "0.0", "10.0"]
+
+ {~c"sources.csv", data} ->
+ assert parse_csv(data) == [
+ ["name", "visitors", "bounce_rate", "visit_duration"],
+ ["Google", "1", "0.0", "10.0"],
+ [""]
+ ]
+
+ {~c"channels.csv", data} ->
+ assert parse_csv(data) == [
+ ["name", "visitors", "bounce_rate", "visit_duration"],
+ ["Paid Search", "1", "0.0", "10.0"],
+ [""]
+ ]
+
+ {~c"utm_mediums.csv", data} ->
+ assert parse_csv(data) == [
+ ["name", "visitors", "bounce_rate", "visit_duration"],
+ ["search", "1", "0.0", "10.0"],
+ [""]
+ ]
+
+ {~c"utm_sources.csv", data} ->
+ assert parse_csv(data) == [
+ ["name", "visitors", "bounce_rate", "visit_duration"],
+ ["google", "1", "0.0", "10.0"],
+ [""]
+ ]
+
+ {~c"utm_campaigns.csv", data} ->
+ assert parse_csv(data) == [
+ ["name", "visitors", "bounce_rate", "visit_duration"],
+ ["ads", "1", "0.0", "10.0"],
+ [""]
+ ]
+
+ {~c"utm_contents.csv", data} ->
+ assert parse_csv(data) == [
+ ["name", "visitors", "bounce_rate", "visit_duration"],
+ ["content", "1", "0.0", "10.0"],
+ [""]
+ ]
+
+ {~c"utm_terms.csv", data} ->
+ assert parse_csv(data) == [
+ ["name", "visitors", "bounce_rate", "visit_duration"],
+ ["term", "1", "0.0", "10.0"],
+ [""]
+ ]
+
+ {~c"pages.csv", data} ->
+ assert parse_csv(data) == [
+ [
+ "name",
+ "visitors",
+ "pageviews",
+ "bounce_rate",
+ "time_on_page",
+ "scroll_depth"
+ ],
+ ["/test", "1", "1", "0.0", "10", ""],
+ [""]
+ ]
+
+ {~c"entry_pages.csv", data} ->
+ assert parse_csv(data) == [
+ [
+ "name",
+ "unique_entrances",
+ "total_entrances",
+ "bounce_rate",
+ "visit_duration"
+ ],
+ ["/test", "1", "1", "0.0", "10.0"],
+ [""]
+ ]
+
+ {~c"exit_pages.csv", data} ->
+ assert parse_csv(data) == [
+ ["name", "unique_exits", "total_exits", "exit_rate"],
+ ["/test", "1", "1", "100.0"],
+ [""]
+ ]
+
+ {~c"countries.csv", data} ->
+ assert parse_csv(data) == [["name", "visitors"], ["Poland", "1"], [""]]
+
+ {~c"regions.csv", data} ->
+ assert parse_csv(data) == [["name", "visitors"], ["Pomerania", "1"], [""]]
+
+ {~c"cities.csv", data} ->
+ assert parse_csv(data) == [["name", "visitors"], ["Gdańsk", "1"], [""]]
+
+ {~c"browsers.csv", data} ->
+ assert parse_csv(data) == [["name", "visitors"], ["Chrome", "1"], [""]]
+
+ {~c"browser_versions.csv", data} ->
+ assert parse_csv(data) == [
+ ["name", "version", "visitors"],
+ ["Chrome", "(not set)", "1"],
+ [""]
+ ]
+
+ {~c"operating_systems.csv", data} ->
+ assert parse_csv(data) == [["name", "visitors"], ["Mac", "1"], [""]]
+
+ {~c"operating_system_versions.csv", data} ->
+ assert parse_csv(data) == [
+ ["name", "version", "visitors"],
+ ["Mac", "(not set)", "1"],
+ [""]
+ ]
+
+ {~c"devices.csv", data} ->
+ assert parse_csv(data) == [["name", "visitors"], ["Desktop", "1"], [""]]
+
+ {~c"conversions.csv", data} ->
+ assert parse_csv(data) == [
+ ["name", "unique_conversions", "total_conversions"],
+ ["Outbound Link: Click", "5", "10"],
+ [""]
+ ]
+
+ {~c"referrers.csv", data} ->
+ assert parse_csv(data) == [
+ ["name", "visitors", "bounce_rate", "visit_duration"],
+ ["Direct / None", "1", "0.0", "10.0"],
+ [""]
+ ]
+ end)
+ end
+ end
+
+ defp parse_csv(file_content) when is_binary(file_content) do
+ file_content
+ |> String.split("\r\n")
+ |> Enum.map(&String.split(&1, ","))
+ end
+
+ describe "POST /api/stats/:domain/export - via shared link" do
+ setup [:create_user, :create_site]
+
+ test "exports data in zipped csvs", %{conn: conn, site: site} do
+ link = insert(:shared_link, site: site)
+
+ populate_exported_stats(site)
+
+ params =
+ %{@base_params | date_range: ["2021-09-20", "2021-10-20"], include: %{imports: false}}
+ |> Map.put(:auth, link.slug)
+
+ conn = do_export(conn, site, params)
+
+ assert_zip(conn, "30d")
+ end
+ end
+
+ describe "POST /api/stats/:domain/export - for past 6 months" do
+ setup [:create_user, :create_site, :log_in]
+
+ test "exports 6 months of data in zipped csvs", %{conn: conn, site: site} do
+ populate_exported_stats(site)
+
+ conn =
+ do_export(conn, site, %{
+ @base_params
+ | date_range: "6mo",
+ relative_date: "2021-11-20",
+ reports: put_in(@default_reports, ["visitors.csv", :dimensions], ["time:month"])
+ })
+
+ assert_zip(conn, "6m")
+ end
+ end
+
+ describe "POST /api/stats/:domain/export - with path filter" do
+ setup [:create_user, :create_site, :log_in]
+
+ test "exports filtered data in zipped csvs", %{conn: conn, site: site} do
+ populate_exported_stats(site)
+
+ conn =
+ do_export(conn, site, %{
+ @base_params
+ | date_range: ["2021-09-20", "2021-10-20"],
+ filters: [["is", "event:page", ["/some-other-page"]]],
+ reports: @page_filtered_reports
+ })
+
+ assert_zip(conn, "30d-filter-path")
+ end
+
+ test "exports scroll depth in visitors.csv", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: ~N[2020-01-05 00:00:00]),
+ build(:engagement,
+ user_id: 12,
+ pathname: "/blog",
+ timestamp: ~N[2020-01-05 00:01:00],
+ scroll_depth: 40
+ ),
+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: ~N[2020-01-05 10:00:00]),
+ build(:engagement,
+ user_id: 12,
+ pathname: "/blog",
+ timestamp: ~N[2020-01-05 10:01:00],
+ scroll_depth: 17
+ ),
+ build(:pageview, user_id: 34, pathname: "/blog", timestamp: ~N[2020-01-07 00:00:00]),
+ build(:engagement,
+ user_id: 34,
+ pathname: "/blog",
+ timestamp: ~N[2020-01-07 00:01:00],
+ scroll_depth: 90
+ )
+ ])
+
+ pages =
+ conn
+ |> do_export(site, %{
+ @base_params
+ | date_range: "7d",
+ relative_date: "2020-01-08",
+ filters: [["is", "event:page", ["/blog"]]],
+ reports: @page_filtered_reports
+ })
+ |> response(200)
+ |> unzip_and_parse_csv(~c"visitors.csv")
+
+ assert pages == [
+ [
+ "date",
+ "visitors",
+ "pageviews",
+ "visits",
+ "bounce_rate",
+ "time_on_page",
+ "scroll_depth"
+ ],
+ ["2020-01-01", "0", "0", "0", "0.0", "", ""],
+ ["2020-01-02", "0", "0", "0", "0.0", "", ""],
+ ["2020-01-03", "0", "0", "0", "0.0", "", ""],
+ ["2020-01-04", "0", "0", "0", "0.0", "", ""],
+ ["2020-01-05", "1", "2", "2", "100", "0", "28"],
+ ["2020-01-06", "0", "0", "0", "0.0", "", ""],
+ ["2020-01-07", "1", "1", "1", "100", "0", "90"],
+ [""]
+ ]
+ end
+ end
+
+ # describe "POST /api/stats/:domain/export - with a custom prop filter" do
+ # setup [:create_user, :create_site, :log_in]
+
+ # test "custom-props.csv only returns the prop and its value in filter", %{
+ # conn: conn,
+ # site: site
+ # } do
+ # {:ok, site} = Plausible.Props.allow(site, ["author", "logged_in"])
+
+ # populate_stats(site, [
+ # build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),
+ # build(:pageview, "meta.key": ["author"], "meta.value": ["marko"]),
+ # build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"])
+ # ])
+
+ # result =
+ # conn
+ # |> do_export(site, %{
+ # @base_params
+ # | date_range: "day",
+ # filters: [["is", "event:props:author", ["marko"]]]
+ # })
+ # |> response(200)
+ # |> unzip_and_parse_csv(~c"custom_props.csv")
+
+ # assert result == [
+ # ["property", "value", "visitors", "events", "percentage"],
+ # ["author", "marko", "1", "1", "100.0"],
+ # [""]
+ # ]
+ # end
+ # end
+
+ defp unzip_and_parse_csv(archive, filename) do
+ {:ok, zip} = :zip.unzip(archive, [:memory])
+ {_filename, data} = Enum.find(zip, &(elem(&1, 0) == filename))
+ parse_csv(data)
+ end
+
+ defp assert_zip(conn, folder) do
+ assert conn.status == 200
+
+ assert {"content-type", "application/zip; charset=utf-8"} =
+ List.keyfind(conn.resp_headers, "content-type", 0)
+
+ {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])
+
+ folder = Path.expand(folder, "test/plausible_web/controllers/CSVs")
+
+ Enum.map(zip, &assert_csv_by_fixture(&1, folder))
+ end
+
+ defp assert_csv_by_fixture({file, downloaded}, folder) do
+ file = Path.expand(file, folder)
+
+ {:ok, content} = File.read(file)
+ msg = "CSV file comparison failed (#{file})"
+ assert downloaded == content, message: msg, left: downloaded, right: content
+ end
+
+ defp do_export(conn, site, params) do
+ conn
+ |> put_req_header("content-type", "application/json")
+ |> post("/api/stats/#{site.domain}/export", Jason.encode!(params))
+ end
+
+ defp populate_exported_stats(site) do
+ populate_stats(site, [
+ build(:pageview,
+ user_id: 123,
+ pathname: "/",
+ timestamp:
+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], minute: -1)
+ |> NaiveDateTime.truncate(:second),
+ country_code: "EE",
+ subdivision1_code: "EE-37",
+ city_geoname_id: 588_409,
+ referrer_source: "Google"
+ ),
+ build(:engagement,
+ user_id: 123,
+ pathname: "/",
+ timestamp: ~N[2021-10-20 12:00:00] |> NaiveDateTime.truncate(:second),
+ engagement_time: 30_000,
+ scroll_depth: 30,
+ country_code: "EE",
+ subdivision1_code: "EE-37",
+ city_geoname_id: 588_409,
+ referrer_source: "Google"
+ ),
+ build(:pageview,
+ user_id: 123,
+ pathname: "/some-other-page",
+ timestamp:
+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], minute: -2)
+ |> NaiveDateTime.truncate(:second),
+ country_code: "EE",
+ subdivision1_code: "EE-37",
+ city_geoname_id: 588_409,
+ referrer_source: "Google"
+ ),
+ build(:engagement,
+ user_id: 123,
+ pathname: "/some-other-page",
+ timestamp:
+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], minute: -1)
+ |> NaiveDateTime.truncate(:second),
+ engagement_time: 60_000,
+ scroll_depth: 30,
+ country_code: "EE",
+ subdivision1_code: "EE-37",
+ city_geoname_id: 588_409,
+ referrer_source: "Google"
+ ),
+ build(:pageview,
+ user_id: 100,
+ pathname: "/",
+ timestamp:
+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], day: -1) |> NaiveDateTime.truncate(:second),
+ utm_medium: "search",
+ utm_campaign: "ads",
+ utm_source: "google",
+ utm_content: "content",
+ utm_term: "term",
+ browser: "Firefox",
+ browser_version: "120",
+ operating_system: "Mac",
+ operating_system_version: "14"
+ ),
+ build(:engagement,
+ user_id: 100,
+ pathname: "/",
+ timestamp:
+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], day: -1, minute: 1)
+ |> NaiveDateTime.truncate(:second),
+ engagement_time: 30_000,
+ scroll_depth: 30,
+ utm_medium: "search",
+ utm_campaign: "ads",
+ utm_source: "google",
+ utm_content: "content",
+ utm_term: "term",
+ browser: "Firefox",
+ browser_version: "120",
+ operating_system: "Mac",
+ operating_system_version: "14"
+ ),
+ build(:pageview,
+ user_id: 200,
+ timestamp:
+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], month: -1)
+ |> NaiveDateTime.truncate(:second),
+ country_code: "EE",
+ browser: "Firefox",
+ browser_version: "120",
+ operating_system: "Mac",
+ operating_system_version: "14"
+ ),
+ build(:engagement,
+ user_id: 200,
+ timestamp:
+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], month: -1, minute: 1)
+ |> NaiveDateTime.truncate(:second),
+ engagement_time: 30_000,
+ scroll_depth: 20,
+ country_code: "EE",
+ browser: "Firefox",
+ browser_version: "120",
+ operating_system: "Mac",
+ operating_system_version: "14"
+ ),
+ build(:pageview,
+ user_id: 300,
+ timestamp:
+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], month: -5)
+ |> NaiveDateTime.truncate(:second),
+ utm_campaign: "ads",
+ country_code: "EE",
+ referrer_source: "Google",
+ click_id_param: "gclid",
+ browser: "FirefoxNoVersion",
+ operating_system: "MacNoVersion"
+ ),
+ build(:engagement,
+ user_id: 300,
+ timestamp:
+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], month: -5, minute: 1)
+ |> NaiveDateTime.truncate(:second),
+ engagement_time: 30_000,
+ scroll_depth: 20,
+ utm_campaign: "ads",
+ country_code: "EE",
+ referrer_source: "Google",
+ click_id_param: "gclid",
+ browser: "FirefoxNoVersion",
+ operating_system: "MacNoVersion"
+ ),
+ build(:pageview,
+ user_id: 456,
+ timestamp:
+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], day: -1, minute: -1)
+ |> NaiveDateTime.truncate(:second),
+ pathname: "/signup",
+ "meta.key": ["variant"],
+ "meta.value": ["A"]
+ ),
+ build(:engagement,
+ user_id: 456,
+ timestamp:
+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], day: -1) |> NaiveDateTime.truncate(:second),
+ pathname: "/signup",
+ engagement_time: 60_000,
+ scroll_depth: 20,
+ "meta.key": ["variant"],
+ "meta.value": ["A"]
+ ),
+ build(:event,
+ user_id: 456,
+ timestamp:
+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], day: -1) |> NaiveDateTime.truncate(:second),
+ name: "Signup",
+ "meta.key": ["variant"],
+ "meta.value": ["A"]
+ )
+ ])
+
+ insert(:goal, %{site: site, event_name: "Signup"})
+ end
+
+ describe "POST /api/stats/:domain/export - with goal filter" do
+ setup [:create_user, :create_site, :log_in]
+
+ test "exports goal-filtered data in zipped csvs", %{conn: conn, site: site} do
+ populate_exported_stats(site)
+
+ conn =
+ do_export(conn, site, %{
+ @base_params
+ | date_range: ["2021-09-20", "2021-10-20"],
+ filters: [["is", "event:goal", ["Signup"]]],
+ reports: @goal_filtered_reports
+ })
+
+ assert_zip(conn, "30d-filter-goal")
+ end
+
+ # test "custom-props.csv only returns the prop names for the goal in filter", %{
+ # conn: conn,
+ # site: site
+ # } do
+ # {:ok, site} = Plausible.Props.allow(site, ["author", "logged_in"])
+
+ # populate_stats(site, [
+ # build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["uku"]),
+ # build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["marko"]),
+ # build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["marko"]),
+ # build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"])
+ # ])
+
+ # insert(:goal, site: site, event_name: "Newsletter Signup")
+
+ # result =
+ # conn
+ # |> do_export(site, %{
+ # @base_params
+ # | date_range: "day",
+ # filters: [["is", "event:goal", ["Newsletter Signup"]]],
+ # reports: @goal_filtered_reports
+ # })
+ # |> response(200)
+ # |> unzip_and_parse_csv(~c"custom_props.csv")
+
+ # assert result == [
+ # ["property", "value", "visitors", "events", "conversion_rate"],
+ # ["author", "marko", "2", "2", "50.0"],
+ # ["author", "uku", "1", "1", "25.0"],
+ # [""]
+ # ]
+ # end
+
+ test "exports conversions and conversion rate for operating system versions", %{
+ conn: conn,
+ site: site
+ } do
+ populate_stats(site, [
+ build(:pageview, operating_system: "Mac", operating_system_version: "14"),
+ build(:event, name: "Signup", operating_system: "Mac", operating_system_version: "14"),
+ build(:event, name: "Signup", operating_system: "Mac", operating_system_version: "14"),
+ build(:event, name: "Signup", operating_system: "Mac", operating_system_version: "14"),
+ build(:event,
+ name: "Signup",
+ operating_system: "Ubuntu",
+ operating_system_version: "20.04"
+ ),
+ build(:event,
+ name: "Signup",
+ operating_system: "Ubuntu",
+ operating_system_version: "20.04"
+ ),
+ build(:event,
+ name: "Signup",
+ operating_system: "Lubuntu",
+ operating_system_version: "20.04"
+ )
+ ])
+
+ insert(:goal, site: site, event_name: "Signup")
+
+ os_versions =
+ conn
+ |> do_export(site, %{
+ @base_params
+ | date_range: "day",
+ filters: [["is", "event:goal", ["Signup"]]],
+ reports: @goal_filtered_reports
+ })
+ |> response(200)
+ |> unzip_and_parse_csv(~c"operating_system_versions.csv")
+
+ assert os_versions == [
+ ["name", "version", "conversions", "conversion_rate"],
+ ["Mac", "14", "3", "75.0"],
+ ["Ubuntu", "20.04", "2", "100.0"],
+ ["Lubuntu", "20.04", "1", "100.0"],
+ [""]
+ ]
+ end
+ end
+end
diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exs
index 41d5432c4f..dc9f858d03 100644
--- a/test/plausible_web/controllers/stats_controller_test.exs
+++ b/test/plausible_web/controllers/stats_controller_test.exs
@@ -433,7 +433,7 @@ defmodule PlausibleWeb.StatsControllerTest do
Application.put_env(:plausible, :super_admin_user_ids, [user.id])
end
- describe "GET /:domain/export" do
+ describe "[LEGACY] GET /:domain/export" do
setup [:create_user, :create_site, :log_in]
test "exports all the necessary CSV files", %{conn: conn, site: site} do
@@ -937,7 +937,7 @@ defmodule PlausibleWeb.StatsControllerTest do
|> Enum.map(&String.split(&1, ","))
end
- describe "GET /:domain/export - via shared link" do
+ describe "[LEGACY] GET /:domain/export - via shared link" do
setup [:create_user, :create_site]
test "exports data in zipped csvs", %{conn: conn, site: site} do
@@ -956,7 +956,7 @@ defmodule PlausibleWeb.StatsControllerTest do
end
end
- describe "GET /:domain/export - for past 6 months" do
+ describe "[LEGACY] GET /:domain/export - for past 6 months" do
setup [:create_user, :create_site, :log_in]
test "exports 6 months of data in zipped csvs", %{conn: conn, site: site} do
@@ -966,7 +966,7 @@ defmodule PlausibleWeb.StatsControllerTest do
end
end
- describe "GET /:domain/export - with path filter" do
+ describe "[LEGACY] GET /:domain/export - with path filter" do
setup [:create_user, :create_site, :log_in]
test "exports filtered data in zipped csvs", %{conn: conn, site: site} do
@@ -1039,7 +1039,7 @@ defmodule PlausibleWeb.StatsControllerTest do
end
end
- describe "GET /:domain/export - with a custom prop filter" do
+ describe "[LEGACY] GET /:domain/export - with a custom prop filter" do
setup [:create_user, :create_site, :log_in]
test "custom-props.csv only returns the prop and its value in filter", %{
@@ -1090,13 +1090,28 @@ defmodule PlausibleWeb.StatsControllerTest do
end
defp assert_csv_by_fixture({file, downloaded}, folder) do
- file = Path.expand(file, folder)
+ file =
+ file
+ |> maybe_get_legacy_variant(folder)
+ |> Path.expand(folder)
{:ok, content} = File.read(file)
msg = "CSV file comparison failed (#{file})"
assert downloaded == content, message: msg, left: downloaded, right: content
end
+ # Remember to clean up the legacy fixture files too once the
+ # legacy CSV export disappears.
+ defp maybe_get_legacy_variant(filename, folder) do
+ path_filter? = String.contains?(folder, "filter-path")
+
+ case {filename, path_filter?} do
+ {~c"exit_pages.csv", true} -> ~c"exit_pages_legacy.csv"
+ {~c"visitors.csv", true} -> ~c"visitors_legacy.csv"
+ {filename, _} -> filename
+ end
+ end
+
defp populate_exported_stats(site) do
populate_stats(site, [
build(:pageview,
@@ -1260,7 +1275,7 @@ defmodule PlausibleWeb.StatsControllerTest do
insert(:goal, %{site: site, event_name: "Signup"})
end
- describe "GET /:domain/export - with goal filter" do
+ describe "[LEGACY] GET /:domain/export - with goal filter" do
setup [:create_user, :create_site, :log_in]
test "exports goal-filtered data in zipped csvs", %{conn: conn, site: site} do
diff --git a/test/support/dev/controllers/e2e_controller.ex b/test/support/dev/controllers/e2e_controller.ex
index bd61fdfbf2..90972b1194 100644
--- a/test/support/dev/controllers/e2e_controller.ex
+++ b/test/support/dev/controllers/e2e_controller.ex
@@ -96,6 +96,11 @@ defmodule PlausibleWeb.E2EController do
send_resp(conn, 200, Jason.encode!(%{"ok" => true}))
end
+ def enable_dashboard_csv_export_v2(conn, %{"domain" => domain}) do
+ FunWithFlags.enable(:dashboard_csv_export_v2, for_actor: "site:#{domain}")
+ send_resp(conn, 200, Jason.encode!(%{"ok" => true}))
+ end
+
defp get_goal(site, name) do
Plausible.Repo.get_by!(Plausible.Goal, site_id: site.id, display_name: name)
end