Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Dashboard csv export v2 (#6436)
RobertJoonasba238ad3c4e7
diff --git a/assets/js/dashboard/api.ts b/assets/js/dashboard/api.tsindex 1eb66db7f9..c2e748b8e9 100644--- a/assets/js/dashboard/api.ts+++ b/assets/js/dashboard/api.ts@@ -6,6 +6,7 @@ import { formatISO } from './util/date'import { serializeApiFilters } from './util/filters'import * as url from './util/url'import { MainGraphResponse } from './stats/graph/fetch-main-graph'+import { CsvExportRequestBody } from './stats/csv-export/csv-export-body'let abortController = new AbortController()let SHARED_LINK_AUTH: null | string = null@@ -140,13 +141,16 @@ function getHeaders(): Record<string, string> {return SHARED_LINK_AUTH ? { 'X-Shared-Link-Auth': SHARED_LINK_AUTH } : {}}-async function handleApiResponse(response: Response) {- const payload = await response.json()+async function throwApiErrorIfNotOk(response: Response) {if (!response.ok) {+ const payload = await response.json()throw new ApiError(payload.error, payload, response.status)}+}- return payload+async function handleApiResponse(response: Response) {+ await throwApiErrorIfNotOk(response)+ return response.json()}function getSharedLinkSearchParams(): Record<string, string> {@@ -175,6 +179,24 @@ export async function stats<return (await handleApiResponse(response)) as TResponse}+export async function csvExport(+ site: PlausibleSite,+ body: CsvExportRequestBody+): Promise<Blob> {+ const sharedLinkParams = getSharedLinkSearchParams()+ const queryString = sharedLinkParams.auth+ ? new URLSearchParams(sharedLinkParams).toString()+ : ''+ const path = url.apiPath(site, '/export')+ const response = await fetch(queryString ? `${path}?${queryString}` : path, {+ method: 'POST',+ headers: { ...getHeaders(), 'Content-Type': 'application/json' },+ body: JSON.stringify(body)+ })+ await throwApiErrorIfNotOk(response)+ return response.blob()+}+export async function get(url: string,dashboardState?: DashboardState,diff --git a/assets/js/dashboard/nav-menu/dashboard-options-menu.tsx b/assets/js/dashboard/nav-menu/dashboard-options-menu.tsxindex 88567690af..ac2041a93a 100644--- a/assets/js/dashboard/nav-menu/dashboard-options-menu.tsx+++ b/assets/js/dashboard/nav-menu/dashboard-options-menu.tsx@@ -1,4 +1,4 @@-import React, { useRef, useState } from 'react'+import React, { useEffect, useRef, useState } from 'react'import { Popover, Transition } from '@headlessui/react'import {EllipsisVerticalIcon,@@ -20,6 +20,7 @@ import { Notice } from '../components/notice'import { isModifierPressed, isTyping, Keybind } from '../keybinding'import { useMatch } from 'react-router-dom'import { rootRoute } from '../router'+import { CsvExportV2, ExportStatus } from '../stats/csv-export/csv-export'function ExportItem({ selectedInterval }: { selectedInterval: string }) {const site = useSiteContext()@@ -91,10 +92,21 @@ function ImportedSwitchItem({ disabled }: { disabled: boolean }) {}function DashboardOptionsMenuItems() {+ const site = useSiteContext()+ const { dashboardState } = useDashboardStateContext()const { selectedInterval, onIntervalClick, availableIntervals } =useGraphIntervalContext()const imports = useImportsIncludedContext()const buttonRef = useRef<HTMLButtonElement>(null)+ const [exportStatus, setExportStatus] = useState<ExportStatus>(+ ExportStatus.idle+ )++ useEffect(() => {+ setExportStatus((prev) =>+ prev === ExportStatus.error ? ExportStatus.idle : prev+ )+ }, [dashboardState])const showIntervalSection = availableIntervals.length > 1@@ -146,7 +158,14 @@ function DashboardOptionsMenuItems() {options={availableIntervals}/>)}- <ExportItem selectedInterval={selectedInterval} />+ {site.flags.dashboard_csv_export_v2 ? (+ <CsvExportV2+ exportStatus={exportStatus}+ setExportStatus={setExportStatus}+ />+ ) : (+ <ExportItem selectedInterval={selectedInterval} />+ )}{imports.status === 'visible' && (<><ImportedSwitchItem disabled={imports.disabled} />diff --git a/assets/js/dashboard/site-context.test.tsx b/assets/js/dashboard/site-context.test.tsxindex 13ddfea577..5822549247 100644--- a/assets/js/dashboard/site-context.test.tsx+++ b/assets/js/dashboard/site-context.test.tsx@@ -32,7 +32,7 @@ describe('parseSiteFromDataset', () => {data-is-dbip="false"data-current-user-role="owner"data-current-user-id="1"- data-flags="{}"+ data-flags='{"dashboard_csv_export_v2": false}'data-valid-intervals-by-period='{"12mo":["day","week","month"],"7d":["hour","day"],"28d":["day","week"],"30d":["day","week"],"91d":["day","week","month"],"6mo":["day","week","month"],"all":["week","month"],"custom":["day","week","month"],"day":["minute","hour"],"month":["day","week"],"realtime":["minute"],"year":["day","week","month"]}'{...attrs}/>@@ -58,7 +58,7 @@ describe('parseSiteFromDataset', () => {embedded: false,background: undefined,isDbip: false,- flags: {},+ flags: { dashboard_csv_export_v2: false },shared: false,isConsolidatedView: false}diff --git a/assets/js/dashboard/site-context.tsx b/assets/js/dashboard/site-context.tsxindex a7ea4d3fc8..2e61d5e9be 100644--- a/assets/js/dashboard/site-context.tsx+++ b/assets/js/dashboard/site-context.tsx@@ -32,7 +32,7 @@ export function parseSiteFromDataset(dataset: DOMStringMap): PlausibleSite {}// Update this object when new feature flags are added to the frontend.-type FeatureFlags = Record<never, boolean>+type FeatureFlags = Record<'dashboard_csv_export_v2', boolean>export const siteContextDefaultValue = {domain: '',diff --git a/assets/js/dashboard/stats-query.ts b/assets/js/dashboard/stats-query.tsindex 2e49f3b1f7..234a593d8b 100644--- a/assets/js/dashboard/stats-query.ts+++ b/assets/js/dashboard/stats-query.ts@@ -7,13 +7,7 @@ import {} from './dashboard-state'import { ComparisonMode, DashboardPeriod } from './dashboard-time-periods'import { formatISO } from './util/date'-import {- Pagination,- SimpleFilterDimensions,- CustomPropertyFilterDimensions,- TimeDimensions,- SortDirection-} from '../types/query-api'+import * as apiTypes from '../types/query-api'import { remapToApiFilters } from './util/filters'export type FilterModifiers = { case_sensitive?: boolean }@@ -23,22 +17,26 @@ export type ApiFilter =| [FilterOperator, FilterKey, FilterClause[], FilterModifiers]export type NonTimeDimension =- | SimpleFilterDimensions- | CustomPropertyFilterDimensions+ | apiTypes.SimpleFilterDimensions+ | apiTypes.CustomPropertyFilterDimensions++export type TimeDimension = apiTypes.TimeDimensions | 'time:minute'-export type Dimension = NonTimeDimension | TimeDimensions | 'time:minute'+export type Dimension = NonTimeDimension | TimeDimension-export type OrderByEntry = [Metric | NonTimeDimension, SortDirection]+export type OrderByEntry = [Metric | NonTimeDimension, apiTypes.SortDirection]export type OrderBy = OrderByEntry[]-type DateRange = DashboardPeriod | [string, string]+export type DateRange = DashboardPeriod | [string, string]+export type RelativeDate = string | null+type IncludeCompare =| ComparisonMode.previous_period| ComparisonMode.year_over_year| string[]| null-type QueryInclude = {+export type QueryInclude = {imports: booleanimports_meta: booleantime_labels: boolean@@ -54,19 +52,19 @@ export type ReportParams = {dimensions: Dimension[]include?: Partial<QueryInclude>order_by?: OrderBy- pagination?: Pagination+ pagination?: apiTypes.PaginationalwaysOnFilters?: ApiFilter[]}export type StatsQuery = {date_range: DateRange- relative_date: string | null+ relative_date: RelativeDatefilters: ApiFilter[]dimensions: Dimension[]metrics: Metric[]include: QueryIncludeorder_by?: OrderBy | null- pagination?: Pagination | null+ pagination?: apiTypes.Pagination | null}export function addFilter(@@ -104,7 +102,7 @@ export function createStatsQuery(}}-function createDateRange(dashboardState: DashboardState): DateRange {+export function createDateRange(dashboardState: DashboardState): DateRange {if (dashboardState.period === DashboardPeriod.custom) {return [formatISO(dashboardState.from), formatISO(dashboardState.to)]} else {diff --git a/assets/js/dashboard/stats/breakdowns.tsx b/assets/js/dashboard/stats/breakdowns.tsxindex 249d7785f7..ac01583061 100644--- a/assets/js/dashboard/stats/breakdowns.tsx+++ b/assets/js/dashboard/stats/breakdowns.tsx@@ -13,7 +13,6 @@ import {StatsQuery} from '../stats-query'import { Filter } from '../dashboard-state'-import { MetricsByContext } from './reports/reports-config'import classNames from 'classnames'import { DIRECT_NONE } from './sources'@@ -214,46 +213,6 @@ export function extractMetricValue(return { metricIndex, value, comparison }}-type MetricContext = {- isRealtime: boolean- isDetailed: boolean- hasConversionGoalFilter: boolean- isRevenueAvailable: boolean-}--export const chooseBreakdownMetricsByContext = (- metricsByContext: MetricsByContext,- context: MetricContext-): Metric[] => {- const {- isRealtime,- isDetailed,- hasConversionGoalFilter,- isRevenueAvailable- } = context-- if (hasConversionGoalFilter && isDetailed && isRevenueAvailable) {- return [- ...metricsByContext.goalFilterDetailedMetrics,- 'total_revenue',- 'average_revenue'- ]- }- if (hasConversionGoalFilter && isDetailed) {- return metricsByContext.goalFilterDetailedMetrics- }- if (hasConversionGoalFilter) {- return metricsByContext.goalFilterIndexMetrics- }- if (isRealtime) {- return metricsByContext.realtimeMetrics- }- if (isDetailed) {- return metricsByContext.defaultDetailedMetrics- }- return metricsByContext.defaultIndexMetrics-}-export function addDimensionSearchFilter(statsQuery: StatsQuery,dimension: string,diff --git a/assets/js/dashboard/stats/csv-export/csv-export-body.ts b/assets/js/dashboard/stats/csv-export/csv-export-body.tsnew file mode 100644index 0000000000..860ee609e0--- /dev/null+++ b/assets/js/dashboard/stats/csv-export/csv-export-body.ts@@ -0,0 +1,142 @@+import { DashboardState } from '../../dashboard-state'+import {+ ApiFilter,+ createDateRange,+ DateRange,+ TimeDimension,+ QueryInclude,+ RelativeDate+} from '../../stats-query'+import { formatISO } from '../../util/date'+import {+ hasConversionGoalFilter,+ hasEventFilters,+ hasPageFilter,+ remapToApiFilters+} from '../../util/filters'+import { Interval } from '../graph/intervals'+import { Metric } from '../metrics'+import {+ BREAKDOWN_REPORTS,+ BreakdownReportConfig,+ BreakdownReportKey+} from '../reports/reports-config'++const BREAKDOWN_CSV_REPORTS = {+ 'pages.csv': BreakdownReportKey.pages,+ 'entry_pages.csv': BreakdownReportKey.entryPages,+ 'exit_pages.csv': BreakdownReportKey.exitPages,+ 'browsers.csv': BreakdownReportKey.browsers,+ 'browser_versions.csv': BreakdownReportKey.browserVersions,+ 'operating_systems.csv': BreakdownReportKey.operatingSystems,+ 'operating_system_versions.csv': BreakdownReportKey.operatingSystemVersions,+ 'devices.csv': BreakdownReportKey.screenSizes,+ 'channels.csv': BreakdownReportKey.channels,+ 'sources.csv': BreakdownReportKey.sources,+ 'referrers.csv': BreakdownReportKey.referrers,+ 'utm_mediums.csv': BreakdownReportKey.utmMediums,+ 'utm_sources.csv': BreakdownReportKey.utmSources,+ 'utm_campaigns.csv': BreakdownReportKey.utmCampaigns,+ 'utm_contents.csv': BreakdownReportKey.utmContents,+ 'utm_terms.csv': BreakdownReportKey.utmTerms,+ 'countries.csv': BreakdownReportKey.countries,+ 'regions.csv': BreakdownReportKey.regions,+ 'cities.csv': BreakdownReportKey.cities+}+type CsvFilename = 'visitors.csv' | keyof typeof BREAKDOWN_CSV_REPORTS++type CsvReportParams = {+ dimensions: [TimeDimension] | BreakdownReportConfig['dimensions']+ metrics: Metric[]+ always_on_filters?: BreakdownReportConfig['alwaysOnFilters']+}+type CsvReportsConfig = Record<CsvFilename, CsvReportParams>++export type CsvExportRequestBody = {+ date_range: DateRange+ relative_date: RelativeDate+ filters: ApiFilter[]+ include: Pick<QueryInclude, 'imports'>+ reports: CsvReportsConfig+}++const DEFAULT_VISITORS_CSV_METRICS: Metric[] = [+ 'visitors',+ 'pageviews',+ 'visits',+ 'views_per_visit',+ 'bounce_rate',+ 'visit_duration'+]+const PAGE_FILTERED_VISITORS_CSV_METRICS: Metric[] = [+ 'visitors',+ 'pageviews',+ 'visits',+ 'bounce_rate',+ 'time_on_page',+ 'scroll_depth'+]+const GOAL_FILTERED_VISITORS_CSV_METRICS: Metric[] = [+ 'visitors',+ 'events',+ 'conversion_rate'+]++function csvDimensions(+ reportKey: BreakdownReportKey+): CsvReportParams['dimensions'] {+ switch (reportKey) {+ case BreakdownReportKey.countries:+ return ['visit:country_name']+ case BreakdownReportKey.regions:+ return ['visit:region_name']+ case BreakdownReportKey.cities:+ return ['visit:city_name']+ default:+ return BREAKDOWN_REPORTS[reportKey].dimensions+ }+}++export function createCsvExportRequestBody(+ dashboardState: DashboardState,+ graphInterval: Interval+): CsvExportRequestBody {+ const isGoalFilter = hasConversionGoalFilter(dashboardState)+ const isPageFilter = hasPageFilter(dashboardState)+ const isEventFilter = hasEventFilters(dashboardState)++ const reports: CsvReportsConfig = {+ 'visitors.csv': {+ dimensions: [`time:${graphInterval}`],+ metrics: isGoalFilter+ ? GOAL_FILTERED_VISITORS_CSV_METRICS+ : isPageFilter+ ? PAGE_FILTERED_VISITORS_CSV_METRICS+ : DEFAULT_VISITORS_CSV_METRICS+ },+ ...Object.entries(BREAKDOWN_CSV_REPORTS).reduce(+ (acc, [filename, reportKey]) => {+ const config = BREAKDOWN_REPORTS[reportKey]+ acc[filename as keyof typeof BREAKDOWN_CSV_REPORTS] = {+ dimensions: csvDimensions(reportKey),+ always_on_filters: config.alwaysOnFilters,+ metrics: config.getMetrics({+ isCsv: true,+ hasConversionGoalFilter: isGoalFilter,+ hasEventFilters: isEventFilter+ })+ }+ return acc+ },+ {} as Omit<CsvReportsConfig, 'visitors.csv'>+ )+ }++ return {+ date_range: createDateRange(dashboardState),+ relative_date: dashboardState.date ? formatISO(dashboardState.date) : null,+ filters: remapToApiFilters(dashboardState.filters),+ include: { imports: dashboardState.with_imported },+ reports: reports+ }+}diff --git a/assets/js/dashboard/stats/csv-export/csv-export.test.tsx b/assets/js/dashboard/stats/csv-export/csv-export.test.tsxnew file mode 100644index 0000000000..5f601a0c6b--- /dev/null+++ b/assets/js/dashboard/stats/csv-export/csv-export.test.tsx@@ -0,0 +1,56 @@+import React from 'react'+import { render, screen } from '@testing-library/react'+import userEvent from '@testing-library/user-event'+import { TestContextProviders } from '../../../../test-utils/app-context-providers'+import { CsvExportV2, ExportStatus } from './csv-export'++const exportButton = () => screen.getByRole('button', { name: /export stats/i })++function renderCsvExport({+ exportStatus,+ setExportStatus+}: {+ exportStatus: ExportStatus+ setExportStatus: () => void+}) {+ render(+ <CsvExportV2+ exportStatus={exportStatus}+ setExportStatus={setExportStatus}+ />,+ {+ wrapper: (props) => (+ <TestContextProviders+ {...props}+ siteOptions={{ flags: { dashboard_csv_export_v2: true } }}+ />+ )+ }+ )+}++test('shows spinner while exporting', () => {+ renderCsvExport({+ exportStatus: ExportStatus.exporting,+ setExportStatus: jest.fn()+ })+ expect(exportButton().querySelector('.animate-spin')).toBeInTheDocument()+})++test('does not start a second export while one is in progress', async () => {+ const setExportStatus = jest.fn()+ renderCsvExport({ exportStatus: ExportStatus.exporting, setExportStatus })++ await userEvent.click(exportButton())++ expect(setExportStatus).not.toHaveBeenCalled()+})++test('shows error icon when export fails', () => {+ renderCsvExport({+ exportStatus: ExportStatus.error,+ setExportStatus: jest.fn()+ })++ expect(screen.getByTestId('export-error-icon')).toBeInTheDocument()+})diff --git a/assets/js/dashboard/stats/csv-export/csv-export.tsx b/assets/js/dashboard/stats/csv-export/csv-export.tsxnew file mode 100644index 0000000000..05f52534f0--- /dev/null+++ b/assets/js/dashboard/stats/csv-export/csv-export.tsx@@ -0,0 +1,96 @@+import React from 'react'+import { useSiteContext } from '../../site-context'+import { popover } from '../../components/popover'+import classNames from 'classnames'+import { Spinner } from '../../components/icons'+import {+ ArrowDownTrayIcon,+ ExclamationCircleIcon+} from '@heroicons/react/24/outline'+import { useDashboardStateContext } from '../../dashboard-state-context'+import { useGraphIntervalContext } from '../graph/graph-interval-context'+import { createCsvExportRequestBody } from './csv-export-body'+import * as api from '../../api'+import { DateRange } from '../../stats-query'+import { Tooltip } from '../../util/tooltip'+import { useBodyPortalRef } from '../breakdowns'++export enum ExportStatus {+ idle = 'idle',+ exporting = 'exporting',+ error = 'error'+}++export function CsvExportV2({+ exportStatus,+ setExportStatus+}: {+ exportStatus: ExportStatus+ setExportStatus: (v: ExportStatus) => void+}) {+ const site = useSiteContext()+ const { dashboardState } = useDashboardStateContext()+ const { selectedInterval } = useGraphIntervalContext()++ const canStartExport = exportStatus === ExportStatus.idle++ const startExport = async () => {+ if (!canStartExport) return+ setExportStatus(ExportStatus.exporting)++ try {+ const body = createCsvExportRequestBody(dashboardState, selectedInterval)+ const blob = await api.csvExport(site, body)+ const url = URL.createObjectURL(blob)+ const a = document.createElement('a')+ a.href = url+ a.download = constructFilename(site.domain, body.date_range)+ document.body.appendChild(a)+ a.click()+ document.body.removeChild(a)+ URL.revokeObjectURL(url)+ setExportStatus(ExportStatus.idle)+ } catch {+ setExportStatus(ExportStatus.error)+ }+ }++ return (+ <button+ onClick={startExport}+ className={classNames(+ popover.items.classNames.navigationLink,+ popover.items.classNames.hoverLink,+ !canStartExport && '!cursor-default'+ )}+ >+ <span className="text-sm">Export stats</span>+ {exportStatus === ExportStatus.exporting ? (+ <Spinner className="animate-spin size-4 text-indigo-500" />+ ) : exportStatus === ExportStatus.error ? (+ <ExportFailedBubble />+ ) : (+ <ArrowDownTrayIcon className="size-4 stroke-2" />+ )}+ </button>+ )+}++function ExportFailedBubble() {+ const portalRef = useBodyPortalRef()+ return (+ <Tooltip info={'Export failed'} containerRef={portalRef}>+ <ExclamationCircleIcon+ data-testid="export-error-icon"+ className="size-4 text-gray-500 dark:text-gray-400"+ />+ </Tooltip>+ )+}++function constructFilename(domain: string, dateRange: DateRange) {+ const dateRangeText = Array.isArray(dateRange)+ ? `${dateRange[0]} to ${dateRange[1]}`+ : dateRange+ return `Plausible export ${domain} ${dateRangeText} .zip`+}diff --git a/assets/js/dashboard/stats/devices/details.tsx b/assets/js/dashboard/stats/devices/details.tsxindex cb01df4eae..506324c0e4 100644--- a/assets/js/dashboard/stats/devices/details.tsx+++ b/assets/js/dashboard/stats/devices/details.tsx@@ -6,11 +6,7 @@ import {hasConversionGoalFilter,isRealTimeDashboard} from '../../util/filters'-import {- chooseBreakdownMetricsByContext,- defaultGetFilterInfo,- getScreenFilterInfo-} from '../breakdowns'+import { defaultGetFilterInfo, getScreenFilterInfo } from '../breakdowns'import {BREAKDOWN_REPORTS,BreakdownReportKey@@ -45,15 +41,12 @@ export function DevicesDetails({const isRevenueAvailable =BUILD_EXTRA && revenueAvailable(dashboardState, site)- const metrics = chooseBreakdownMetricsByContext(- reportConfig.metricsByContext,- {- hasConversionGoalFilter: hasConversionGoalFilter(dashboardState),- isRealtime: isRealTimeDashboard(dashboardState),- isDetailed: true,- isRevenueAvailable: isRevenueAvailable- }- )+ const metrics = reportConfig.getMetrics({+ hasConversionGoalFilter: hasConversionGoalFilter(dashboardState),+ isRealtime: isRealTimeDashboard(dashboardState),+ isDetailed: true,+ isRevenueAvailable: isRevenueAvailable+ })const DimensionElement = {[BreakdownReportKey.browsers]: BrowsersDimensionCell,diff --git a/assets/js/dashboard/stats/devices/index.tsx b/assets/js/dashboard/stats/devices/index.tsxindex 7cc262c174..6c985079d6 100644--- a/assets/js/dashboard/stats/devices/index.tsx+++ b/assets/js/dashboard/stats/devices/index.tsx@@ -24,11 +24,7 @@ import {DimensionCellWithBarProps,IndexBreakdown} from '../reports/index-breakdown'-import {- chooseBreakdownMetricsByContext,- defaultGetFilterInfo,- getScreenFilterInfo-} from '../breakdowns'+import { defaultGetFilterInfo, getScreenFilterInfo } from '../breakdowns'import { DashboardState } from '../../dashboard-state'import { BrowserIcon, OsIcon, ScreenSizeIcon } from './icons'@@ -53,15 +49,10 @@ export function Devices() {[BreakdownReportKey.screenSizes]: ScreenSizesDimensionCell}[reportKey]- const metrics = chooseBreakdownMetricsByContext(- reportConfig.metricsByContext,- {- isRealtime: isRealTimeDashboard(dashboardState),- isDetailed: false,- hasConversionGoalFilter: hasConversionGoalFilter(dashboardState),- isRevenueAvailable: false- }- )+ const metrics = reportConfig.getMetrics({+ isRealtime: isRealTimeDashboard(dashboardState),+ hasConversionGoalFilter: hasConversionGoalFilter(dashboardState)+ })function switchTab(tab: TabKey) {storage.setItem(storageKey, tab)diff --git a/assets/js/dashboard/stats/locations/details.tsx b/assets/js/dashboard/stats/locations/details.tsxindex 7d2353bf71..6ef1dc44b0 100644--- a/assets/js/dashboard/stats/locations/details.tsx+++ b/assets/js/dashboard/stats/locations/details.tsx@@ -6,7 +6,6 @@ import {hasConversionGoalFilter,isRealTimeDashboard} from '../../util/filters'-import { chooseBreakdownMetricsByContext } from '../breakdowns'import {BREAKDOWN_REPORTS,BreakdownReportKey@@ -38,15 +37,12 @@ export function LocationsDetails({const isRevenueAvailable =BUILD_EXTRA && revenueAvailable(dashboardState, site)- const metrics = chooseBreakdownMetricsByContext(- reportConfig.metricsByContext,- {- hasConversionGoalFilter: hasConversionGoalFilter(dashboardState),- isRealtime: isRealTimeDashboard(dashboardState),- isDetailed: true,- isRevenueAvailable: isRevenueAvailable- }- )+ const metrics = reportConfig.getMetrics({+ hasConversionGoalFilter: hasConversionGoalFilter(dashboardState),+ isRealtime: isRealTimeDashboard(dashboardState),+ isDetailed: true,+ isRevenueAvailable: isRevenueAvailable+ })const DimensionElement = DIMENSION_ELEMENTS[reportKey]diff --git a/assets/js/dashboard/stats/locations/index.tsx b/assets/js/dashboard/stats/locations/index.tsxindex a899dcf7cd..501ebf00db 100644--- a/assets/js/dashboard/stats/locations/index.tsx+++ b/assets/js/dashboard/stats/locations/index.tsx@@ -26,7 +26,6 @@ import {DimensionCellWithBarProps,IndexBreakdown} from '../reports/index-breakdown'-import { chooseBreakdownMetricsByContext } from '../breakdowns'import { FilterInfo } from '../../components/drilldown-link'import { NonTimeDimension } from '../../stats-query'import { FlagEmoji } from './flag-emoji'@@ -124,15 +123,10 @@ export function Locations() {tab === 'map' ? BreakdownReportKey.countries : tabconst reportConfig = BREAKDOWN_REPORTS[selectedListKey]- const metrics = chooseBreakdownMetricsByContext(- reportConfig.metricsByContext,- {- isRealtime: isRealTimeDashboard(dashboardState),- isDetailed: false,- hasConversionGoalFilter: hasConversionGoalFilter(dashboardState),- isRevenueAvailable: false- }- )+ const metrics = reportConfig.getMetrics({+ isRealtime: isRealTimeDashboard(dashboardState),+ hasConversionGoalFilter: hasConversionGoalFilter(dashboardState)+ })const moreLinkState = currentData? currentData.results.length > 0diff --git a/assets/js/dashboard/stats/modals/details-breakdown.test.tsx b/assets/js/dashboard/stats/modals/details-breakdown.test.tsxindex 6083bd6eb3..37dc53e256 100644--- a/assets/js/dashboard/stats/modals/details-breakdown.test.tsx+++ b/assets/js/dashboard/stats/modals/details-breakdown.test.tsx@@ -17,8 +17,12 @@ const queryPath = `/api/stats/${domain}/query/`const MOCK_RESPONSE_DELAY_MS = 50const SMALL_LOADING_SPINNER_TEST_ID = 'small-loading-spinner'-const PAGES_DETAILED_METRICS =- BREAKDOWN_REPORTS.pages.metricsByContext.defaultDetailedMetrics+const PAGES_DETAILED_METRICS = BREAKDOWN_REPORTS.pages.getMetrics({+ isDetailed: true,+ isRealtime: false,+ hasConversionGoalFilter: false,+ isRevenueAvailable: false+})function buildResponse(results: { dimensions: string[]; metrics: number[] }[] = []diff --git a/assets/js/dashboard/stats/pages/details.tsx b/assets/js/dashboard/stats/pages/details.tsxindex 0616043edf..0852e325fe 100644--- a/assets/js/dashboard/stats/pages/details.tsx+++ b/assets/js/dashboard/stats/pages/details.tsx@@ -7,10 +7,7 @@ import {hasEventFilters,isRealTimeDashboard} from '../../util/filters'-import {- chooseBreakdownMetricsByContext,- defaultGetFilterInfo-} from '../breakdowns'+import { defaultGetFilterInfo } from '../breakdowns'import {BREAKDOWN_REPORTS,BreakdownReportKey@@ -38,17 +35,14 @@ export function PagesDetails({const isRevenueAvailable =BUILD_EXTRA && revenueAvailable(dashboardState, site)- let metrics = chooseBreakdownMetricsByContext(reportConfig.metricsByContext, {+ const metrics = reportConfig.getMetrics({hasConversionGoalFilter: hasConversionGoalFilter(dashboardState),isRealtime: isRealTimeDashboard(dashboardState),isDetailed: true,- isRevenueAvailable: isRevenueAvailable+ isRevenueAvailable: isRevenueAvailable,+ hasEventFilters: hasEventFilters(dashboardState)})- if (metrics.includes('exit_rate') && hasEventFilters(dashboardState)) {- metrics = metrics.filter((m) => m !== 'exit_rate')- }-return (<Modal><DetailsBreakdowndiff --git a/assets/js/dashboard/stats/pages/index.tsx b/assets/js/dashboard/stats/pages/index.tsxindex 980a2f3949..e099f84554 100644--- a/assets/js/dashboard/stats/pages/index.tsx+++ b/assets/js/dashboard/stats/pages/index.tsx@@ -23,10 +23,7 @@ import {IndexBreakdown,DimensionCellWithBarProps} from '../reports/index-breakdown'-import {- chooseBreakdownMetricsByContext,- defaultGetFilterInfo-} from '../breakdowns'+import { defaultGetFilterInfo } from '../breakdowns'import { externalLinkForPage, trimURL } from '../../util/url'import { IndexExternalLink } from './external-link'@@ -44,15 +41,10 @@ export default function Pages() {const reportKey = getReportKey(tab)const reportConfig = BREAKDOWN_REPORTS[reportKey]- const metrics = chooseBreakdownMetricsByContext(- reportConfig.metricsByContext,- {- isRealtime: isRealTimeDashboard(dashboardState),- isDetailed: false,- hasConversionGoalFilter: hasConversionGoalFilter(dashboardState),- isRevenueAvailable: false- }- )+ const metrics = reportConfig.getMetrics({+ isRealtime: isRealTimeDashboard(dashboardState),+ hasConversionGoalFilter: hasConversionGoalFilter(dashboardState)+ })function switchTab(tab: TabKey) {storage.setItem(storageKey, tab)diff --git a/assets/js/dashboard/stats/reports/reports-config.ts b/assets/js/dashboard/stats/reports/reports-config.tsindex 3d37e6ad3b..d89ac4d9f0 100644--- a/assets/js/dashboard/stats/reports/reports-config.ts+++ b/assets/js/dashboard/stats/reports/reports-config.ts@@ -1,24 +1,35 @@import { ApiFilter, NonTimeDimension } from '../../stats-query'import { Metric } from '../metrics'+export type MetricContext = {+ hasConversionGoalFilter: boolean+ isRealtime?: boolean+ isCsv?: boolean+ isDetailed?: boolean+ isRevenueAvailable?: boolean+ hasEventFilters?: boolean+}+export type MetricsByContext = {realtimeMetrics: Metric[]defaultIndexMetrics: Metric[]defaultDetailedMetrics: Metric[]+ defaultCsvMetrics: Metric[]goalFilterIndexMetrics: Metric[]goalFilterDetailedMetrics: Metric[]+ goalFilterCsvMetrics: Metric[]}export type BreakdownReportConfig = {dimensions: [NonTimeDimension, ...NonTimeDimension[]]- metricsByContext: MetricsByContext+ getMetrics: (context: MetricContext) => Metric[]detailsTitle: stringdetailsPath: stringdimensionLabel: stringalwaysOnFilters?: ApiFilter[]}-const COMMON_METRICS_BY_CONTEXT: MetricsByContext = {+export const COMMON_BREAKDOWN_METRICS_BY_CONTEXT: MetricsByContext = {realtimeMetrics: ['visitors', 'percentage'],defaultIndexMetrics: ['visitors', 'percentage'],defaultDetailedMetrics: [@@ -27,12 +38,56 @@ const COMMON_METRICS_BY_CONTEXT: MetricsByContext = {'bounce_rate','visit_duration'],+ defaultCsvMetrics: ['visitors', 'bounce_rate', 'visit_duration'],goalFilterIndexMetrics: ['visitors', 'group_conversion_rate'],goalFilterDetailedMetrics: ['total_visitors','visitors','group_conversion_rate'- ]+ ],+ goalFilterCsvMetrics: ['visitors', 'group_conversion_rate']+}++function chooseMetrics(mbc: MetricsByContext, ctx: MetricContext): Metric[] {+ const {+ isRealtime,+ isCsv,+ isDetailed,+ hasConversionGoalFilter,+ isRevenueAvailable+ } = ctx+ if (isCsv && hasConversionGoalFilter) {+ return mbc.goalFilterCsvMetrics+ }+ if (isCsv) {+ return mbc.defaultCsvMetrics+ }+ if (hasConversionGoalFilter && isDetailed && isRevenueAvailable) {+ return [+ ...mbc.goalFilterDetailedMetrics,+ 'total_revenue',+ 'average_revenue'+ ]+ }+ if (hasConversionGoalFilter && isDetailed) {+ return mbc.goalFilterDetailedMetrics+ }+ if (hasConversionGoalFilter) {+ return mbc.goalFilterIndexMetrics+ }+ if (isRealtime) {+ return mbc.realtimeMetrics+ }+ if (isDetailed) {+ return mbc.defaultDetailedMetrics+ }+ return mbc.defaultIndexMetrics+}++function createGetMetricsFn(+ mbc: MetricsByContext+): (context: MetricContext) => Metric[] {+ return (ctx) => chooseMetrics(mbc, ctx)}export enum BreakdownReportKey {@@ -63,8 +118,8 @@ export const BREAKDOWN_REPORTS: Record<> = {[BreakdownReportKey.pages]: {dimensions: ['event:page'],- metricsByContext: {- ...COMMON_METRICS_BY_CONTEXT,+ getMetrics: createGetMetricsFn({+ ...COMMON_BREAKDOWN_METRICS_BY_CONTEXT,defaultDetailedMetrics: ['visitors','percentage',@@ -72,24 +127,32 @@ export const BREAKDOWN_REPORTS: Record<'bounce_rate','time_on_page','scroll_depth'+ ],+ defaultCsvMetrics: [+ 'visitors',+ 'pageviews',+ 'bounce_rate',+ 'time_on_page',+ 'scroll_depth']- },+ }),detailsTitle: 'Top pages',detailsPath: 'pages',dimensionLabel: 'Page'},[BreakdownReportKey.entryPages]: {dimensions: ['visit:entry_page'],- metricsByContext: {- ...COMMON_METRICS_BY_CONTEXT,+ getMetrics: createGetMetricsFn({+ ...COMMON_BREAKDOWN_METRICS_BY_CONTEXT,defaultDetailedMetrics: ['visitors','percentage','visits','bounce_rate','visit_duration'- ]- },+ ],+ defaultCsvMetrics: ['visitors', 'visits', 'bounce_rate', 'visit_duration']+ }),detailsTitle: 'Entry pages',detailsPath: 'entry-pages',dimensionLabel: 'Entry page',@@ -97,9 +160,22 @@ export const BREAKDOWN_REPORTS: Record<},[BreakdownReportKey.exitPages]: {dimensions: ['visit:exit_page'],- metricsByContext: {- ...COMMON_METRICS_BY_CONTEXT,- defaultDetailedMetrics: ['visitors', 'percentage', 'visits', 'exit_rate']+ getMetrics: (ctx) => {+ const base = chooseMetrics(+ {+ ...COMMON_BREAKDOWN_METRICS_BY_CONTEXT,+ defaultDetailedMetrics: [+ 'visitors',+ 'percentage',+ 'visits',+ 'exit_rate'+ ],+ defaultCsvMetrics: ['visitors', 'visits', 'exit_rate']+ },+ ctx+ )++ return ctx.hasEventFilters ? base.filter((m) => m !== 'exit_rate') : base},detailsTitle: 'Exit pages',detailsPath: 'exit-pages',@@ -108,63 +184,78 @@ export const BREAKDOWN_REPORTS: Record<},[BreakdownReportKey.browsers]: {dimensions: ['visit:browser'],- metricsByContext: COMMON_METRICS_BY_CONTEXT,+ getMetrics: createGetMetricsFn({+ ...COMMON_BREAKDOWN_METRICS_BY_CONTEXT,+ defaultCsvMetrics: ['visitors']+ }),detailsTitle: 'Browsers',detailsPath: 'browsers',dimensionLabel: 'Browser'},[BreakdownReportKey.browserVersions]: {dimensions: ['visit:browser_version', 'visit:browser'],- metricsByContext: COMMON_METRICS_BY_CONTEXT,+ getMetrics: createGetMetricsFn({+ ...COMMON_BREAKDOWN_METRICS_BY_CONTEXT,+ defaultCsvMetrics: ['visitors']+ }),detailsTitle: 'Browser versions',detailsPath: 'browser-versions',dimensionLabel: 'Browser version'},[BreakdownReportKey.operatingSystems]: {dimensions: ['visit:os'],- metricsByContext: COMMON_METRICS_BY_CONTEXT,+ getMetrics: createGetMetricsFn({+ ...COMMON_BREAKDOWN_METRICS_BY_CONTEXT,+ defaultCsvMetrics: ['visitors']+ }),detailsTitle: 'Operating systems',detailsPath: 'operating-systems',dimensionLabel: 'Operating system'},[BreakdownReportKey.operatingSystemVersions]: {dimensions: ['visit:os_version', 'visit:os'],- metricsByContext: COMMON_METRICS_BY_CONTEXT,+ getMetrics: createGetMetricsFn({+ ...COMMON_BREAKDOWN_METRICS_BY_CONTEXT,+ defaultCsvMetrics: ['visitors']+ }),detailsTitle: 'Operating system versions',detailsPath: 'operating-system-versions',dimensionLabel: 'Operating system version'},[BreakdownReportKey.screenSizes]: {dimensions: ['visit:device'],- metricsByContext: COMMON_METRICS_BY_CONTEXT,+ getMetrics: createGetMetricsFn({+ ...COMMON_BREAKDOWN_METRICS_BY_CONTEXT,+ defaultCsvMetrics: ['visitors']+ }),detailsTitle: 'Devices',detailsPath: 'screen-sizes',dimensionLabel: 'Device'},[BreakdownReportKey.channels]: {dimensions: ['visit:channel'],- metricsByContext: COMMON_METRICS_BY_CONTEXT,+ getMetrics: createGetMetricsFn(COMMON_BREAKDOWN_METRICS_BY_CONTEXT),detailsTitle: 'Top acquisition channels',detailsPath: 'channels',dimensionLabel: 'Channel'},[BreakdownReportKey.sources]: {dimensions: ['visit:source'],- metricsByContext: COMMON_METRICS_BY_CONTEXT,+ getMetrics: createGetMetricsFn(COMMON_BREAKDOWN_METRICS_BY_CONTEXT),detailsTitle: 'Top sources',detailsPath: 'sources',dimensionLabel: 'Source'},[BreakdownReportKey.referrers]: {dimensions: ['visit:referrer'],- metricsByContext: COMMON_METRICS_BY_CONTEXT,+ getMetrics: createGetMetricsFn(COMMON_BREAKDOWN_METRICS_BY_CONTEXT),detailsTitle: 'Referrer drilldown',detailsPath: 'referrers/:referrer',dimensionLabel: 'Referrer'},[BreakdownReportKey.utmMediums]: {dimensions: ['visit:utm_medium'],- metricsByContext: COMMON_METRICS_BY_CONTEXT,+ getMetrics: createGetMetricsFn(COMMON_BREAKDOWN_METRICS_BY_CONTEXT),detailsTitle: 'UTM mediums',detailsPath: 'utm_mediums',dimensionLabel: 'UTM medium',@@ -172,7 +263,7 @@ export const BREAKDOWN_REPORTS: Record<},[BreakdownReportKey.utmSources]: {dimensions: ['visit:utm_source'],- metricsByContext: COMMON_METRICS_BY_CONTEXT,+ getMetrics: createGetMetricsFn(COMMON_BREAKDOWN_METRICS_BY_CONTEXT),detailsTitle: 'UTM sources',detailsPath: 'utm_sources',dimensionLabel: 'UTM source',@@ -180,7 +271,7 @@ export const BREAKDOWN_REPORTS: Record<},[BreakdownReportKey.utmCampaigns]: {dimensions: ['visit:utm_campaign'],- metricsByContext: COMMON_METRICS_BY_CONTEXT,+ getMetrics: createGetMetricsFn(COMMON_BREAKDOWN_METRICS_BY_CONTEXT),detailsTitle: 'UTM campaigns',detailsPath: 'utm_campaigns',dimensionLabel: 'UTM campaign',@@ -188,7 +279,7 @@ export const BREAKDOWN_REPORTS: Record<},[BreakdownReportKey.utmContents]: {dimensions: ['visit:utm_content'],- metricsByContext: COMMON_METRICS_BY_CONTEXT,+ getMetrics: createGetMetricsFn(COMMON_BREAKDOWN_METRICS_BY_CONTEXT),detailsTitle: 'UTM contents',detailsPath: 'utm_contents',dimensionLabel: 'UTM content',@@ -196,7 +287,7 @@ export const BREAKDOWN_REPORTS: Record<},[BreakdownReportKey.utmTerms]: {dimensions: ['visit:utm_term'],- metricsByContext: COMMON_METRICS_BY_CONTEXT,+ getMetrics: createGetMetricsFn(COMMON_BREAKDOWN_METRICS_BY_CONTEXT),detailsTitle: 'UTM terms',detailsPath: 'utm_terms',dimensionLabel: 'UTM term',@@ -204,10 +295,11 @@ export const BREAKDOWN_REPORTS: Record<},[BreakdownReportKey.countries]: {dimensions: ['visit:country_name', 'visit:country'],- metricsByContext: {- ...COMMON_METRICS_BY_CONTEXT,- defaultDetailedMetrics: ['visitors', 'percentage']- },+ getMetrics: createGetMetricsFn({+ ...COMMON_BREAKDOWN_METRICS_BY_CONTEXT,+ defaultDetailedMetrics: ['visitors', 'percentage'],+ defaultCsvMetrics: ['visitors']+ }),detailsTitle: 'Top countries',detailsPath: 'countries',dimensionLabel: 'Country',@@ -216,10 +308,11 @@ export const BREAKDOWN_REPORTS: Record<[BreakdownReportKey.regions]: {// the 3rd dimension "visit:country" is needed to render the country flagdimensions: ['visit:region_name', 'visit:region', 'visit:country'],- metricsByContext: {- ...COMMON_METRICS_BY_CONTEXT,- defaultDetailedMetrics: ['visitors', 'percentage']- },+ getMetrics: createGetMetricsFn({+ ...COMMON_BREAKDOWN_METRICS_BY_CONTEXT,+ defaultDetailedMetrics: ['visitors', 'percentage'],+ defaultCsvMetrics: ['visitors']+ }),detailsTitle: 'Top regions',detailsPath: 'regions',dimensionLabel: 'Region',@@ -228,10 +321,11 @@ export const BREAKDOWN_REPORTS: Record<[BreakdownReportKey.cities]: {// the 3rd dimension "visit:country" is needed to render the country flagdimensions: ['visit:city_name', 'visit:city', 'visit:country'],- metricsByContext: {- ...COMMON_METRICS_BY_CONTEXT,- defaultDetailedMetrics: ['visitors', 'percentage']- },+ getMetrics: createGetMetricsFn({+ ...COMMON_BREAKDOWN_METRICS_BY_CONTEXT,+ defaultDetailedMetrics: ['visitors', 'percentage'],+ defaultCsvMetrics: ['visitors']+ }),detailsTitle: 'Top cities',detailsPath: 'cities',dimensionLabel: 'City',diff --git a/assets/js/dashboard/stats/sources/details.tsx b/assets/js/dashboard/stats/sources/details.tsxindex 25447b485e..741e9d3cdb 100644--- a/assets/js/dashboard/stats/sources/details.tsx+++ b/assets/js/dashboard/stats/sources/details.tsx@@ -6,11 +6,7 @@ import {hasConversionGoalFilter,isRealTimeDashboard} from '../../util/filters'-import {- chooseBreakdownMetricsByContext,- defaultGetFilterInfo,- getReferrerUrlFilterInfo-} from '../breakdowns'+import { defaultGetFilterInfo, getReferrerUrlFilterInfo } from '../breakdowns'import {BREAKDOWN_REPORTS,BreakdownReportKey@@ -45,15 +41,12 @@ export function SourcesDetails({ reportKey }: { reportKey: SourcesReportKey }) {const isRevenueAvailable =BUILD_EXTRA && revenueAvailable(dashboardState, site)- const metrics = chooseBreakdownMetricsByContext(- reportConfig.metricsByContext,- {- hasConversionGoalFilter: hasConversionGoalFilter(dashboardState),- isRealtime: isRealTimeDashboard(dashboardState),- isDetailed: true,- isRevenueAvailable: isRevenueAvailable- }- )+ const metrics = reportConfig.getMetrics({+ hasConversionGoalFilter: hasConversionGoalFilter(dashboardState),+ isRealtime: isRealTimeDashboard(dashboardState),+ isDetailed: true,+ isRevenueAvailable: isRevenueAvailable+ })const DimensionElement =reportKey === BreakdownReportKey.sourcesdiff --git a/assets/js/dashboard/stats/sources/index.tsx b/assets/js/dashboard/stats/sources/index.tsxindex b50c34baf3..087e0fe546 100644--- a/assets/js/dashboard/stats/sources/index.tsx+++ b/assets/js/dashboard/stats/sources/index.tsx@@ -28,10 +28,7 @@ import {DimensionCellWithBarProps,IndexBreakdown} from '../reports/index-breakdown'-import {- chooseBreakdownMetricsByContext,- defaultGetFilterInfo-} from '../breakdowns'+import { defaultGetFilterInfo } from '../breakdowns'import ImportedWarningBubble from '../imported-warning-bubble'import { IndexExternalLink } from '../pages/external-link'import { SearchTerms } from './search-terms'@@ -193,15 +190,10 @@ export default function Sources() {const reportConfig = BREAKDOWN_REPORTS[currentReportKey]- const metrics = chooseBreakdownMetricsByContext(- reportConfig.metricsByContext,- {- isRealtime: isRealTimeDashboard(dashboardState),- isDetailed: false,- hasConversionGoalFilter: hasConversionGoalFilter(dashboardState),- isRevenueAvailable: false- }- )+ const metrics = reportConfig.getMetrics({+ isRealtime: isRealTimeDashboard(dashboardState),+ hasConversionGoalFilter: hasConversionGoalFilter(dashboardState)+ })const DimensionElement = {[BreakdownReportKey.channels]: ChannelsDimensionCell,diff --git a/assets/test-utils/app-context-providers.tsx b/assets/test-utils/app-context-providers.tsxindex e0fcd5588b..f0b2f59270 100644--- a/assets/test-utils/app-context-providers.tsx+++ b/assets/test-utils/app-context-providers.tsx@@ -47,7 +47,7 @@ export const DEFAULT_SITE: PlausibleSite = {embedded: false,background: '',isDbip: false,- flags: {},+ flags: { dashboard_csv_export_v2: false },shared: false,isConsolidatedView: false}diff --git a/e2e/package-lock.json b/e2e/package-lock.jsonindex 4cbd79a532..e8d62c33f0 100644--- a/e2e/package-lock.json+++ b/e2e/package-lock.json@@ -15,6 +15,7 @@"eslint": "^9.39.2","eslint-config-prettier": "^10.1.8","eslint-plugin-playwright": "^2.5.1",+ "fflate": "^0.8.0","prettier": "^3.8.1","typescript": "^5.9.3","typescript-eslint": "^8.55.0"@@ -1007,6 +1008,13 @@}}},+ "node_modules/fflate": {+ "version": "0.8.3",+ "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.3.tgz",+ "integrity": "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==",+ "dev": true,+ "license": "MIT"+ },"node_modules/file-entry-cache": {"version": "8.0.0","resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz",diff --git a/e2e/package.json b/e2e/package.jsonindex c02121e332..04b77d6e55 100644--- a/e2e/package.json+++ b/e2e/package.json@@ -10,6 +10,7 @@"license": "MIT","devDependencies": {"@eslint/js": "^9.39.2",+ "fflate": "^0.8.0","@js-joda/core": "^5.7.0","@js-joda/locale_en-us": "^4.15.3","@playwright/test": "^1.58.0",diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.tsindex d9e9e062ae..84408c4989 100644--- a/e2e/playwright.config.ts+++ b/e2e/playwright.config.ts@@ -23,6 +23,8 @@ export default defineConfig({/* Base URL to use in actions like `await page.goto('')`. */baseURL,+ // launchOptions: {slowMo: 500},+/* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */trace: 'on-first-retry'},diff --git a/e2e/tests/dashboard/csv-export.spec.ts b/e2e/tests/dashboard/csv-export.spec.tsnew file mode 100644index 0000000000..dc87e89344--- /dev/null+++ b/e2e/tests/dashboard/csv-export.spec.ts@@ -0,0 +1,369 @@+import { test, expect } from '@playwright/test'+import type { Page } from '@playwright/test'+import { unzipSync } from 'fflate'+import * as fs from 'fs'+import {+ setupSite,+ populateStats,+ enableDashboardCsvExportV2,+ addGoal,+ addCustomProp,+ StatsEntry+} from '../fixtures'++const EXPECTED_HEADERS = {+ defaultVisitorsCsv: [+ 'date',+ 'visitors',+ 'pageviews',+ 'visits',+ 'views_per_visit',+ 'bounce_rate',+ 'visit_duration'+ ],+ pageFilteredVisitorsCsv: [+ 'date',+ 'visitors',+ 'pageviews',+ 'visits',+ 'bounce_rate',+ 'time_on_page',+ 'scroll_depth'+ ],+ goalFilteredVisitorsCsv: [+ 'date',+ 'unique_conversions',+ 'total_conversions',+ 'conversion_rate'+ ],+ pagesBreakdown: [+ 'name',+ 'visitors',+ 'pageviews',+ 'bounce_rate',+ 'time_on_page',+ 'scroll_depth'+ ],+ entryPagesBreakdown: [+ 'name',+ 'unique_entrances',+ 'total_entrances',+ 'bounce_rate',+ 'visit_duration'+ ],+ exitPagesBreakdown: ['name', 'unique_exits', 'total_exits', 'exit_rate'],+ exitPagesNoExitRate: ['name', 'unique_exits', 'total_exits'],+ commonBreakdown: ['name', 'visitors', 'bounce_rate', 'visit_duration'],+ commonGoalFilteredBreakdown: ['name', 'conversions', 'conversion_rate'],+ nameAndVisitors: ['name', 'visitors'],+ nameVersionAndVisitors: ['name', 'version', 'visitors'],+ nameVersionGoalFiltered: ['name', 'version', 'conversions', 'conversion_rate']+}++const UTM_AND_SOURCE_REPORTS = [+ 'sources.csv',+ 'channels.csv',+ 'referrers.csv',+ 'utm_mediums.csv',+ 'utm_sources.csv',+ 'utm_campaigns.csv',+ 'utm_contents.csv',+ 'utm_terms.csv'+]++const SIMPLE_LOCATION_AND_DEVICE_REPORTS = [+ 'countries.csv',+ 'regions.csv',+ 'cities.csv',+ 'browsers.csv',+ 'operating_systems.csv',+ 'devices.csv'+]++const VERSION_REPORTS = [+ 'browser_versions.csv',+ 'operating_system_versions.csv'+]++function parseAllCsvs(+ files: Record<string, string>+): Record<string, string[][]> {+ return Object.fromEntries(+ Object.entries(files).map(([filename, content]) => [+ filename,+ content+ .split(/\r?\n/)+ .filter((row) => !!row.length)+ .map((line) => line.split(','))+ ])+ )+}++function getCsv(+ csvs: Record<string, string[][]>,+ filename: string+): string[][] {+ const rows = csvs[filename]+ if (rows === undefined) throw new Error(`Expected ${filename} in CSV export`)+ return rows+}++test('csv export column headers match expected metrics for each report', async ({+ page,+ request+}) => {+ const { domain } = await setupSite({ page, request })+ await enableDashboardCsvExportV2({ request, domain })+ await addCustomProp({ page, domain, name: 'author' })++ await populateStats({+ request,+ domain,+ events: [{ name: 'pageview', timestamp: '2021-01-01 12:00:00' }]+ })++ await test.step('without any filters', async () => {+ await page.goto(`/${domain}`, {+ waitUntil: 'commit'+ })++ const csvs = parseAllCsvs(await triggerExportAndAwaitDownload(page))++ expect(getCsv(csvs, 'visitors.csv')[0]).toEqual(+ EXPECTED_HEADERS.defaultVisitorsCsv+ )+ expect(getCsv(csvs, 'pages.csv')[0]).toEqual(+ EXPECTED_HEADERS.pagesBreakdown+ )+ expect(getCsv(csvs, 'entry_pages.csv')[0]).toEqual(+ EXPECTED_HEADERS.entryPagesBreakdown+ )+ expect(getCsv(csvs, 'exit_pages.csv')[0]).toEqual(+ EXPECTED_HEADERS.exitPagesBreakdown+ )++ for (const filename of UTM_AND_SOURCE_REPORTS) {+ expect(getCsv(csvs, filename)[0]).toEqual(+ EXPECTED_HEADERS.commonBreakdown+ )+ }+ for (const filename of SIMPLE_LOCATION_AND_DEVICE_REPORTS) {+ expect(getCsv(csvs, filename)[0]).toEqual(+ EXPECTED_HEADERS.nameAndVisitors+ )+ }+ for (const filename of VERSION_REPORTS) {+ expect(getCsv(csvs, filename)[0]).toEqual(+ EXPECTED_HEADERS.nameVersionAndVisitors+ )+ }+ })++ await test.step('with a page filter', async () => {+ await page.goto(`/${domain}?f=is,page,/blog`, {+ waitUntil: 'commit'+ })++ const csvs = parseAllCsvs(await triggerExportAndAwaitDownload(page))++ expect(getCsv(csvs, 'visitors.csv')[0]).toEqual(+ EXPECTED_HEADERS.pageFilteredVisitorsCsv+ )+ expect(getCsv(csvs, 'pages.csv')[0]).toEqual(+ EXPECTED_HEADERS.pagesBreakdown+ )+ expect(getCsv(csvs, 'entry_pages.csv')[0]).toEqual(+ EXPECTED_HEADERS.entryPagesBreakdown+ )+ expect(getCsv(csvs, 'exit_pages.csv')[0]).toEqual(+ EXPECTED_HEADERS.exitPagesNoExitRate+ )++ for (const filename of UTM_AND_SOURCE_REPORTS) {+ expect(getCsv(csvs, filename)[0]).toEqual(+ EXPECTED_HEADERS.commonBreakdown+ )+ }+ for (const filename of SIMPLE_LOCATION_AND_DEVICE_REPORTS) {+ expect(getCsv(csvs, filename)[0]).toEqual(+ EXPECTED_HEADERS.nameAndVisitors+ )+ }+ for (const filename of VERSION_REPORTS) {+ expect(getCsv(csvs, filename)[0]).toEqual(+ EXPECTED_HEADERS.nameVersionAndVisitors+ )+ }+ })++ await test.step('with a goal filter', async () => {+ await addGoal({ request, domain, params: { event_name: 'Signup' } })++ await page.goto(`/${domain}?f=is,goal,Signup`, {+ waitUntil: 'commit'+ })++ const csvs = parseAllCsvs(await triggerExportAndAwaitDownload(page))++ expect(getCsv(csvs, 'visitors.csv')[0]).toEqual(+ EXPECTED_HEADERS.goalFilteredVisitorsCsv+ )++ for (const filename of [+ 'pages.csv',+ 'entry_pages.csv',+ 'exit_pages.csv',+ ...UTM_AND_SOURCE_REPORTS,+ ...SIMPLE_LOCATION_AND_DEVICE_REPORTS+ ]) {+ expect(getCsv(csvs, filename)[0]).toEqual(+ EXPECTED_HEADERS.commonGoalFilteredBreakdown+ )+ }+ for (const filename of VERSION_REPORTS) {+ expect(getCsv(csvs, filename)[0]).toEqual(+ EXPECTED_HEADERS.nameVersionGoalFiltered+ )+ }+ })++ await test.step('with a custom prop filter', async () => {+ await page.goto(`/${domain}?f=is,props:author,john`, {+ waitUntil: 'commit'+ })++ const csvs = parseAllCsvs(await triggerExportAndAwaitDownload(page))++ expect(getCsv(csvs, 'visitors.csv')[0]).toEqual(+ EXPECTED_HEADERS.defaultVisitorsCsv+ )+ expect(getCsv(csvs, 'pages.csv')[0]).toEqual(+ EXPECTED_HEADERS.pagesBreakdown+ )+ expect(getCsv(csvs, 'entry_pages.csv')[0]).toEqual(+ EXPECTED_HEADERS.entryPagesBreakdown+ )+ expect(getCsv(csvs, 'exit_pages.csv')[0]).toEqual(+ EXPECTED_HEADERS.exitPagesNoExitRate+ )++ for (const filename of UTM_AND_SOURCE_REPORTS) {+ expect(getCsv(csvs, filename)[0]).toEqual(+ EXPECTED_HEADERS.commonBreakdown+ )+ }+ for (const filename of SIMPLE_LOCATION_AND_DEVICE_REPORTS) {+ expect(getCsv(csvs, filename)[0]).toEqual(+ EXPECTED_HEADERS.nameAndVisitors+ )+ }+ for (const filename of VERSION_REPORTS) {+ expect(getCsv(csvs, filename)[0]).toEqual(+ EXPECTED_HEADERS.nameVersionAndVisitors+ )+ }+ })+})++test('filters out empty visit:* dimension values', async ({+ page,+ request+}) => {+ const { domain } = await setupSite({ page, request })+ await enableDashboardCsvExportV2({ request, domain })++ const base: StatsEntry = {+ name: 'pageview',+ pathname: '/',+ timestamp: '2021-01-01 12:00:00',+ utm_medium: 't',+ utm_source: 't',+ utm_campaign: 't',+ utm_content: 't',+ utm_term: 't',+ country_code: 'EE',+ subdivision1_code: 'EE-37',+ city_geoname_id: 588_409+ }++ await populateStats({+ request,+ domain,+ events: [+ // session with empty entry page — excluded from entry_pages.csv+ { ...base, user_id: 1, pathname: '' },+ { ...base, user_id: 1, pathname: '/', timestamp: '2021-01-01 12:01:00' },+ // session with empty exit page — excluded from exit_pages.csv+ { ...base, user_id: 2, pathname: '/' },+ { ...base, user_id: 2, pathname: '', timestamp: '2021-01-01 12:01:00' },+ // one bad utm_* per session — each excluded from its own report+ { ...base, user_id: 3, utm_medium: '' },+ { ...base, user_id: 4, utm_source: '' },+ { ...base, user_id: 5, utm_campaign: '' },+ { ...base, user_id: 6, utm_content: '' },+ { ...base, user_id: 7, utm_term: '' },+ // one bad location per session — each excluded from its own report+ { ...base, user_id: 8, country_code: 'ZZ' },+ { ...base, user_id: 9, subdivision1_code: '' },+ { ...base, user_id: 10, city_geoname_id: 0 }+ ]+ })++ await page.goto(`/${domain}?period=day&date=2021-01-01`, {+ waitUntil: 'commit'+ })++ const csvs = parseAllCsvs(await triggerExportAndAwaitDownload(page))++ // assert that each+ for (const filename of [+ 'entry_pages.csv',+ 'exit_pages.csv',+ 'utm_mediums.csv',+ 'utm_sources.csv',+ 'utm_campaigns.csv',+ 'utm_contents.csv',+ 'utm_terms.csv',+ 'countries.csv',+ 'regions.csv',+ 'cities.csv'+ ]) {+ await test.step(filename, async () => {+ expect(getCsv(csvs, filename).slice(1)).toHaveLength(1)+ })+ }+})++async function triggerExportAndAwaitDownload(+ page: Page+): Promise<Record<string, string>> {+ await page.getByTestId('dashboard-options-menu').click()+ await page.getByText('Export stats').waitFor({ state: 'visible' })++ const [result] = await Promise.all([+ Promise.race([+ page+ .waitForEvent('download')+ .then((d) => ({ ok: true as const, download: d })),+ page+ .waitForEvent('pageerror')+ .then((e) => ({ ok: false as const, error: e }))+ ]),+ page.getByText('Export stats').click()+ ])++ if (!result.ok) throw result.error+ const { download } = result++ const zipPath = await download.path()+ if (!zipPath) throw new Error('Download path unavailable')++ const files = unzipSync(new Uint8Array(fs.readFileSync(zipPath)))++ return Object.fromEntries(+ Object.entries(files).map(([filename, data]) => [+ filename,+ new TextDecoder().decode(data)+ ])+ )+}diff --git a/e2e/tests/fixtures.ts b/e2e/tests/fixtures.tsindex 0d15f21b93..7537118e71 100644--- a/e2e/tests/fixtures.ts+++ b/e2e/tests/fixtures.ts@@ -477,6 +477,31 @@ export async function addAllCustomProps({)}+export async function enableDashboardCsvExportV2({+ request,+ domain+}: {+ request: APIRequestContext+ domain: string+}) {+ const response = await request.post(+ '/e2e-tests/enable-dashboard-csv-export-v2',+ {+ headers: {+ 'Content-Type': 'application/json',+ Accept: 'application/json'+ },+ data: { domain }+ }+ )+ if (!response.ok()) {+ const body = await response.text()+ throw new Error(+ `enableDashboardCsvExportV2 failed (${response.status()}): ${body}`+ )+ }+}+export async function addFunnel({request,domain,diff --git a/lib/plausible/stats/dashboard/csv_export.ex b/lib/plausible/stats/dashboard/csv_export.exnew file mode 100644index 0000000000..af78a18e6d--- /dev/null+++ b/lib/plausible/stats/dashboard/csv_export.ex@@ -0,0 +1,214 @@+defmodule Plausible.Stats.Dashboard.CsvExport do+ @moduledoc false++ alias Plausible.Stats.Dashboard.QueryParser+ alias Plausible.Stats.{Time, ParsedQueryParams, Query, QueryBuilder, QueryRunner, QueryResult}++ @csv_filenames [+ "visitors.csv",+ "pages.csv",+ "entry_pages.csv",+ "exit_pages.csv",+ "browsers.csv",+ "browser_versions.csv",+ "operating_systems.csv",+ "operating_system_versions.csv",+ "devices.csv",+ "channels.csv",+ "sources.csv",+ "referrers.csv",+ "utm_mediums.csv",+ "utm_sources.csv",+ "utm_campaigns.csv",+ "utm_contents.csv",+ "utm_terms.csv",+ "countries.csv",+ "regions.csv",+ "cities.csv"+ ]++ def get_csvs(site, params, debug_metadata) do+ with {:ok, queries_by_filename} <- create_queries_by_filename(site, params, debug_metadata) do+ csv_values =+ queries_by_filename+ |> Enum.map(fn {filename, query} ->+ fn -> run_query_into_csv(site, filename, query) end+ end)+ |> Plausible.ClickhouseRepo.parallel_tasks()++ csvs =+ queries_by_filename+ |> Enum.map(fn {filename, _query} -> filename end)+ |> Enum.zip(csv_values)+ |> Enum.map(fn {k, v} -> {String.to_charlist(k), IO.iodata_to_binary(v)} end)++ {:ok, csvs}+ end+ end++ defp create_queries_by_filename(site, params, debug_metadata) do+ requested = Map.keys(params["reports"] || %{}) |> Enum.filter(&(&1 in @csv_filenames))++ Enum.reduce_while(requested, {:ok, []}, fn filename, acc ->+ case parse_csv_query(site, filename, params, debug_metadata) do+ {:ok, query} -> {:cont, {:ok, elem(acc, 1) ++ [{filename, query}]}}+ error_tuple -> {:halt, error_tuple}+ end+ end)+ end++ defp parse_csv_query(site, filename, params, debug_metadata) do+ params = construct_raw_query_params(filename, params)++ with {:ok, %ParsedQueryParams{} = params} <- QueryParser.parse(params) do+ QueryBuilder.build(site, params, debug_metadata)+ end+ end++ defp construct_raw_query_params(filename, params) do+ report_params = params["reports"][filename]++ filters =+ case report_params["always_on_filters"] do+ always_on when is_list(always_on) -> params["filters"] ++ always_on+ _ -> params["filters"]+ end++ %{+ "dimensions" => report_params["dimensions"],+ "metrics" => report_params["metrics"],+ "date_range" => params["date_range"],+ "relative_date" => params["relative_date"],+ "filters" => filters,+ "include" => params["include"]+ }+ end++ defp run_query_into_csv(site, "visitors.csv", query) do+ query =+ query+ |> Query.set(order_by: [{Time.time_dimension(query), :asc}])+ |> Query.set_include(:time_labels, true)+ |> Query.set_include(:empty_metrics, true)++ QueryRunner.run(site, query) |> timeseries_query_result_to_csv()+ end++ defp run_query_into_csv(site, filename, query) do+ # The export is limited to 300 entries for other reports and 100 entries for+ # pages because bigger result sets start causing failures. Since we request+ # data like time on page or bounce_rate for pages in a separate query using+ # the IN filter, it causes the requests to balloon in payload size.+ limit = if filename in ["pages.csv", "exit_pages.csv"], do: 100, else: 300+ order_by = [{:visitors, :desc} | Enum.map(query.dimensions, &{&1, :asc})]++ query =+ query+ |> Query.set(order_by: order_by)+ |> Query.set(pagination: %{limit: limit, offset: 0})++ QueryRunner.run(site, query) |> query_result_to_csv()+ end++ defp query_result_to_csv(%QueryResult{results: results, query: query}) do+ results+ |> Enum.reduce([csv_first_row(query)], fn row, acc ->+ acc ++ [csv_dimension_values(query[:dimensions], row) ++ row.metrics]+ end)+ |> NimbleCSV.RFC4180.dump_to_iodata()+ end++ defp timeseries_query_result_to_csv(%QueryResult{results: results, query: query, meta: meta}) do+ meta[:time_labels]+ |> Enum.reduce([csv_first_row(query)], fn timelabel, acc ->+ if result_item = find_result_item_by_timelabel(results, timelabel) do+ acc ++ [[timelabel | result_item.metrics]]+ else+ acc ++ [[timelabel | meta[:empty_metrics]]]+ end+ end)+ |> NimbleCSV.RFC4180.dump_to_iodata()+ end++ defp find_result_item_by_timelabel(results, timelabel) do+ results+ |> Map.new(fn entry -> {Enum.at(entry.dimensions, 0), entry} end)+ |> Map.get(timelabel)+ end++ defp csv_first_row(query) do+ csv_dimension_labels(query[:dimensions]) ++ csv_metric_labels(query)+ end++ defp csv_dimension_labels(["time:" <> _]) do+ [:date]+ end++ defp csv_dimension_labels(["visit:browser_version", "visit:browser"]) do+ [:name, :version]+ end++ defp csv_dimension_labels(["visit:os_version", "visit:os"]) do+ [:name, :version]+ end++ defp csv_dimension_labels([_dimension]) do+ [:name]+ end++ defp get_goal_filtered_metric_label_overrides(query) do+ case query[:dimensions] do+ ["time:" <> _] ->+ %{+ visitors: :unique_conversions,+ events: :total_conversions,+ group_conversion_rate: :conversion_rate+ }++ _ ->+ %{+ visitors: :conversions,+ group_conversion_rate: :conversion_rate+ }+ end+ end++ @dimension_specific_metric_label_overrides %{+ ["visit:entry_page"] => %{+ visitors: :unique_entrances,+ visits: :total_entrances+ },+ ["visit:exit_page"] => %{+ visitors: :unique_exits,+ visits: :total_exits+ }+ }++ defp csv_metric_labels(query) do+ goal_filter? =+ Enum.any?(query[:filters], fn [_operator, key | _rest] ->+ key == "event:goal"+ end)++ metric_label_overrides =+ if goal_filter? do+ get_goal_filtered_metric_label_overrides(query)+ else+ Map.get(@dimension_specific_metric_label_overrides, query[:dimensions], %{})+ end++ Enum.map(query[:metrics], &(metric_label_overrides[&1] || &1))+ end++ defp csv_dimension_values(["visit:browser_version", "visit:browser"], row) do+ row[:dimensions] |> Enum.reverse()+ end++ defp csv_dimension_values(["visit:os_version", "visit:os"], row) do+ row[:dimensions] |> Enum.reverse()+ end++ defp csv_dimension_values([_dimension], row) do+ row[:dimensions]+ end+enddiff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 945f2e23d3..30e205525c 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -17,6 +17,7 @@ defmodule PlausibleWeb.Api.StatsController doQueryError}+ alias Plausible.Stats.Dashboard.CsvExportalias PlausibleWeb.Api.Helpers, as: Hrequire Logger@@ -24,7 +25,7 @@ defmodule PlausibleWeb.Api.StatsController do@revenue_metrics on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])@not_set "(not set)"- plug(:date_validation_plug when action not in [:query])+ plug(:date_validation_plug when action not in [:query, :csv_export_v2])plug(:validate_required_filters_plug when action not in [:current_visitors])def query(conn, params) do@@ -46,6 +47,26 @@ defmodule PlausibleWeb.Api.StatsController doendend+ def csv_export_v2(conn, params) do+ site = conn.assigns.site++ case CsvExport.get_csvs(site, params, debug_metadata(conn)) do+ {:ok, csvs} ->+ {:ok, {_, zip_content}} = :zip.create(~c"export.zip", csvs, [:memory])++ conn+ |> put_resp_content_type("application/zip")+ |> put_resp_header(+ "content-disposition",+ Plausible.Exports.content_disposition("export.zip")+ )+ |> send_resp(200, zip_content)++ {:error, %QueryError{message: message}} ->+ H.bad_request(conn, message)+ end+ end+def sources(conn, params) dosite = conn.assigns[:site]params = Map.put(params, "property", "visit:source")diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.exindex a627ca3f2c..7e563d44df 100644--- a/lib/plausible_web/controllers/stats_controller.ex+++ b/lib/plausible_web/controllers/stats_controller.ex@@ -550,7 +550,7 @@ defmodule PlausibleWeb.StatsController dodefp get_flags(user, site),do:- []+ [:dashboard_csv_export_v2]|> Enum.map(fn flag ->{flag, FunWithFlags.enabled?(flag, for: user) || FunWithFlags.enabled?(flag, for: site)}end)diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.exindex b1a6b609b3..71f4db39e9 100644--- a/lib/plausible_web/router.ex+++ b/lib/plausible_web/router.ex@@ -175,6 +175,7 @@ defmodule PlausibleWeb.Router dopost "/stats", E2EController, :populate_statspost "/funnel", E2EController, :create_funnelpost "/goal", E2EController, :create_goal+ post "/enable-dashboard-csv-export-v2", E2EController, :enable_dashboard_csv_export_v2endendend@@ -284,6 +285,7 @@ defmodule PlausibleWeb.Router doscope private: %{allow_consolidated_views: true} dopost "/:domain/query", StatsController, :query+ post "/:domain/export", StatsController, :csv_export_v2get "/:domain/google-search-terms", StatsController, :google_search_termsget "/:domain/current-visitors", StatsController, :current_visitorsget "/:domain/sources", StatsController, :sourcesdiff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/exit_pages.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/exit_pages.csvindex 736ba6d58f..fd838ca9ed 100644--- a/test/plausible_web/controllers/CSVs/30d-filter-path/exit_pages.csv+++ b/test/plausible_web/controllers/CSVs/30d-filter-path/exit_pages.csv@@ -1,2 +1,2 @@-name,unique_exits,total_exits,exit_rate-/some-other-page,1,1,+name,unique_exits,total_exits+/some-other-page,1,1diff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/exit_pages_legacy.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/exit_pages_legacy.csvnew file mode 100644index 0000000000..736ba6d58f--- /dev/null+++ b/test/plausible_web/controllers/CSVs/30d-filter-path/exit_pages_legacy.csv@@ -0,0 +1,2 @@+name,unique_exits,total_exits,exit_rate+/some-other-page,1,1,diff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/visitors.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/visitors.csvindex c1e30fab06..e345da62c5 100644--- a/test/plausible_web/controllers/CSVs/30d-filter-path/visitors.csv+++ b/test/plausible_web/controllers/CSVs/30d-filter-path/visitors.csv@@ -1,32 +1,32 @@-date,visitors,pageviews,visits,views_per_visit,bounce_rate,visit_duration,scroll_depth-2021-09-20,0,0,0,0.0,0.0,,-2021-09-21,0,0,0,0.0,0.0,,-2021-09-22,0,0,0,0.0,0.0,,-2021-09-23,0,0,0,0.0,0.0,,-2021-09-24,0,0,0,0.0,0.0,,-2021-09-25,0,0,0,0.0,0.0,,-2021-09-26,0,0,0,0.0,0.0,,-2021-09-27,0,0,0,0.0,0.0,,-2021-09-28,0,0,0,0.0,0.0,,-2021-09-29,0,0,0,0.0,0.0,,-2021-09-30,0,0,0,0.0,0.0,,-2021-10-01,0,0,0,0.0,0.0,,-2021-10-02,0,0,0,0.0,0.0,,-2021-10-03,0,0,0,0.0,0.0,,-2021-10-04,0,0,0,0.0,0.0,,-2021-10-05,0,0,0,0.0,0.0,,-2021-10-06,0,0,0,0.0,0.0,,-2021-10-07,0,0,0,0.0,0.0,,-2021-10-08,0,0,0,0.0,0.0,,-2021-10-09,0,0,0,0.0,0.0,,-2021-10-10,0,0,0,0.0,0.0,,-2021-10-11,0,0,0,0.0,0.0,,-2021-10-12,0,0,0,0.0,0.0,,-2021-10-13,0,0,0,0.0,0.0,,-2021-10-14,0,0,0,0.0,0.0,,-2021-10-15,0,0,0,0.0,0.0,,-2021-10-16,0,0,0,0.0,0.0,,-2021-10-17,0,0,0,0.0,0.0,,-2021-10-18,0,0,0,0.0,0.0,,-2021-10-19,0,0,0,0.0,0.0,,-2021-10-20,1,1,1,2.0,0,60,30+date,visitors,pageviews,visits,bounce_rate,time_on_page,scroll_depth+2021-09-20,0,0,0,0.0,,+2021-09-21,0,0,0,0.0,,+2021-09-22,0,0,0,0.0,,+2021-09-23,0,0,0,0.0,,+2021-09-24,0,0,0,0.0,,+2021-09-25,0,0,0,0.0,,+2021-09-26,0,0,0,0.0,,+2021-09-27,0,0,0,0.0,,+2021-09-28,0,0,0,0.0,,+2021-09-29,0,0,0,0.0,,+2021-09-30,0,0,0,0.0,,+2021-10-01,0,0,0,0.0,,+2021-10-02,0,0,0,0.0,,+2021-10-03,0,0,0,0.0,,+2021-10-04,0,0,0,0.0,,+2021-10-05,0,0,0,0.0,,+2021-10-06,0,0,0,0.0,,+2021-10-07,0,0,0,0.0,,+2021-10-08,0,0,0,0.0,,+2021-10-09,0,0,0,0.0,,+2021-10-10,0,0,0,0.0,,+2021-10-11,0,0,0,0.0,,+2021-10-12,0,0,0,0.0,,+2021-10-13,0,0,0,0.0,,+2021-10-14,0,0,0,0.0,,+2021-10-15,0,0,0,0.0,,+2021-10-16,0,0,0,0.0,,+2021-10-17,0,0,0,0.0,,+2021-10-18,0,0,0,0.0,,+2021-10-19,0,0,0,0.0,,+2021-10-20,1,1,1,0,60,30diff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/visitors_legacy.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/visitors_legacy.csvnew file mode 100644index 0000000000..c1e30fab06--- /dev/null+++ b/test/plausible_web/controllers/CSVs/30d-filter-path/visitors_legacy.csv@@ -0,0 +1,32 @@+date,visitors,pageviews,visits,views_per_visit,bounce_rate,visit_duration,scroll_depth+2021-09-20,0,0,0,0.0,0.0,,+2021-09-21,0,0,0,0.0,0.0,,+2021-09-22,0,0,0,0.0,0.0,,+2021-09-23,0,0,0,0.0,0.0,,+2021-09-24,0,0,0,0.0,0.0,,+2021-09-25,0,0,0,0.0,0.0,,+2021-09-26,0,0,0,0.0,0.0,,+2021-09-27,0,0,0,0.0,0.0,,+2021-09-28,0,0,0,0.0,0.0,,+2021-09-29,0,0,0,0.0,0.0,,+2021-09-30,0,0,0,0.0,0.0,,+2021-10-01,0,0,0,0.0,0.0,,+2021-10-02,0,0,0,0.0,0.0,,+2021-10-03,0,0,0,0.0,0.0,,+2021-10-04,0,0,0,0.0,0.0,,+2021-10-05,0,0,0,0.0,0.0,,+2021-10-06,0,0,0,0.0,0.0,,+2021-10-07,0,0,0,0.0,0.0,,+2021-10-08,0,0,0,0.0,0.0,,+2021-10-09,0,0,0,0.0,0.0,,+2021-10-10,0,0,0,0.0,0.0,,+2021-10-11,0,0,0,0.0,0.0,,+2021-10-12,0,0,0,0.0,0.0,,+2021-10-13,0,0,0,0.0,0.0,,+2021-10-14,0,0,0,0.0,0.0,,+2021-10-15,0,0,0,0.0,0.0,,+2021-10-16,0,0,0,0.0,0.0,,+2021-10-17,0,0,0,0.0,0.0,,+2021-10-18,0,0,0,0.0,0.0,,+2021-10-19,0,0,0,0.0,0.0,,+2021-10-20,1,1,1,2.0,0,60,30diff --git a/test/plausible_web/controllers/api/stats_controller/dashboard_csv_export_test.exs b/test/plausible_web/controllers/api/stats_controller/dashboard_csv_export_test.exsnew file mode 100644index 0000000000..ecc13363e7--- /dev/null+++ b/test/plausible_web/controllers/api/stats_controller/dashboard_csv_export_test.exs@@ -0,0 +1,1083 @@+defmodule PlausibleWeb.Api.StatsController.DashboardCsvExportTest do+ use PlausibleWeb.ConnCase, async: false+ use Plausible.Repo++ @default_reports %{+ "visitors.csv" => %{+ dimensions: ["time:day"],+ metrics: [+ "visitors",+ "pageviews",+ "visits",+ "views_per_visit",+ "bounce_rate",+ "visit_duration"+ ]+ },+ "sources.csv" => %{+ dimensions: ["visit:source"],+ metrics: ["visitors", "bounce_rate", "visit_duration"]+ },+ "channels.csv" => %{+ dimensions: ["visit:channel"],+ metrics: ["visitors", "bounce_rate", "visit_duration"]+ },+ "referrers.csv" => %{+ dimensions: ["visit:referrer"],+ metrics: ["visitors", "bounce_rate", "visit_duration"]+ },+ "utm_mediums.csv" => %{+ dimensions: ["visit:utm_medium"],+ always_on_filters: [["is_not", "visit:utm_medium", [""]]],+ metrics: ["visitors", "bounce_rate", "visit_duration"]+ },+ "utm_sources.csv" => %{+ dimensions: ["visit:utm_source"],+ always_on_filters: [["is_not", "visit:utm_source", [""]]],+ metrics: ["visitors", "bounce_rate", "visit_duration"]+ },+ "utm_campaigns.csv" => %{+ dimensions: ["visit:utm_campaign"],+ always_on_filters: [["is_not", "visit:utm_campaign", [""]]],+ metrics: ["visitors", "bounce_rate", "visit_duration"]+ },+ "utm_contents.csv" => %{+ dimensions: ["visit:utm_content"],+ always_on_filters: [["is_not", "visit:utm_content", [""]]],+ metrics: ["visitors", "bounce_rate", "visit_duration"]+ },+ "utm_terms.csv" => %{+ dimensions: ["visit:utm_term"],+ always_on_filters: [["is_not", "visit:utm_term", [""]]],+ metrics: ["visitors", "bounce_rate", "visit_duration"]+ },+ "pages.csv" => %{+ dimensions: ["event:page"],+ metrics: ["visitors", "pageviews", "bounce_rate", "time_on_page", "scroll_depth"]+ },+ "entry_pages.csv" => %{+ dimensions: ["visit:entry_page"],+ always_on_filters: [["is_not", "visit:entry_page", [""]]],+ metrics: ["visitors", "visits", "bounce_rate", "visit_duration"]+ },+ "exit_pages.csv" => %{+ dimensions: ["visit:exit_page"],+ always_on_filters: [["is_not", "visit:exit_page", [""]]],+ metrics: ["visitors", "visits", "exit_rate"]+ },+ "browsers.csv" => %{+ dimensions: ["visit:browser"],+ metrics: ["visitors"]+ },+ "browser_versions.csv" => %{+ dimensions: ["visit:browser_version", "visit:browser"],+ metrics: ["visitors"]+ },+ "operating_systems.csv" => %{+ dimensions: ["visit:os"],+ metrics: ["visitors"]+ },+ "operating_system_versions.csv" => %{+ dimensions: ["visit:os_version", "visit:os"],+ metrics: ["visitors"]+ },+ "devices.csv" => %{+ dimensions: ["visit:device"],+ metrics: ["visitors"]+ },+ "countries.csv" => %{+ dimensions: ["visit:country_name"],+ always_on_filters: [["is_not", "visit:country", ["\0\0", "ZZ"]]],+ metrics: ["visitors"]+ },+ "regions.csv" => %{+ dimensions: ["visit:region_name"],+ always_on_filters: [["is_not", "visit:region", [""]]],+ metrics: ["visitors"]+ },+ "cities.csv" => %{+ dimensions: ["visit:city_name"],+ always_on_filters: [["is_not", "visit:city", [0]]],+ metrics: ["visitors"]+ }+ }++ @page_filtered_reports put_in(@default_reports, ["visitors.csv", :metrics], [+ "visitors",+ "pageviews",+ "visits",+ "bounce_rate",+ "time_on_page",+ "scroll_depth"+ ])+ |> put_in(+ ["exit_pages.csv", :metrics],+ get_in(@default_reports, ["exit_pages.csv", :metrics]) -- ["exit_rate"]+ )++ @goal_filtered_reports Map.new(@default_reports, fn+ {"visitors.csv" = filename, params} ->+ {filename,+ %{params | metrics: ["visitors", "events", "group_conversion_rate"]}}++ {filename, params} ->+ {filename,+ %{params | metrics: ["visitors", "group_conversion_rate"]}}+ end)++ @base_params %{+ date_range: "30d",+ relative_date: nil,+ filters: [],+ include: %{imports: false},+ reports: @default_reports+ }++ describe "POST /api/stats/:domain/export" do+ setup [:create_user, :create_site, :log_in]++ test "exports all the necessary CSV files", %{conn: conn, site: site} do+ conn = do_export(conn, site, @base_params)++ assert {"content-type", "application/zip; charset=utf-8"} =+ List.keyfind(conn.resp_headers, "content-type", 0)++ {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])++ zip = Enum.map(zip, fn {filename, _} -> filename end)++ assert ~c"visitors.csv" in zip+ assert ~c"browsers.csv" in zip+ assert ~c"browser_versions.csv" in zip+ assert ~c"cities.csv" in zip+ # assert ~c"conversions.csv" in zip+ assert ~c"countries.csv" in zip+ assert ~c"devices.csv" in zip+ assert ~c"entry_pages.csv" in zip+ assert ~c"exit_pages.csv" in zip+ assert ~c"operating_systems.csv" in zip+ assert ~c"operating_system_versions.csv" in zip+ assert ~c"pages.csv" in zip+ assert ~c"regions.csv" in zip+ assert ~c"sources.csv" in zip+ assert ~c"channels.csv" in zip+ assert ~c"utm_campaigns.csv" in zip+ assert ~c"utm_contents.csv" in zip+ assert ~c"utm_mediums.csv" in zip+ assert ~c"utm_sources.csv" in zip+ assert ~c"utm_terms.csv" in zip+ end++ test "limits pages.csv and exit_pages.csv to 100 rows", %{conn: conn, site: site} do+ events = for i <- 1..101, do: build(:pageview, pathname: "/page-#{i}")+ populate_stats(site, events)++ zip_content =+ conn+ |> do_export(site, %{@base_params | date_range: "day"})+ |> response(200)++ pages = unzip_and_parse_csv(zip_content, ~c"pages.csv")+ # header + 100 data rows + trailing empty row+ assert length(pages) == 102++ exit_pages = unzip_and_parse_csv(zip_content, ~c"exit_pages.csv")+ assert length(exit_pages) == 102+ end++ test "limits other breakdown reports to 300 rows", %{conn: conn, site: site} do+ events = for i <- 1..301, do: build(:pageview, referrer_source: "Source #{i}")+ populate_stats(site, events)++ sources =+ conn+ |> do_export(site, %{@base_params | date_range: "day"})+ |> response(200)+ |> unzip_and_parse_csv(~c"sources.csv")++ # header + 300 data rows + trailing empty row+ assert length(sources) == 302+ end++ test "exports scroll depth metric in pages.csv", %{conn: conn, site: site} do+ t0 = ~N[2020-01-01 00:00:00]+ [t1, t2, t3] = for i <- 1..3, do: NaiveDateTime.add(t0, i, :minute)++ populate_stats(site, [+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: t0),+ build(:engagement,+ user_id: 12,+ pathname: "/blog",+ timestamp: t1,+ scroll_depth: 20,+ engagement_time: 60_000+ ),+ build(:pageview, user_id: 12, pathname: "/another", timestamp: t1),+ build(:engagement,+ user_id: 12,+ pathname: "/another",+ timestamp: t2,+ scroll_depth: 24,+ engagement_time: 60_000+ ),+ build(:pageview, user_id: 34, pathname: "/blog", timestamp: t0),+ build(:engagement,+ user_id: 34,+ pathname: "/blog",+ timestamp: t1,+ scroll_depth: 17,+ engagement_time: 60_000+ ),+ build(:pageview, user_id: 34, pathname: "/another", timestamp: t1),+ build(:engagement,+ user_id: 34,+ pathname: "/another",+ timestamp: t2,+ scroll_depth: 26,+ engagement_time: 60_000+ ),+ build(:pageview, user_id: 34, pathname: "/blog", timestamp: t2),+ build(:engagement,+ user_id: 34,+ pathname: "/blog",+ timestamp: t3,+ scroll_depth: 60,+ engagement_time: 60_000+ ),+ build(:pageview, user_id: 56, pathname: "/blog", timestamp: t0),+ build(:engagement,+ user_id: 56,+ pathname: "/blog",+ timestamp: t1,+ scroll_depth: 100,+ engagement_time: 60_000+ )+ ])++ pages =+ conn+ |> do_export(site, %{@base_params | date_range: "day", relative_date: "2020-01-01"})+ |> response(200)+ |> unzip_and_parse_csv(~c"pages.csv")++ assert pages == [+ ["name", "visitors", "pageviews", "bounce_rate", "time_on_page", "scroll_depth"],+ ["/blog", "3", "4", "33", "80", "60"],+ ["/another", "2", "2", "0", "60", "25"],+ [""]+ ]+ end++ # test "exports only internally used props in custom_props.csv for a growth plan", %{+ # conn: conn,+ # site: site+ # } do+ # {:ok, site} = Plausible.Props.allow(site, ["author"])++ # [owner | _] = Repo.preload(site, :owners).owners+ # subscribe_to_growth_plan(owner)++ # populate_stats(site, [+ # build(:pageview, "meta.key": ["author"], "meta.value": ["a"]),+ # build(:event, name: "File Download", "meta.key": ["url"], "meta.value": ["b"])+ # ])++ # result =+ # conn+ # |> do_export(site, %{@base_params | date_range: "day"})+ # |> response(200)+ # |> unzip_and_parse_csv(~c"custom_props.csv")++ # assert result == [+ # ["property", "value", "visitors", "events", "percentage"],+ # ["url", "(none)", "1", "1", "50.0"],+ # ["url", "b", "1", "1", "50.0"],+ # [""]+ # ]+ # end++ # test "does not include custom_props.csv for a growth plan if no internal props used", %{+ # conn: conn,+ # site: site+ # } do+ # {:ok, site} = Plausible.Props.allow(site, ["author"])++ # [owner | _] = Repo.preload(site, :owners).owners+ # subscribe_to_growth_plan(owner)++ # populate_stats(site, [+ # build(:pageview, "meta.key": ["author"], "meta.value": ["a"])+ # ])++ # {:ok, zip} =+ # conn+ # |> do_export(site, %{@base_params | date_range: "day"})+ # |> response(200)+ # |> :zip.unzip([:memory])++ # files = Map.new(zip)++ # refute Map.has_key?(files, ~c"custom_props.csv")+ # end++ test "exports data in zipped csvs", %{conn: conn, site: site} do+ populate_exported_stats(site)++ conn =+ do_export(conn, site, %{@base_params | date_range: ["2021-09-20", "2021-10-20"]})++ assert_zip(conn, "30d")+ end++ # test "exports allowed event props for a trial account", %{conn: conn, site: site} do+ # {:ok, site} = Plausible.Props.allow(site, ["author", "logged_in"])++ # populate_stats(site, [+ # build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),+ # build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),+ # build(:event, "meta.key": ["author"], "meta.value": ["marko"], name: "Newsletter Signup"),+ # build(:pageview, user_id: 999, "meta.key": ["logged_in"], "meta.value": ["true"]),+ # build(:pageview, user_id: 999, "meta.key": ["logged_in"], "meta.value": ["true"]),+ # build(:pageview, "meta.key": ["disallowed"], "meta.value": ["whatever"]),+ # build(:pageview)+ # ])++ # result =+ # conn+ # |> do_export(site, %{@base_params | date_range: "day"})+ # |> response(200)+ # |> unzip_and_parse_csv(~c"custom_props.csv")++ # assert result == [+ # ["property", "value", "visitors", "events", "percentage"],+ # ["author", "(none)", "3", "4", "50.0"],+ # ["author", "uku", "2", "2", "33.33"],+ # ["author", "marko", "1", "1", "16.67"],+ # ["logged_in", "(none)", "5", "5", "83.33"],+ # ["logged_in", "true", "1", "2", "16.67"],+ # [""]+ # ]+ # end++ test "exports data grouped by interval", %{conn: conn, site: site} do+ populate_exported_stats(site)++ params =+ @base_params+ |> Map.put(:date_range, ["2021-09-20", "2021-10-20"])+ |> put_in([:reports, "visitors.csv", :dimensions], ["time:week"])++ visitors =+ conn+ |> do_export(site, params)+ |> response(200)+ |> unzip_and_parse_csv(~c"visitors.csv")++ assert visitors == [+ [+ "date",+ "visitors",+ "pageviews",+ "visits",+ "views_per_visit",+ "bounce_rate",+ "visit_duration"+ ],+ ["2021-09-20", "1", "1", "1", "1.0", "100", "0"],+ ["2021-09-27", "0", "0", "0", "0.0", "0.0", ""],+ ["2021-10-04", "0", "0", "0", "0.0", "0.0", ""],+ ["2021-10-11", "0", "0", "0", "0.0", "0.0", ""],+ ["2021-10-18", "3", "4", "3", "1.33", "33", "40"],+ [""]+ ]+ end++ test "exports operating system versions", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, operating_system: "Mac", operating_system_version: "14"),+ build(:pageview, operating_system: "Mac", operating_system_version: "14"),+ build(:pageview, operating_system: "Mac", operating_system_version: "14"),+ build(:pageview, operating_system: "Ubuntu", operating_system_version: "20.04"),+ build(:pageview, operating_system: "Ubuntu", operating_system_version: "20.04"),+ build(:pageview, operating_system: "Mac", operating_system_version: "13")+ ])++ os_versions =+ conn+ |> do_export(site, %{@base_params | date_range: "day"})+ |> response(200)+ |> unzip_and_parse_csv(~c"operating_system_versions.csv")++ assert os_versions == [+ ["name", "version", "visitors"],+ ["Mac", "14", "3"],+ ["Ubuntu", "20.04", "2"],+ ["Mac", "13", "1"],+ [""]+ ]+ end++ test "exports imported data when requested", %{conn: conn, site: site} do+ site_import = insert(:site_import, site: site)++ insert(:goal, site: site, event_name: "Outbound Link: Click")++ populate_stats(site, site_import.id, [+ build(:imported_visitors, visitors: 9),+ build(:imported_browsers, browser: "Chrome", pageviews: 1),+ build(:imported_devices, device: "Desktop", pageviews: 1),+ build(:imported_entry_pages, entry_page: "/test", pageviews: 1),+ build(:imported_exit_pages, exit_page: "/test", pageviews: 1),+ build(:imported_locations, country: "PL", region: "PL-22", city: 3_099_434, pageviews: 1),+ build(:imported_operating_systems, operating_system: "Mac", pageviews: 1),+ build(:imported_pages, page: "/test", pageviews: 1),+ build(:imported_sources,+ source: "Google",+ channel: "Paid Search",+ utm_medium: "search",+ utm_campaign: "ads",+ utm_source: "google",+ utm_content: "content",+ utm_term: "term",+ pageviews: 1+ ),+ build(:imported_custom_events,+ name: "Outbound Link: Click",+ link_url: "https://example.com",+ visitors: 5,+ events: 10+ )+ ])++ tomorrow = Date.utc_today() |> Date.add(1) |> Date.to_iso8601()++ params = %{@base_params | relative_date: tomorrow, include: %{imports: true}}++ assert response = do_export(conn, site, params) |> response(200)+ {:ok, zip} = :zip.unzip(response, [:memory])++ filenames = zip |> Enum.map(fn {filename, _} -> to_string(filename) end)++ # NOTE: currently, custom_props.csv is not populated from imported data+ expected_filenames = [+ "visitors.csv",+ "sources.csv",+ "channels.csv",+ "utm_mediums.csv",+ "utm_sources.csv",+ "utm_campaigns.csv",+ "utm_contents.csv",+ "utm_terms.csv",+ "pages.csv",+ "entry_pages.csv",+ "exit_pages.csv",+ "countries.csv",+ "regions.csv",+ "cities.csv",+ "browsers.csv",+ "browser_versions.csv",+ "operating_systems.csv",+ "operating_system_versions.csv",+ "devices.csv",+ # "conversions.csv",+ "referrers.csv"+ ]++ Enum.each(expected_filenames, fn expected -> assert expected in filenames end)++ Enum.each(zip, fn+ {~c"visitors.csv", data} ->+ csv = parse_csv(data)++ assert List.first(csv) == [+ "date",+ "visitors",+ "pageviews",+ "visits",+ "views_per_visit",+ "bounce_rate",+ "visit_duration"+ ]++ assert Enum.at(csv, -2) ==+ [Date.to_iso8601(Date.utc_today()), "9", "1", "1", "1.0", "0.0", "10.0"]++ {~c"sources.csv", data} ->+ assert parse_csv(data) == [+ ["name", "visitors", "bounce_rate", "visit_duration"],+ ["Google", "1", "0.0", "10.0"],+ [""]+ ]++ {~c"channels.csv", data} ->+ assert parse_csv(data) == [+ ["name", "visitors", "bounce_rate", "visit_duration"],+ ["Paid Search", "1", "0.0", "10.0"],+ [""]+ ]++ {~c"utm_mediums.csv", data} ->+ assert parse_csv(data) == [+ ["name", "visitors", "bounce_rate", "visit_duration"],+ ["search", "1", "0.0", "10.0"],+ [""]+ ]++ {~c"utm_sources.csv", data} ->+ assert parse_csv(data) == [+ ["name", "visitors", "bounce_rate", "visit_duration"],+ ["google", "1", "0.0", "10.0"],+ [""]+ ]++ {~c"utm_campaigns.csv", data} ->+ assert parse_csv(data) == [+ ["name", "visitors", "bounce_rate", "visit_duration"],+ ["ads", "1", "0.0", "10.0"],+ [""]+ ]++ {~c"utm_contents.csv", data} ->+ assert parse_csv(data) == [+ ["name", "visitors", "bounce_rate", "visit_duration"],+ ["content", "1", "0.0", "10.0"],+ [""]+ ]++ {~c"utm_terms.csv", data} ->+ assert parse_csv(data) == [+ ["name", "visitors", "bounce_rate", "visit_duration"],+ ["term", "1", "0.0", "10.0"],+ [""]+ ]++ {~c"pages.csv", data} ->+ assert parse_csv(data) == [+ [+ "name",+ "visitors",+ "pageviews",+ "bounce_rate",+ "time_on_page",+ "scroll_depth"+ ],+ ["/test", "1", "1", "0.0", "10", ""],+ [""]+ ]++ {~c"entry_pages.csv", data} ->+ assert parse_csv(data) == [+ [+ "name",+ "unique_entrances",+ "total_entrances",+ "bounce_rate",+ "visit_duration"+ ],+ ["/test", "1", "1", "0.0", "10.0"],+ [""]+ ]++ {~c"exit_pages.csv", data} ->+ assert parse_csv(data) == [+ ["name", "unique_exits", "total_exits", "exit_rate"],+ ["/test", "1", "1", "100.0"],+ [""]+ ]++ {~c"countries.csv", data} ->+ assert parse_csv(data) == [["name", "visitors"], ["Poland", "1"], [""]]++ {~c"regions.csv", data} ->+ assert parse_csv(data) == [["name", "visitors"], ["Pomerania", "1"], [""]]++ {~c"cities.csv", data} ->+ assert parse_csv(data) == [["name", "visitors"], ["Gdańsk", "1"], [""]]++ {~c"browsers.csv", data} ->+ assert parse_csv(data) == [["name", "visitors"], ["Chrome", "1"], [""]]++ {~c"browser_versions.csv", data} ->+ assert parse_csv(data) == [+ ["name", "version", "visitors"],+ ["Chrome", "(not set)", "1"],+ [""]+ ]++ {~c"operating_systems.csv", data} ->+ assert parse_csv(data) == [["name", "visitors"], ["Mac", "1"], [""]]++ {~c"operating_system_versions.csv", data} ->+ assert parse_csv(data) == [+ ["name", "version", "visitors"],+ ["Mac", "(not set)", "1"],+ [""]+ ]++ {~c"devices.csv", data} ->+ assert parse_csv(data) == [["name", "visitors"], ["Desktop", "1"], [""]]++ {~c"conversions.csv", data} ->+ assert parse_csv(data) == [+ ["name", "unique_conversions", "total_conversions"],+ ["Outbound Link: Click", "5", "10"],+ [""]+ ]++ {~c"referrers.csv", data} ->+ assert parse_csv(data) == [+ ["name", "visitors", "bounce_rate", "visit_duration"],+ ["Direct / None", "1", "0.0", "10.0"],+ [""]+ ]+ end)+ end+ end++ defp parse_csv(file_content) when is_binary(file_content) do+ file_content+ |> String.split("\r\n")+ |> Enum.map(&String.split(&1, ","))+ end++ describe "POST /api/stats/:domain/export - via shared link" do+ setup [:create_user, :create_site]++ test "exports data in zipped csvs", %{conn: conn, site: site} do+ link = insert(:shared_link, site: site)++ populate_exported_stats(site)++ params =+ %{@base_params | date_range: ["2021-09-20", "2021-10-20"], include: %{imports: false}}+ |> Map.put(:auth, link.slug)++ conn = do_export(conn, site, params)++ assert_zip(conn, "30d")+ end+ end++ describe "POST /api/stats/:domain/export - for past 6 months" do+ setup [:create_user, :create_site, :log_in]++ test "exports 6 months of data in zipped csvs", %{conn: conn, site: site} do+ populate_exported_stats(site)++ conn =+ do_export(conn, site, %{+ @base_params+ | date_range: "6mo",+ relative_date: "2021-11-20",+ reports: put_in(@default_reports, ["visitors.csv", :dimensions], ["time:month"])+ })++ assert_zip(conn, "6m")+ end+ end++ describe "POST /api/stats/:domain/export - with path filter" do+ setup [:create_user, :create_site, :log_in]++ test "exports filtered data in zipped csvs", %{conn: conn, site: site} do+ populate_exported_stats(site)++ conn =+ do_export(conn, site, %{+ @base_params+ | date_range: ["2021-09-20", "2021-10-20"],+ filters: [["is", "event:page", ["/some-other-page"]]],+ reports: @page_filtered_reports+ })++ assert_zip(conn, "30d-filter-path")+ end++ test "exports scroll depth in visitors.csv", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: ~N[2020-01-05 00:00:00]),+ build(:engagement,+ user_id: 12,+ pathname: "/blog",+ timestamp: ~N[2020-01-05 00:01:00],+ scroll_depth: 40+ ),+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: ~N[2020-01-05 10:00:00]),+ build(:engagement,+ user_id: 12,+ pathname: "/blog",+ timestamp: ~N[2020-01-05 10:01:00],+ scroll_depth: 17+ ),+ build(:pageview, user_id: 34, pathname: "/blog", timestamp: ~N[2020-01-07 00:00:00]),+ build(:engagement,+ user_id: 34,+ pathname: "/blog",+ timestamp: ~N[2020-01-07 00:01:00],+ scroll_depth: 90+ )+ ])++ pages =+ conn+ |> do_export(site, %{+ @base_params+ | date_range: "7d",+ relative_date: "2020-01-08",+ filters: [["is", "event:page", ["/blog"]]],+ reports: @page_filtered_reports+ })+ |> response(200)+ |> unzip_and_parse_csv(~c"visitors.csv")++ assert pages == [+ [+ "date",+ "visitors",+ "pageviews",+ "visits",+ "bounce_rate",+ "time_on_page",+ "scroll_depth"+ ],+ ["2020-01-01", "0", "0", "0", "0.0", "", ""],+ ["2020-01-02", "0", "0", "0", "0.0", "", ""],+ ["2020-01-03", "0", "0", "0", "0.0", "", ""],+ ["2020-01-04", "0", "0", "0", "0.0", "", ""],+ ["2020-01-05", "1", "2", "2", "100", "0", "28"],+ ["2020-01-06", "0", "0", "0", "0.0", "", ""],+ ["2020-01-07", "1", "1", "1", "100", "0", "90"],+ [""]+ ]+ end+ end++ # describe "POST /api/stats/:domain/export - with a custom prop filter" do+ # setup [:create_user, :create_site, :log_in]++ # test "custom-props.csv only returns the prop and its value in filter", %{+ # conn: conn,+ # site: site+ # } do+ # {:ok, site} = Plausible.Props.allow(site, ["author", "logged_in"])++ # populate_stats(site, [+ # build(:pageview, "meta.key": ["author"], "meta.value": ["uku"]),+ # build(:pageview, "meta.key": ["author"], "meta.value": ["marko"]),+ # build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"])+ # ])++ # result =+ # conn+ # |> do_export(site, %{+ # @base_params+ # | date_range: "day",+ # filters: [["is", "event:props:author", ["marko"]]]+ # })+ # |> response(200)+ # |> unzip_and_parse_csv(~c"custom_props.csv")++ # assert result == [+ # ["property", "value", "visitors", "events", "percentage"],+ # ["author", "marko", "1", "1", "100.0"],+ # [""]+ # ]+ # end+ # end++ defp unzip_and_parse_csv(archive, filename) do+ {:ok, zip} = :zip.unzip(archive, [:memory])+ {_filename, data} = Enum.find(zip, &(elem(&1, 0) == filename))+ parse_csv(data)+ end++ defp assert_zip(conn, folder) do+ assert conn.status == 200++ assert {"content-type", "application/zip; charset=utf-8"} =+ List.keyfind(conn.resp_headers, "content-type", 0)++ {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])++ folder = Path.expand(folder, "test/plausible_web/controllers/CSVs")++ Enum.map(zip, &assert_csv_by_fixture(&1, folder))+ end++ defp assert_csv_by_fixture({file, downloaded}, folder) do+ file = Path.expand(file, folder)++ {:ok, content} = File.read(file)+ msg = "CSV file comparison failed (#{file})"+ assert downloaded == content, message: msg, left: downloaded, right: content+ end++ defp do_export(conn, site, params) do+ conn+ |> put_req_header("content-type", "application/json")+ |> post("/api/stats/#{site.domain}/export", Jason.encode!(params))+ end++ defp populate_exported_stats(site) do+ populate_stats(site, [+ build(:pageview,+ user_id: 123,+ pathname: "/",+ timestamp:+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], minute: -1)+ |> NaiveDateTime.truncate(:second),+ country_code: "EE",+ subdivision1_code: "EE-37",+ city_geoname_id: 588_409,+ referrer_source: "Google"+ ),+ build(:engagement,+ user_id: 123,+ pathname: "/",+ timestamp: ~N[2021-10-20 12:00:00] |> NaiveDateTime.truncate(:second),+ engagement_time: 30_000,+ scroll_depth: 30,+ country_code: "EE",+ subdivision1_code: "EE-37",+ city_geoname_id: 588_409,+ referrer_source: "Google"+ ),+ build(:pageview,+ user_id: 123,+ pathname: "/some-other-page",+ timestamp:+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], minute: -2)+ |> NaiveDateTime.truncate(:second),+ country_code: "EE",+ subdivision1_code: "EE-37",+ city_geoname_id: 588_409,+ referrer_source: "Google"+ ),+ build(:engagement,+ user_id: 123,+ pathname: "/some-other-page",+ timestamp:+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], minute: -1)+ |> NaiveDateTime.truncate(:second),+ engagement_time: 60_000,+ scroll_depth: 30,+ country_code: "EE",+ subdivision1_code: "EE-37",+ city_geoname_id: 588_409,+ referrer_source: "Google"+ ),+ build(:pageview,+ user_id: 100,+ pathname: "/",+ timestamp:+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], day: -1) |> NaiveDateTime.truncate(:second),+ utm_medium: "search",+ utm_campaign: "ads",+ utm_source: "google",+ utm_content: "content",+ utm_term: "term",+ browser: "Firefox",+ browser_version: "120",+ operating_system: "Mac",+ operating_system_version: "14"+ ),+ build(:engagement,+ user_id: 100,+ pathname: "/",+ timestamp:+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], day: -1, minute: 1)+ |> NaiveDateTime.truncate(:second),+ engagement_time: 30_000,+ scroll_depth: 30,+ utm_medium: "search",+ utm_campaign: "ads",+ utm_source: "google",+ utm_content: "content",+ utm_term: "term",+ browser: "Firefox",+ browser_version: "120",+ operating_system: "Mac",+ operating_system_version: "14"+ ),+ build(:pageview,+ user_id: 200,+ timestamp:+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], month: -1)+ |> NaiveDateTime.truncate(:second),+ country_code: "EE",+ browser: "Firefox",+ browser_version: "120",+ operating_system: "Mac",+ operating_system_version: "14"+ ),+ build(:engagement,+ user_id: 200,+ timestamp:+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], month: -1, minute: 1)+ |> NaiveDateTime.truncate(:second),+ engagement_time: 30_000,+ scroll_depth: 20,+ country_code: "EE",+ browser: "Firefox",+ browser_version: "120",+ operating_system: "Mac",+ operating_system_version: "14"+ ),+ build(:pageview,+ user_id: 300,+ timestamp:+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], month: -5)+ |> NaiveDateTime.truncate(:second),+ utm_campaign: "ads",+ country_code: "EE",+ referrer_source: "Google",+ click_id_param: "gclid",+ browser: "FirefoxNoVersion",+ operating_system: "MacNoVersion"+ ),+ build(:engagement,+ user_id: 300,+ timestamp:+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], month: -5, minute: 1)+ |> NaiveDateTime.truncate(:second),+ engagement_time: 30_000,+ scroll_depth: 20,+ utm_campaign: "ads",+ country_code: "EE",+ referrer_source: "Google",+ click_id_param: "gclid",+ browser: "FirefoxNoVersion",+ operating_system: "MacNoVersion"+ ),+ build(:pageview,+ user_id: 456,+ timestamp:+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], day: -1, minute: -1)+ |> NaiveDateTime.truncate(:second),+ pathname: "/signup",+ "meta.key": ["variant"],+ "meta.value": ["A"]+ ),+ build(:engagement,+ user_id: 456,+ timestamp:+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], day: -1) |> NaiveDateTime.truncate(:second),+ pathname: "/signup",+ engagement_time: 60_000,+ scroll_depth: 20,+ "meta.key": ["variant"],+ "meta.value": ["A"]+ ),+ build(:event,+ user_id: 456,+ timestamp:+ NaiveDateTime.shift(~N[2021-10-20 12:00:00], day: -1) |> NaiveDateTime.truncate(:second),+ name: "Signup",+ "meta.key": ["variant"],+ "meta.value": ["A"]+ )+ ])++ insert(:goal, %{site: site, event_name: "Signup"})+ end++ describe "POST /api/stats/:domain/export - with goal filter" do+ setup [:create_user, :create_site, :log_in]++ test "exports goal-filtered data in zipped csvs", %{conn: conn, site: site} do+ populate_exported_stats(site)++ conn =+ do_export(conn, site, %{+ @base_params+ | date_range: ["2021-09-20", "2021-10-20"],+ filters: [["is", "event:goal", ["Signup"]]],+ reports: @goal_filtered_reports+ })++ assert_zip(conn, "30d-filter-goal")+ end++ # test "custom-props.csv only returns the prop names for the goal in filter", %{+ # conn: conn,+ # site: site+ # } do+ # {:ok, site} = Plausible.Props.allow(site, ["author", "logged_in"])++ # populate_stats(site, [+ # build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["uku"]),+ # build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["marko"]),+ # build(:event, name: "Newsletter Signup", "meta.key": ["author"], "meta.value": ["marko"]),+ # build(:pageview, "meta.key": ["logged_in"], "meta.value": ["true"])+ # ])++ # insert(:goal, site: site, event_name: "Newsletter Signup")++ # result =+ # conn+ # |> do_export(site, %{+ # @base_params+ # | date_range: "day",+ # filters: [["is", "event:goal", ["Newsletter Signup"]]],+ # reports: @goal_filtered_reports+ # })+ # |> response(200)+ # |> unzip_and_parse_csv(~c"custom_props.csv")++ # assert result == [+ # ["property", "value", "visitors", "events", "conversion_rate"],+ # ["author", "marko", "2", "2", "50.0"],+ # ["author", "uku", "1", "1", "25.0"],+ # [""]+ # ]+ # end++ test "exports conversions and conversion rate for operating system versions", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview, operating_system: "Mac", operating_system_version: "14"),+ build(:event, name: "Signup", operating_system: "Mac", operating_system_version: "14"),+ build(:event, name: "Signup", operating_system: "Mac", operating_system_version: "14"),+ build(:event, name: "Signup", operating_system: "Mac", operating_system_version: "14"),+ build(:event,+ name: "Signup",+ operating_system: "Ubuntu",+ operating_system_version: "20.04"+ ),+ build(:event,+ name: "Signup",+ operating_system: "Ubuntu",+ operating_system_version: "20.04"+ ),+ build(:event,+ name: "Signup",+ operating_system: "Lubuntu",+ operating_system_version: "20.04"+ )+ ])++ insert(:goal, site: site, event_name: "Signup")++ os_versions =+ conn+ |> do_export(site, %{+ @base_params+ | date_range: "day",+ filters: [["is", "event:goal", ["Signup"]]],+ reports: @goal_filtered_reports+ })+ |> response(200)+ |> unzip_and_parse_csv(~c"operating_system_versions.csv")++ assert os_versions == [+ ["name", "version", "conversions", "conversion_rate"],+ ["Mac", "14", "3", "75.0"],+ ["Ubuntu", "20.04", "2", "100.0"],+ ["Lubuntu", "20.04", "1", "100.0"],+ [""]+ ]+ end+ end+enddiff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exsindex 41d5432c4f..dc9f858d03 100644--- a/test/plausible_web/controllers/stats_controller_test.exs+++ b/test/plausible_web/controllers/stats_controller_test.exs@@ -433,7 +433,7 @@ defmodule PlausibleWeb.StatsControllerTest doApplication.put_env(:plausible, :super_admin_user_ids, [user.id])end- describe "GET /:domain/export" do+ describe "[LEGACY] GET /:domain/export" dosetup [:create_user, :create_site, :log_in]test "exports all the necessary CSV files", %{conn: conn, site: site} do@@ -937,7 +937,7 @@ defmodule PlausibleWeb.StatsControllerTest do|> Enum.map(&String.split(&1, ","))end- describe "GET /:domain/export - via shared link" do+ describe "[LEGACY] GET /:domain/export - via shared link" dosetup [:create_user, :create_site]test "exports data in zipped csvs", %{conn: conn, site: site} do@@ -956,7 +956,7 @@ defmodule PlausibleWeb.StatsControllerTest doendend- describe "GET /:domain/export - for past 6 months" do+ describe "[LEGACY] GET /:domain/export - for past 6 months" dosetup [:create_user, :create_site, :log_in]test "exports 6 months of data in zipped csvs", %{conn: conn, site: site} do@@ -966,7 +966,7 @@ defmodule PlausibleWeb.StatsControllerTest doendend- describe "GET /:domain/export - with path filter" do+ describe "[LEGACY] GET /:domain/export - with path filter" dosetup [:create_user, :create_site, :log_in]test "exports filtered data in zipped csvs", %{conn: conn, site: site} do@@ -1039,7 +1039,7 @@ defmodule PlausibleWeb.StatsControllerTest doendend- describe "GET /:domain/export - with a custom prop filter" do+ describe "[LEGACY] GET /:domain/export - with a custom prop filter" dosetup [:create_user, :create_site, :log_in]test "custom-props.csv only returns the prop and its value in filter", %{@@ -1090,13 +1090,28 @@ defmodule PlausibleWeb.StatsControllerTest doenddefp assert_csv_by_fixture({file, downloaded}, folder) do- file = Path.expand(file, folder)+ file =+ file+ |> maybe_get_legacy_variant(folder)+ |> Path.expand(folder){:ok, content} = File.read(file)msg = "CSV file comparison failed (#{file})"assert downloaded == content, message: msg, left: downloaded, right: contentend+ # Remember to clean up the legacy fixture files too once the+ # legacy CSV export disappears.+ defp maybe_get_legacy_variant(filename, folder) do+ path_filter? = String.contains?(folder, "filter-path")++ case {filename, path_filter?} do+ {~c"exit_pages.csv", true} -> ~c"exit_pages_legacy.csv"+ {~c"visitors.csv", true} -> ~c"visitors_legacy.csv"+ {filename, _} -> filename+ end+ end+defp populate_exported_stats(site) dopopulate_stats(site, [build(:pageview,@@ -1260,7 +1275,7 @@ defmodule PlausibleWeb.StatsControllerTest doinsert(:goal, %{site: site, event_name: "Signup"})end- describe "GET /:domain/export - with goal filter" do+ describe "[LEGACY] GET /:domain/export - with goal filter" dosetup [:create_user, :create_site, :log_in]test "exports goal-filtered data in zipped csvs", %{conn: conn, site: site} dodiff --git a/test/support/dev/controllers/e2e_controller.ex b/test/support/dev/controllers/e2e_controller.exindex bd61fdfbf2..90972b1194 100644--- a/test/support/dev/controllers/e2e_controller.ex+++ b/test/support/dev/controllers/e2e_controller.ex@@ -96,6 +96,11 @@ defmodule PlausibleWeb.E2EController dosend_resp(conn, 200, Jason.encode!(%{"ok" => true}))end+ def enable_dashboard_csv_export_v2(conn, %{"domain" => domain}) do+ FunWithFlags.enable(:dashboard_csv_export_v2, for_actor: "site:#{domain}")+ send_resp(conn, 200, Jason.encode!(%{"ok" => true}))+ end+defp get_goal(site, name) doPlausible.Repo.get_by!(Plausible.Goal, site_id: site.id, display_name: name)end