Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Implement filtering for imported data (#4118)
RobertJoonas1d3b0682332b
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 485433e7ba..d55b88656f 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file.### Added- Snippet integration verification+- Limited filtering support for imported data in the dashboard and via Stats API### Removeddiff --git a/assets/js/dashboard/historical.js b/assets/js/dashboard/historical.jsindex 4b93d806be..8d946525a4 100644--- a/assets/js/dashboard/historical.js+++ b/assets/js/dashboard/historical.js@@ -31,7 +31,7 @@ function Historical(props) {<ComparisonInput site={props.site} query={props.query} /></div></div>- <VisitorGraph site={props.site} query={props.query} />+ <VisitorGraph site={props.site} query={props.query} updateImportedDataInView={props.updateImportedDataInView}/><div className="w-full md:flex"><div className={ statsBoxClass }>@@ -51,7 +51,7 @@ function Historical(props) {</div></div>- <Behaviours site={props.site} query={props.query} currentUserRole={props.currentUserRole} />+ <Behaviours site={props.site} query={props.query} currentUserRole={props.currentUserRole} importedDataInView={props.importedDataInView}/></div>)}diff --git a/assets/js/dashboard/index.js b/assets/js/dashboard/index.jsindex 3fe0f7be02..b60d2f0f39 100644--- a/assets/js/dashboard/index.js+++ b/assets/js/dashboard/index.js@@ -13,8 +13,10 @@ class Dashboard extends React.Component {constructor(props) {super(props)this.updateLastLoadTimestamp = this.updateLastLoadTimestamp.bind(this)+ this.updateImportedDataInView = this.updateImportedDataInView.bind(this)this.state = {query: parseQuery(props.location.search, this.props.site),+ importedDataInView: false,lastLoadTimestamp: new Date()}}@@ -35,6 +37,10 @@ class Dashboard extends React.Component {this.setState({lastLoadTimestamp: new Date()})}+ updateImportedDataInView(newBoolean) {+ this.setState({importedDataInView: newBoolean})+ }+render() {const { site, loggedIn, currentUserRole } = this.propsconst { query, lastLoadTimestamp } = this.state@@ -42,7 +48,7 @@ class Dashboard extends React.Component {if (this.state.query.period === 'realtime') {return <Realtime site={site} loggedIn={loggedIn} currentUserRole={currentUserRole} query={query} lastLoadTimestamp={lastLoadTimestamp}/>} else {- return <Historical site={site} loggedIn={loggedIn} currentUserRole={currentUserRole} query={query} lastLoadTimestamp={lastLoadTimestamp}/>+ return <Historical site={site} loggedIn={loggedIn} currentUserRole={currentUserRole} query={query} lastLoadTimestamp={lastLoadTimestamp} importedDataInView={this.state.importedDataInView} updateImportedDataInView={this.updateImportedDataInView}/>}}}diff --git a/assets/js/dashboard/stats/behaviours/conversions.js b/assets/js/dashboard/stats/behaviours/conversions.jsindex c318c11d39..c474b57867 100644--- a/assets/js/dashboard/stats/behaviours/conversions.js+++ b/assets/js/dashboard/stats/behaviours/conversions.js@@ -6,7 +6,7 @@ import { CR_METRIC } from '../reports/metrics';import ListReport from '../reports/list';export default function Conversions(props) {- const { site, query } = props+ const { site, query, afterFetchData } = propsfunction fetchConversions() {return api.get(url.apiPath(site, '/conversions'), query, { limit: 9 })@@ -23,6 +23,7 @@ export default function Conversions(props) {return (<ListReportfetchData={fetchConversions}+ afterFetchData={afterFetchData}getFilterFor={getFilterFor}keyLabel="Goal"onClick={props.onGoalFilterClick}diff --git a/assets/js/dashboard/stats/behaviours/goal-conversions.js b/assets/js/dashboard/stats/behaviours/goal-conversions.jsindex bd3c867878..2d7b5263cc 100644--- a/assets/js/dashboard/stats/behaviours/goal-conversions.js+++ b/assets/js/dashboard/stats/behaviours/goal-conversions.js@@ -31,7 +31,7 @@ export function specialTitleWhenGoalFilter(query, defaultTitle) {}function SpecialPropBreakdown(props) {- const { site, query, prop } = props+ const { site, query, prop, afterFetchData } = propsfunction fetchData() {return api.get(url.apiPath(site, `/custom-prop-values/${prop}`), query)@@ -55,6 +55,7 @@ function SpecialPropBreakdown(props) {return (<ListReportfetchData={fetchData}+ afterFetchData={afterFetchData}getFilterFor={getFilterFor}keyLabel={prop}metrics={[@@ -73,12 +74,12 @@ function SpecialPropBreakdown(props) {}export default function GoalConversions(props) {- const {site, query} = props+ const {site, query, afterFetchData} = propsconst specialGoal = getSpecialGoal(query)if (specialGoal) {- return <SpecialPropBreakdown site={site} query={props.query} prop={specialGoal.prop} />+ return <SpecialPropBreakdown site={site} query={props.query} prop={specialGoal.prop} afterFetchData={afterFetchData} />} else {- return <Conversions site={site} query={props.query} onGoalFilterClick={props.onGoalFilterClick}/>+ return <Conversions site={site} query={props.query} onGoalFilterClick={props.onGoalFilterClick} afterFetchData={afterFetchData} />}}diff --git a/assets/js/dashboard/stats/behaviours/index.js b/assets/js/dashboard/stats/behaviours/index.jsindex 55966a703c..fba5997278 100644--- a/assets/js/dashboard/stats/behaviours/index.js+++ b/assets/js/dashboard/stats/behaviours/index.js@@ -3,7 +3,7 @@ import { Menu, Transition } from '@headlessui/react'import { ChevronDownIcon } from '@heroicons/react/20/solid'import classNames from 'classnames'import * as storage from '../../util/storage'-+import ImportedQueryUnsupportedWarning from '../imported-query-unsupported-warning'import GoalConversions, { specialTitleWhenGoalFilter } from './goal-conversions'import Properties from './props'import { FeatureSetupNotice } from '../../components/notice'@@ -48,6 +48,8 @@ export default function Behaviours(props) {const [showingPropsForGoalFilter, setShowingPropsForGoalFilter] = useState(false)+ const [importedQueryUnsupported, setImportedQueryUnsupported] = useState(false)+const onGoalFilterClick = useCallback((e) => {const goalName = e.target.innerHTMLconst isSpecialGoal = Object.keys(SPECIAL_GOALS).includes(goalName)@@ -170,9 +172,14 @@ export default function Behaviours(props) {)}+ function afterFetchData(apiResponse) {+ const unsupportedQuery = apiResponse.skip_imported_reason === 'unsupported_query'+ setImportedQueryUnsupported(unsupportedQuery && !isRealtime())+ }+function renderConversions() {if (site.hasGoals) {- return <GoalConversions site={site} query={query} onGoalFilterClick={onGoalFilterClick} />+ return <GoalConversions site={site} query={query} onGoalFilterClick={onGoalFilterClick} afterFetchData={afterFetchData}/>}else if (adminAccess) {return (@@ -224,7 +231,7 @@ export default function Behaviours(props) {function renderProps() {if (site.hasProps && site.propsAvailable) {- return <Properties site={site} query={query} />+ return <Properties site={site} query={query} afterFetchData={afterFetchData}/>} else if (adminAccess) {let callToAction@@ -330,9 +337,14 @@ export default function Behaviours(props) {<div className="items-start justify-between block w-full mt-6 md:flex"><div className="w-full p-4 bg-white rounded shadow-xl dark:bg-gray-825"><div className="flex justify-between w-full">- <h3 className="font-bold dark:text-gray-100">- {sectionTitle() + (isRealtime() ? ' (last 30min)' : '')}- </h3>+ <div className="flex gap-x-1">+ <h3 className="font-bold dark:text-gray-100">+ {sectionTitle() + (isRealtime() ? ' (last 30min)' : '')}+ </h3>+ <ImportedQueryUnsupportedWarning condition={mode === CONVERSIONS && importedQueryUnsupported}/>+ <ImportedQueryUnsupportedWarning condition={mode === PROPS && importedQueryUnsupported} message="Imported data is unavailable in this view"/>+ <ImportedQueryUnsupportedWarning condition={mode === FUNNELS && props.importedDataInView} message="Imported data is unavailable in this view"/>+ </div>{tabs()}</div>{renderContent()}diff --git a/assets/js/dashboard/stats/behaviours/props.js b/assets/js/dashboard/stats/behaviours/props.jsindex 22d32980d3..17ebe5e245 100644--- a/assets/js/dashboard/stats/behaviours/props.js+++ b/assets/js/dashboard/stats/behaviours/props.js@@ -75,6 +75,7 @@ export default function Properties(props) {return (<ListReportfetchData={fetchProps}+ afterFetchData={props.afterFetchData}getFilterFor={getFilterFor}keyLabel={propKey}metrics={[diff --git a/assets/js/dashboard/stats/devices/index.js b/assets/js/dashboard/stats/devices/index.jsindex efc22fbc4e..9abb656c78 100644--- a/assets/js/dashboard/stats/devices/index.js+++ b/assets/js/dashboard/stats/devices/index.js@@ -1,13 +1,13 @@-import React from 'react';-+import React, {useState} from 'react';import * as storage from '../../util/storage'import { getFiltersByKeyPrefix, isFilteringOnFixedValue } from '../../util/filters'import ListReport from '../reports/list'import * as api from '../../api'import * as url from '../../util/url'import { VISITORS_METRIC, PERCENTAGE_METRIC, maybeWithCR } from '../reports/metrics';+import ImportedQueryUnsupportedWarning from '../imported-query-unsupported-warning';-function Browsers({ query, site }) {+function Browsers({ query, site, afterFetchData }) {function fetchData() {return api.get(url.apiPath(site, '/browsers'), query)}@@ -22,6 +22,7 @@ function Browsers({ query, site }) {return (<ListReportfetchData={fetchData}+ afterFetchData={afterFetchData}getFilterFor={getFilterFor}keyLabel="Browser"metrics={maybeWithCR([VISITORS_METRIC, PERCENTAGE_METRIC], query)}@@ -30,7 +31,7 @@ function Browsers({ query, site }) {)}-function BrowserVersions({ query, site }) {+function BrowserVersions({ query, site, afterFetchData }) {function fetchData() {return api.get(url.apiPath(site, '/browser-versions'), query)}@@ -48,6 +49,7 @@ function BrowserVersions({ query, site }) {return (<ListReportfetchData={fetchData}+ afterFetchData={afterFetchData}getFilterFor={getFilterFor}keyLabel="Browser version"metrics={maybeWithCR([VISITORS_METRIC, PERCENTAGE_METRIC], query)}@@ -57,7 +59,7 @@ function BrowserVersions({ query, site }) {}-function OperatingSystems({ query, site }) {+function OperatingSystems({ query, site, afterFetchData }) {function fetchData() {return api.get(url.apiPath(site, '/operating-systems'), query)}@@ -72,6 +74,7 @@ function OperatingSystems({ query, site }) {return (<ListReportfetchData={fetchData}+ afterFetchData={afterFetchData}getFilterFor={getFilterFor}keyLabel="Operating system"metrics={maybeWithCR([VISITORS_METRIC, PERCENTAGE_METRIC], query)}@@ -80,7 +83,7 @@ function OperatingSystems({ query, site }) {)}-function OperatingSystemVersions({ query, site }) {+function OperatingSystemVersions({ query, site, afterFetchData }) {function fetchData() {return api.get(url.apiPath(site, '/operating-system-versions'), query)}@@ -98,6 +101,7 @@ function OperatingSystemVersions({ query, site }) {return (<ListReportfetchData={fetchData}+ afterFetchData={afterFetchData}getFilterFor={getFilterFor}keyLabel="Operating System Version"metrics={maybeWithCR([VISITORS_METRIC, PERCENTAGE_METRIC], query)}@@ -107,7 +111,7 @@ function OperatingSystemVersions({ query, site }) {}-function ScreenSizes({ query, site }) {+function ScreenSizes({ query, site, afterFetchData }) {function fetchData() {return api.get(url.apiPath(site, '/screen-sizes'), query)}@@ -126,6 +130,7 @@ function ScreenSizes({ query, site }) {return (<ListReportfetchData={fetchData}+ afterFetchData={afterFetchData}getFilterFor={getFilterFor}keyLabel="Screen size"metrics={maybeWithCR([VISITORS_METRIC, PERCENTAGE_METRIC], query)}@@ -157,45 +162,44 @@ function iconFor(screenSize) {}}-export default class Devices extends React.Component {- constructor(props) {- super(props)- this.tabKey = `deviceTab__${props.site.domain}`- const storedTab = storage.getItem(this.tabKey)- this.state = {- mode: storedTab || 'browser'- }+export default function Devices(props) {+ const {site, query} = props+ const tabKey = `deviceTab__${site.domain}`+ const storedTab = storage.getItem(tabKey)+ const [mode, setMode] = useState(storedTab || 'browser')+ const [importedQueryUnsupported, setImportedQueryUnsupported] = useState(false)++ function switchTab(mode) {+ storage.setItem(tabKey, mode)+ setMode(mode)}- setMode(mode) {- return () => {- storage.setItem(this.tabKey, mode)- this.setState({ mode })- }+ function afterFetchData(apiResponse) {+ const unsupportedQuery = apiResponse.skip_imported_reason === 'unsupported_query'+ const isRealtime = query.period === 'realtime'+ setImportedQueryUnsupported(unsupportedQuery && !isRealtime)}- renderContent() {- switch (this.state.mode) {+ function renderContent() {+ switch (mode) {case 'browser':- if (isFilteringOnFixedValue(this.props.query, 'browser')) {- return <BrowserVersions site={this.props.site} query={this.props.query} />+ if (isFilteringOnFixedValue(query, 'browser')) {+ return <BrowserVersions site={site} query={query} afterFetchData={afterFetchData} />}- return <Browsers site={this.props.site} query={this.props.query} />+ return <Browsers site={site} query={query} afterFetchData={afterFetchData} />case 'os':- if (isFilteringOnFixedValue(this.props.query, 'os')) {- return <OperatingSystemVersions site={this.props.site} query={this.props.query} />+ if (isFilteringOnFixedValue(query, 'os')) {+ return <OperatingSystemVersions site={site} query={query} afterFetchData={afterFetchData} />}- return <OperatingSystems site={this.props.site} query={this.props.query} />+ return <OperatingSystems site={site} query={query} afterFetchData={afterFetchData} />case 'size':default:- return (- <ScreenSizes site={this.props.site} query={this.props.query} />- )+ return <ScreenSizes site={site} query={query} afterFetchData={afterFetchData} />}}- renderPill(name, mode) {- const isActive = this.state.mode === mode+ function renderPill(name, pill) {+ const isActive = mode === pillif (isActive) {return (@@ -210,28 +214,29 @@ export default class Devices extends React.Component {return (<buttonclassName="cursor-pointer hover:text-indigo-600"- onClick={this.setMode(mode)}+ onClick={() => switchTab(pill)}>{name}</button>)}- render() {- return (- <div>- <div className="flex justify-between w-full">+ return (+ <div>+ <div className="flex justify-between w-full">+ <div className="flex gap-x-1"><h3 className="font-bold dark:text-gray-100">Devices</h3>- <div className="flex text-xs font-medium text-gray-500 dark:text-gray-400 space-x-2">- {this.renderPill('Browser', 'browser')}- {this.renderPill('OS', 'os')}- {this.renderPill('Size', 'size')}- </div>+ <ImportedQueryUnsupportedWarning condition={importedQueryUnsupported}/>+ </div>+ <div className="flex text-xs font-medium text-gray-500 dark:text-gray-400 space-x-2">+ {renderPill('Browser', 'browser')}+ {renderPill('OS', 'os')}+ {renderPill('Size', 'size')}</div>- {this.renderContent()}</div>- )- }+ {renderContent()}+ </div>+ )}function getSingleFilter(query, filterKey) {diff --git a/assets/js/dashboard/stats/graph/visitor-graph.js b/assets/js/dashboard/stats/graph/visitor-graph.jsindex a9b51f7aeb..c6ffa0a30a 100644--- a/assets/js/dashboard/stats/graph/visitor-graph.js+++ b/assets/js/dashboard/stats/graph/visitor-graph.js@@ -81,6 +81,9 @@ export default function VisitorGraph(props) {function fetchTopStatsAndGraphData() {fetchTopStats(site, query).then((res) => {+ if (props.updateImportedDataInView) {+ props.updateImportedDataInView(res.includes_imported)+ }setTopStatData(res)setTopStatsLoading(false)})diff --git a/assets/js/dashboard/stats/graph/with-imported-switch.js b/assets/js/dashboard/stats/graph/with-imported-switch.jsindex b376e1ee87..b7c81befa3 100644--- a/assets/js/dashboard/stats/graph/with-imported-switch.js+++ b/assets/js/dashboard/stats/graph/with-imported-switch.js@@ -22,7 +22,7 @@ export default function WithImportedSwitch({site, topStatData}) {const isComparingImportedPeriod = isBeforeNativeStats(topStatData.comparing_from)if (isQueryingImportedPeriod || isComparingImportedPeriod) {- const withImported = topStatData.with_imported;+ const withImported = topStatData.includes_imported;const toggleColor = withImported ? " dark:text-gray-300 text-gray-700" : " dark:text-gray-500 text-gray-400"const target = url.setQuery('with_imported', (!withImported).toString())const tip = withImported ? "" : "do not ";diff --git a/assets/js/dashboard/stats/imported-query-unsupported-warning.js b/assets/js/dashboard/stats/imported-query-unsupported-warning.jsnew file mode 100644index 0000000000..5c01bc78a6--- /dev/null+++ b/assets/js/dashboard/stats/imported-query-unsupported-warning.js@@ -0,0 +1,16 @@+import React from "react";+import { ExclamationCircleIcon } from '@heroicons/react/24/outline'++export default function ImportedQueryUnsupportedWarning({condition, message}) {+ const tooltipMessage = message || "Imported data is excluded due to applied filters"++ if (condition) {+ return (+ <span tooltip={tooltipMessage}>+ <ExclamationCircleIcon className="w-6 h-6 dark:text-gray-100" />+ </span>+ )+ } else {+ return null+ }+}\ No newline at end of filediff --git a/assets/js/dashboard/stats/locations/index.js b/assets/js/dashboard/stats/locations/index.jsindex 60fad2d0e8..0a80592c0c 100644--- a/assets/js/dashboard/stats/locations/index.js+++ b/assets/js/dashboard/stats/locations/index.js@@ -8,8 +8,9 @@ import {apiPath, sitePath} from '../../util/url'import ListReport from '../reports/list'import { VISITORS_METRIC, maybeWithCR } from '../reports/metrics';import { getFiltersByKeyPrefix } from '../../util/filters';+import ImportedQueryUnsupportedWarning from '../imported-query-unsupported-warning';-function Countries({query, site, onClick}) {+function Countries({query, site, onClick, afterFetchData}) {function fetchData() {return api.get(apiPath(site, '/countries'), query, { limit: 9 })}@@ -29,6 +30,7 @@ function Countries({query, site, onClick}) {return (<ListReportfetchData={fetchData}+ afterFetchData={afterFetchData}getFilterFor={getFilterFor}onClick={onClick}keyLabel="Country"@@ -41,7 +43,7 @@ function Countries({query, site, onClick}) {)}-function Regions({query, site, onClick}) {+function Regions({query, site, onClick, afterFetchData}) {function fetchData() {return api.get(apiPath(site, '/regions'), query, {limit: 9})}@@ -61,6 +63,7 @@ function Regions({query, site, onClick}) {return (<ListReportfetchData={fetchData}+ afterFetchData={afterFetchData}getFilterFor={getFilterFor}onClick={onClick}keyLabel="Region"@@ -73,7 +76,7 @@ function Regions({query, site, onClick}) {)}-function Cities({query, site}) {+function Cities({query, site, afterFetchData}) {function fetchData() {return api.get(apiPath(site, '/cities'), query, {limit: 9})}@@ -93,6 +96,7 @@ function Cities({query, site}) {return (<ListReportfetchData={fetchData}+ afterFetchData={afterFetchData}getFilterFor={getFilterFor}keyLabel="City"metrics={maybeWithCR([VISITORS_METRIC], query)}@@ -116,10 +120,12 @@ export default class Locations extends React.Component {super(props)this.onCountryFilter = this.onCountryFilter.bind(this)this.onRegionFilter = this.onRegionFilter.bind(this)+ this.afterFetchData = this.afterFetchData.bind(this)this.tabKey = `geoTab__${ props.site.domain}`const storedTab = storage.getItem(this.tabKey)this.state = {- mode: storedTab || 'map'+ mode: storedTab || 'map',+ importedQueryUnsupported: false}}@@ -156,17 +162,23 @@ export default class Locations extends React.Component {this.setMode('cities')()}+ afterFetchData(apiResponse) {+ const unsupportedQuery = apiResponse.skip_imported_reason === 'unsupported_query'+ const isRealtime = this.props.query.period === 'realtime'+ this.setState({importedQueryUnsupported: unsupportedQuery && !isRealtime})+ }+renderContent() {switch(this.state.mode) {case "cities":- return <Cities site={this.props.site} query={this.props.query} />+ return <Cities site={this.props.site} query={this.props.query} afterFetchData={this.afterFetchData} />case "regions":- return <Regions onClick={this.onRegionFilter} site={this.props.site} query={this.props.query} />+ return <Regions onClick={this.onRegionFilter} site={this.props.site} query={this.props.query} afterFetchData={this.afterFetchData} />case "countries":- return <Countries onClick={this.onCountryFilter('countries')} site={this.props.site} query={this.props.query} />+ return <Countries onClick={this.onCountryFilter('countries')} site={this.props.site} query={this.props.query} afterFetchData={this.afterFetchData} />case "map":default:- return <CountriesMap onClick={this.onCountryFilter('map')} site={this.props.site} query={this.props.query}/>+ return <CountriesMap onClick={this.onCountryFilter('map')} site={this.props.site} query={this.props.query} afterFetchData={this.afterFetchData} />}}@@ -197,9 +209,12 @@ export default class Locations extends React.Component {return (<div><div className="w-full flex justify-between">- <h3 className="font-bold dark:text-gray-100">- {labelFor[this.state.mode] || 'Locations'}- </h3>+ <div className="flex gap-x-1">+ <h3 className="font-bold dark:text-gray-100">+ {labelFor[this.state.mode] || 'Locations'}+ </h3>+ <ImportedQueryUnsupportedWarning condition={this.state.importedQueryUnsupported} />+ </div><div className="flex text-xs font-medium text-gray-500 dark:text-gray-400 space-x-2">{ this.renderPill('Map', 'map') }{ this.renderPill('Countries', 'countries') }diff --git a/assets/js/dashboard/stats/locations/map.js b/assets/js/dashboard/stats/locations/map.jsindex fbc29b0f34..4006607c80 100644--- a/assets/js/dashboard/stats/locations/map.js+++ b/assets/js/dashboard/stats/locations/map.js@@ -76,8 +76,11 @@ class Countries extends React.Component {fetchCountries() {return api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/countries`, this.props.query, {limit: 300}).then((response) => {- const results = response.results ? response.results : response- this.setState({loading: false, countries: results})+ if (this.props.afterFetchData) {+ this.props.afterFetchData(response)+ }++ this.setState({loading: false, countries: response.results})})}diff --git a/assets/js/dashboard/stats/modals/conversions.js b/assets/js/dashboard/stats/modals/conversions.jsindex eb49612dcf..1a245918cc 100644--- a/assets/js/dashboard/stats/modals/conversions.js+++ b/assets/js/dashboard/stats/modals/conversions.js@@ -37,11 +37,10 @@ function ConversionsModal(props) {function fetchData() {api.get(url.apiPath(site, `/conversions`), query, { limit: 100, page }).then((response) => {- const results = response.results ? response.results : responsesetLoading(false)- setList(list.concat(results))+ setList(list.concat(response.results))setPage(page + 1)- setMoreResultsAvailable(results.length >= 100)+ setMoreResultsAvailable(response.results.length >= 100)})}diff --git a/assets/js/dashboard/stats/modals/entry-pages.js b/assets/js/dashboard/stats/modals/entry-pages.jsindex f94ccdd919..3c7004f421 100644--- a/assets/js/dashboard/stats/modals/entry-pages.js+++ b/assets/js/dashboard/stats/modals/entry-pages.js@@ -33,15 +33,13 @@ class EntryPagesModal extends React.Component {query,{ limit: 100, page })- .then((response) => {- const results = response.results ? response.results : response-- this.setState((state) => ({+ .then(+ (response) => this.setState((state) => ({loading: false,- pages: state.pages.concat(results),- moreResultsAvailable: results.length === 100+ pages: state.pages.concat(response.results),+ moreResultsAvailable: response.results.length === 100}))- })+ )}loadMore() {diff --git a/assets/js/dashboard/stats/modals/exit-pages.js b/assets/js/dashboard/stats/modals/exit-pages.jsindex e64bd7ebb0..8dc82bec08 100644--- a/assets/js/dashboard/stats/modals/exit-pages.js+++ b/assets/js/dashboard/stats/modals/exit-pages.js@@ -28,10 +28,7 @@ class ExitPagesModal extends React.Component {const { query, page } = this.state;api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/exit-pages`, query, { limit: 100, page })- .then((response) => {- const results = response.results ? response.results : response- this.setState((state) => ({ loading: false, pages: state.pages.concat(results), moreResultsAvailable: results.length === 100 }))- })+ .then((response) => this.setState((state) => ({ loading: false, pages: state.pages.concat(response.results), moreResultsAvailable: response.results.length === 100 })))}loadMore() {diff --git a/assets/js/dashboard/stats/modals/pages.js b/assets/js/dashboard/stats/modals/pages.jsindex cad2805d4b..643938352f 100644--- a/assets/js/dashboard/stats/modals/pages.js+++ b/assets/js/dashboard/stats/modals/pages.js@@ -30,10 +30,7 @@ class PagesModal extends React.Component {const { query, page } = this.state;api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/pages`, query, { limit: 100, page, detailed })- .then((response) => {- const results = response.results ? response.results : response- this.setState((state) => ({ loading: false, pages: state.pages.concat(results), moreResultsAvailable: results.length === 100 }))- })+ .then((response) => this.setState((state) => ({ loading: false, pages: state.pages.concat(response.results), moreResultsAvailable: response.results.length === 100 })))}loadMore() {diff --git a/assets/js/dashboard/stats/modals/props.js b/assets/js/dashboard/stats/modals/props.jsindex 23876c3339..0fcd1c04f5 100644--- a/assets/js/dashboard/stats/modals/props.js+++ b/assets/js/dashboard/stats/modals/props.js@@ -39,12 +39,10 @@ function PropsModal(props) {function fetchData() {api.get(url.apiPath(site, `/custom-prop-values/${propKey}`), query, { limit: 100, page }).then((response) => {- const results = response.results ? response.results : response-setLoading(false)- setList(list.concat(results))+ setList(list.concat(response.results))setPage(page + 1)- setMoreResultsAvailable(results.length >= 100)+ setMoreResultsAvailable(response.results.length >= 100)})}diff --git a/assets/js/dashboard/stats/modals/referrer-drilldown.js b/assets/js/dashboard/stats/modals/referrer-drilldown.jsindex a4045c233d..49d33c5476 100644--- a/assets/js/dashboard/stats/modals/referrer-drilldown.js+++ b/assets/js/dashboard/stats/modals/referrer-drilldown.js@@ -21,10 +21,7 @@ class ReferrerDrilldownModal extends React.Component {const detailed = this.showExtra()api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/referrers/${this.props.match.params.referrer}`, this.state.query, {limit: 100, detailed})- .then((response) => {- const results = response.results ? response.results : response- this.setState({loading: false, referrers: results})- })+ .then((response) => this.setState({loading: false, referrers: response.results}))}showExtra() {diff --git a/assets/js/dashboard/stats/modals/sources.js b/assets/js/dashboard/stats/modals/sources.jsindex 55b9448d77..2f272f19e0 100644--- a/assets/js/dashboard/stats/modals/sources.js+++ b/assets/js/dashboard/stats/modals/sources.js@@ -35,10 +35,7 @@ class SourcesModal extends React.Component {const detailed = this.showExtra()api.get(`/api/stats/${encodeURIComponent(site.domain)}/${this.currentView()}`, query, { limit: 100, page, detailed })- .then((response) => {- const results = response.results ? response.results : response- this.setState({ loading: false, sources: sources.concat(results), moreResultsAvailable: results.length === 100 })- })+ .then((response) => this.setState({ loading: false, sources: sources.concat(response.results), moreResultsAvailable: response.results.length === 100 }))}componentDidMount() {diff --git a/assets/js/dashboard/stats/modals/table.js b/assets/js/dashboard/stats/modals/table.jsindex db02943cf6..20eea170c3 100644--- a/assets/js/dashboard/stats/modals/table.js+++ b/assets/js/dashboard/stats/modals/table.js@@ -19,10 +19,7 @@ class ModalTable extends React.Component {componentDidMount() {api.get(this.props.endpoint, this.state.query, {limit: 100})- .then((response) => {- const results = response.results ? response.results : response- this.setState({loading: false, list: results})- })+ .then((response) => this.setState({loading: false, list: response.results}))}showConversionRate() {diff --git a/assets/js/dashboard/stats/pages/index.js b/assets/js/dashboard/stats/pages/index.jsindex 8e730fc2d4..ec023ee2c9 100644--- a/assets/js/dashboard/stats/pages/index.js+++ b/assets/js/dashboard/stats/pages/index.js@@ -1,12 +1,13 @@-import React from 'react';+import React, { useState } from 'react';import * as storage from '../../util/storage'import * as url from '../../util/url'import * as api from '../../api'import ListReport from './../reports/list'import { VISITORS_METRIC, maybeWithCR } from './../reports/metrics';+import ImportedQueryUnsupportedWarning from '../imported-query-unsupported-warning';-function EntryPages({ query, site }) {+function EntryPages({ query, site, afterFetchData }) {function fetchData() {return api.get(url.apiPath(site, '/entry-pages'), query, { limit: 9 })}@@ -25,6 +26,7 @@ function EntryPages({ query, site }) {return (<ListReportfetchData={fetchData}+ afterFetchData={afterFetchData}getFilterFor={getFilterFor}keyLabel="Entry page"metrics={maybeWithCR([{ ...VISITORS_METRIC, label: 'Unique Entrances' }], query)}@@ -36,7 +38,7 @@ function EntryPages({ query, site }) {)}-function ExitPages({ query, site }) {+function ExitPages({ query, site, afterFetchData }) {function fetchData() {return api.get(url.apiPath(site, '/exit-pages'), query, { limit: 9 })}@@ -55,6 +57,7 @@ function ExitPages({ query, site }) {return (<ListReportfetchData={fetchData}+ afterFetchData={afterFetchData}getFilterFor={getFilterFor}keyLabel="Exit page"metrics={maybeWithCR([{ ...VISITORS_METRIC, label: "Unique Exits" }], query)}@@ -66,7 +69,7 @@ function ExitPages({ query, site }) {)}-function TopPages({ query, site }) {+function TopPages({ query, site, afterFetchData }) {function fetchData() {return api.get(url.apiPath(site, '/pages'), query, { limit: 9 })}@@ -85,6 +88,7 @@ function TopPages({ query, site }) {return (<ListReportfetchData={fetchData}+ afterFetchData={afterFetchData}getFilterFor={getFilterFor}keyLabel="Page"metrics={maybeWithCR([VISITORS_METRIC], query)}@@ -102,38 +106,39 @@ const labelFor = {'exit-pages': 'Exit Pages',}-export default class Pages extends React.Component {- constructor(props) {- super(props)- this.tabKey = `pageTab__${props.site.domain}`- const storedTab = storage.getItem(this.tabKey)- this.state = {- mode: storedTab || 'pages'- }+export default function Pages(props) {+ const {site, query} = props+ const tabKey = `pageTab__${site.domain}`+ const storedTab = storage.getItem(tabKey)+ const [mode, setMode] = useState(storedTab || 'pages')+ const [importedQueryUnsupported, setImportedQueryUnsupported] = useState(false)++ function switchTab(mode) {+ storage.setItem(tabKey, mode)+ setMode(mode)}- setMode(mode) {- return () => {- storage.setItem(this.tabKey, mode)- this.setState({ mode })- }+ function afterFetchData(apiResponse) {+ const unsupportedQuery = apiResponse.skip_imported_reason === 'unsupported_query'+ const isRealtime = query.period === 'realtime'+ setImportedQueryUnsupported(unsupportedQuery && !isRealtime)}- renderContent() {- switch (this.state.mode) {+ function renderContent() {+ switch (mode) {case "entry-pages":- return <EntryPages site={this.props.site} query={this.props.query} />+ return <EntryPages site={site} query={query} afterFetchData={afterFetchData} />case "exit-pages":- return <ExitPages site={this.props.site} query={this.props.query} />+ return <ExitPages site={site} query={query} afterFetchData={afterFetchData} />case "pages":default:- return <TopPages site={this.props.site} query={this.props.query} />+ return <TopPages site={site} query={query} afterFetchData={afterFetchData} />}}- renderPill(name, mode) {- const isActive = this.state.mode === mode+ function renderPill(name, pill) {+ const isActive = mode === pillif (isActive) {return (@@ -148,30 +153,31 @@ export default class Pages extends React.Component {return (<buttonclassName="hover:text-indigo-600 cursor-pointer"- onClick={this.setMode(mode)}+ onClick={() => switchTab(pill)}>{name}</button>)}- render() {- return (- <div>- {/* Header Container */}- <div className="w-full flex justify-between">+ return (+ <div>+ {/* Header Container */}+ <div className="w-full flex justify-between">+ <div className="flex gap-x-1"><h3 className="font-bold dark:text-gray-100">- {labelFor[this.state.mode] || 'Page Visits'}+ {labelFor[mode] || 'Page Visits'}</h3>- <div className="flex font-medium text-xs text-gray-500 dark:text-gray-400 space-x-2">- {this.renderPill('Top Pages', 'pages')}- {this.renderPill('Entry Pages', 'entry-pages')}- {this.renderPill('Exit Pages', 'exit-pages')}- </div>+ <ImportedQueryUnsupportedWarning condition={importedQueryUnsupported}/>+ </div>+ <div className="flex font-medium text-xs text-gray-500 dark:text-gray-400 space-x-2">+ {renderPill('Top Pages', 'pages')}+ {renderPill('Entry Pages', 'entry-pages')}+ {renderPill('Exit Pages', 'exit-pages')}</div>- {/* Main Contents */}- {this.renderContent()}</div>- )- }+ {/* Main Contents */}+ {renderContent()}+ </div>+ )}diff --git a/assets/js/dashboard/stats/reports/list.js b/assets/js/dashboard/stats/reports/list.jsindex c81ae9a0fd..52fe138db5 100644--- a/assets/js/dashboard/stats/reports/list.js+++ b/assets/js/dashboard/stats/reports/list.js@@ -110,6 +110,12 @@ function ExternalLink({ item, externalLinkDest }) {// * `color` - color of the comparison bars in light-mode+// * `afterFetchData` - a function to be called directly after `fetchData`. Receives the,+// raw API response as an argument. The return value is ignored by ListReport. Allows+// hooking into the request lifecycle and doing actions with returned metadata. For+// example, the parent component might want to control what happens when imported data+// is included or not.+export default function ListReport(props) {const [state, setState] = useState({ loading: true, list: null })const [visible, setVisible] = useState(false)@@ -125,8 +131,11 @@ export default function ListReport(props) {}props.fetchData().then((response) => {- const results = response.results ? response.results : response- setState({ loading: false, list: results })+ if (props.afterFetchData) {+ props.afterFetchData(response)+ }++ setState({ loading: false, list: response.results })})}, [props.keyLabel, props.query])diff --git a/assets/js/dashboard/stats/sources/referrer-list.js b/assets/js/dashboard/stats/sources/referrer-list.jsindex 62800335b4..daadbab21c 100644--- a/assets/js/dashboard/stats/sources/referrer-list.js+++ b/assets/js/dashboard/stats/sources/referrer-list.js@@ -1,14 +1,23 @@-import React from 'react';+import React, { useState } from 'react';import * as api from '../../api'import * as url from '../../util/url'import { VISITORS_METRIC, maybeWithCR } from '../reports/metrics'import ListReport from '../reports/list'+import ImportedQueryUnsupportedWarning from '../../stats/imported-query-unsupported-warning'export default function Referrers({source, site, query}) {+ const [importedQueryUnsupported, setImportedQueryUnsupported] = useState(false)+function fetchReferrers() {return api.get(url.apiPath(site, `/referrers/${encodeURIComponent(source)}`), query, {limit: 9})}+ function afterFetchReferrers(apiResponse) {+ const unsupportedQuery = apiResponse.skip_imported_reason === 'unsupported_query'+ const isRealtime = query.period === 'realtime'+ setImportedQueryUnsupported(unsupportedQuery && !isRealtime)+ }+function externalLinkDest(referrer) {if (referrer.name === 'Direct / None') { return null }return `https://${referrer.name}`@@ -35,9 +44,13 @@ export default function Referrers({source, site, query}) {return (<div className="flex flex-col flex-grow">- <h3 className="font-bold dark:text-gray-100">Top Referrers</h3>+ <div className="flex gap-x-1">+ <h3 className="font-bold dark:text-gray-100">Top Referrers</h3>+ <ImportedQueryUnsupportedWarning condition={importedQueryUnsupported}/>+ </div><ListReportfetchData={fetchReferrers}+ afterFetchData={afterFetchReferrers}getFilterFor={getFilterFor}keyLabel="Referrer"metrics={maybeWithCR([VISITORS_METRIC], query)}diff --git a/assets/js/dashboard/stats/sources/source-list.js b/assets/js/dashboard/stats/sources/source-list.jsindex c9bd1a7f8e..07a36bfc30 100644--- a/assets/js/dashboard/stats/sources/source-list.js+++ b/assets/js/dashboard/stats/sources/source-list.js@@ -8,6 +8,7 @@ import { VISITORS_METRIC, maybeWithCR } from '../reports/metrics';import { Menu, Transition } from '@headlessui/react'import { ChevronDownIcon } from '@heroicons/react/20/solid'import classNames from 'classnames'+import ImportedQueryUnsupportedWarning from '../imported-query-unsupported-warning';const UTM_TAGS = {utm_medium: { label: 'UTM Medium', shortLabel: 'UTM Medium', endpoint: '/utm_mediums' },@@ -43,6 +44,7 @@ function AllSources(props) {return (<ListReportfetchData={fetchData}+ afterFetchData={props.afterFetchData}getFilterFor={getFilterFor}keyLabel="Source"metrics={maybeWithCR([VISITORS_METRIC], query)}@@ -72,6 +74,7 @@ function UTMSources(props) {return (<ListReportfetchData={fetchData}+ afterFetchData={props.afterFetchData}getFilterFor={getFilterFor}keyLabel={utmTag.label}metrics={maybeWithCR([VISITORS_METRIC], query)}@@ -87,6 +90,7 @@ export default function SourceList(props) {const tabKey = 'sourceTab__' + props.site.domainconst storedTab = storage.getItem(tabKey)const [currentTab, setCurrentTab] = useState(storedTab || 'all')+ const [importedQueryUnsupported, setImportedQueryUnsupported] = useState(false)function setTab(tab) {return () => {@@ -152,19 +156,28 @@ export default function SourceList(props) {function renderContent() {if (currentTab === 'all') {- return <AllSources site={site} query={query} />+ return <AllSources site={site} query={query} afterFetchData={afterFetchData} />} else {- return <UTMSources tab={currentTab} site={site} query={query} />+ return <UTMSources tab={currentTab} site={site} query={query} afterFetchData={afterFetchData} />}}+ function afterFetchData(apiResponse) {+ const unsupportedQuery = apiResponse.skip_imported_reason === 'unsupported_query'+ const isRealtime = query.period === 'realtime'+ setImportedQueryUnsupported(unsupportedQuery && !isRealtime)+ }+return (<div>{/* Header Container */}<div className="w-full flex justify-between">- <h3 className="font-bold dark:text-gray-100">- Top Sources- </h3>+ <div className="flex gap-x-1">+ <h3 className="font-bold dark:text-gray-100">+ Top Sources+ </h3>+ <ImportedQueryUnsupportedWarning condition={importedQueryUnsupported}/>+ </div>{renderTabs()}</div>{/* Main Contents */}diff --git a/lib/plausible/imported.ex b/lib/plausible/imported.exindex b8e00f19d8..35fa348922 100644--- a/lib/plausible/imported.ex+++ b/lib/plausible/imported.ex@@ -54,6 +54,11 @@ defmodule Plausible.Imported do@max_complete_importsend+ @spec imported_custom_props() :: [String.t()]+ def imported_custom_props do+ Plausible.Props.internal_keys()+ end+@spec goals_with_url() :: [String.t()]def goals_with_url() do@goals_with_urldiff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.exindex 07d14a7871..108e29f7f9 100644--- a/lib/plausible/stats/base.ex+++ b/lib/plausible/stats/base.ex@@ -308,8 +308,10 @@ defmodule Plausible.Stats.Base dodef add_percentage_metric(q, site, query, metrics) doif :percentage in metrics do+ total_query = Query.set_property(query, nil, skip_refresh_imported_opts: true)+q- |> select_merge(^%{__total_visitors: total_visitors_subquery(site, query)})+ |> select_merge(^%{__total_visitors: total_visitors_subquery(site, total_query)})|> select_merge(%{percentage:fragment(@@ -329,7 +331,10 @@ defmodule Plausible.Stats.Base do# filters.def maybe_add_conversion_rate(q, site, query, metrics) doif :conversion_rate in metrics do- total_query = query |> Query.remove_filters(["event:goal", "event:props"])+ total_query =+ query+ |> Query.remove_filters(["event:goal", "event:props"], skip_refresh_imported_opts: true)+ |> Query.set_property(nil, skip_refresh_imported_opts: true)# :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALLsubquery(q)diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex 305d80838b..23db74a9e0 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -37,11 +37,10 @@ defmodule Plausible.Stats.Breakdown do{event_goals, pageview_goals} = Enum.split_with(site.goals, & &1.event_name)events = Enum.map(event_goals, & &1.event_name)- event_query = %Query{+ event_query =query- | filters: query.filters ++ [[:member, "event:name", events]],- property: "event:name"- }+ |> Query.put_filter([:member, "event:name", events])+ |> Query.set_property("event:name")if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)@@ -74,12 +73,14 @@ defmodule Plausible.Stats.Breakdown dopage_q =if Enum.any?(pageview_goals) do+ page_query = Query.set_property(query, "event:page")+page_exprs = Enum.map(pageview_goals, & &1.page_path)page_regexes = Enum.map(page_exprs, &page_regex/1)select_columns = metrics_to_select |> select_event_metrics |> mark_revenue_as_nil- from(e in base_event_query(site, query),+ from(e in base_event_query(site, page_query),order_by: [desc: fragment("uniq(?)", e.user_id)],where:fragment(@@ -94,7 +95,7 @@ defmodule Plausible.Stats.Breakdown do})|> select_merge(^select_columns)- |> merge_imported_pageview_goals(site, query, page_exprs, metrics_to_select)+ |> merge_imported_pageview_goals(site, page_query, page_exprs, metrics_to_select)|> apply_pagination(pagination)elsenil@@ -180,8 +181,9 @@ defmodule Plausible.Stats.Breakdown dopages ->query+ |> Query.remove_filters(["event:page"])|> Query.put_filter([:member, "visit:entry_page", Enum.map(pages, & &1[:page])])- |> struct!(property: "visit:entry_page")+ |> Query.set_property("visit:entry_page")endif Enum.any?(event_metrics) && Enum.empty?(event_result) do@@ -244,24 +246,25 @@ defmodule Plausible.Stats.Breakdown do"visit:entry_page","visit:referrer"] do- update_hostname(query, "visit:entry_page_hostname")+ update_hostname_filter_prop(query, "visit:entry_page_hostname")enddefp maybe_update_breakdown_filters(%Query{property: "visit:exit_page"} = query) do- update_hostname(query, "visit:exit_page_hostname")+ update_hostname_filter_prop(query, "visit:exit_page_hostname")enddefp maybe_update_breakdown_filters(query) doqueryend- defp update_hostname(query, visit_prop) do+ defp update_hostname_filter_prop(query, visit_prop) docase Query.get_filter(query, "event:hostname") donil ->query[op, "event:hostname", value] ->- Plausible.Stats.Query.put_filter(query, [op, visit_prop, value])+ query+ |> Query.put_filter([op, visit_prop, value])endenddiff --git a/lib/plausible/stats/comparisons.ex b/lib/plausible/stats/comparisons.exindex 86ee0f3554..dcd34bcd79 100644--- a/lib/plausible/stats/comparisons.ex+++ b/lib/plausible/stats/comparisons.ex@@ -63,7 +63,7 @@ defmodule Plausible.Stats.Comparisons dowith :ok <- validate_mode(source_query, mode),{:ok, comparison_query} <- do_compare(source_query, mode, opts),- comparison_query <- maybe_include_imported(comparison_query, source_query, site),+ comparison_query <- maybe_include_imported(comparison_query, source_query),do: {:ok, comparison_query}end@@ -162,12 +162,10 @@ defmodule Plausible.Stats.Comparisons doDate.add(date, -days_to_subtract)end- defp maybe_include_imported(query, %Query{imported_data_requested: false}, _site) do- %Stats.Query{query | include_imported: false}- end+ defp maybe_include_imported(query, source_query) do+ requested? = source_query.imported_data_requested- defp maybe_include_imported(query, _source_query, site) do- case Query.ensure_include_imported(query, site) do+ case Query.ensure_include_imported(query, requested?) do:ok ->struct!(query,imported_data_requested: true,@@ -176,7 +174,7 @@ defmodule Plausible.Stats.Comparisons do{:error, reason} ->struct!(query,- imported_data_requested: true,+ imported_data_requested: requested?,include_imported: false,skip_imported_reason: reason)diff --git a/lib/plausible/stats/email_report.ex b/lib/plausible/stats/email_report.exindex 63809e3af8..675597731b 100644--- a/lib/plausible/stats/email_report.ex+++ b/lib/plausible/stats/email_report.ex@@ -40,7 +40,7 @@ defmodule Plausible.Stats.EmailReport doenddefp put_top_5_pages(stats, site, query) do- query = struct!(query, property: "event:page")+ query = Query.set_property(query, "event:page")pages = Stats.breakdown(site, query, [:visitors], {5, 1})Map.put(stats, :pages, pages)end@@ -49,7 +49,7 @@ defmodule Plausible.Stats.EmailReport doquery =query|> Query.put_filter([:is_not, "visit:source", "Direct / None"])- |> struct!(property: "visit:source")+ |> Query.set_property("visit:source")sources = Stats.breakdown(site, query, [:visitors], {5, 1})diff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.exindex 6e38044f99..8713919e3e 100644--- a/lib/plausible/stats/filter_suggestions.ex+++ b/lib/plausible/stats/filter_suggestions.ex@@ -2,9 +2,12 @@ defmodule Plausible.Stats.FilterSuggestions douse Plausible.Repouse Plausible.ClickhouseRepouse Plausible.Stats.Fragments+import Plausible.Stats.Baseimport Ecto.Query+alias Plausible.Stats.Query+ alias Plausible.Stats.Importeddef filter_suggestions(site, query, "country", filter_search) domatches = Location.search_country(filter_search)@@ -16,6 +19,7 @@ defmodule Plausible.Stats.FilterSuggestions doorder_by: [desc: fragment("count(*)")],select: e.country_code)+ |> Imported.merge_imported_country_suggestions(site, query)ClickhouseRepo.all(q)|> Enum.map(fn c -> Enum.find(matches, fn x -> x.alpha_2 == c end) end)@@ -35,33 +39,60 @@ defmodule Plausible.Stats.FilterSuggestions dogroup_by: e.subdivision1_code,order_by: [desc: fragment("count(*)")],select: e.subdivision1_code,- where: e.subdivision1_code != "",- limit: 24+ where: e.subdivision1_code != "")+ |> Imported.merge_imported_region_suggestions(site, query)+ |> limit(24)|> ClickhouseRepo.all()|> Enum.map(fn c ->subdiv = Location.get_subdivision(c)- %{- value: c,- label: subdiv.name- }+ if subdiv do+ %{+ value: c,+ label: subdiv.name+ }+ else+ %{+ value: c,+ label: c+ }+ endend)enddef filter_suggestions(site, query, "region", filter_search) domatches = Location.search_subdivision(filter_search)+ filter_search = String.downcase(filter_search)q =from(e in query_sessions(site, query),group_by: e.subdivision1_code,order_by: [desc: fragment("count(*)")],- select: e.subdivision1_code+ select: e.subdivision1_code,+ where: e.subdivision1_code != "")+ |> Imported.merge_imported_region_suggestions(site, query)ClickhouseRepo.all(q)- |> Enum.map(fn c -> Enum.find(matches, fn x -> x.code == c end) end)+ |> Enum.map(fn c ->+ match = Enum.find(matches, fn x -> x.code == c end)++ cond do+ match ->+ match++ String.contains?(String.downcase(c), filter_search) ->+ %{+ code: c,+ name: c+ }++ true ->+ nil+ end+ end)|> Enum.filter(& &1)|> Enum.slice(0..24)|> Enum.map(fn subdiv ->@@ -78,9 +109,10 @@ defmodule Plausible.Stats.FilterSuggestions dogroup_by: e.city_geoname_id,order_by: [desc: fragment("count(*)")],select: e.city_geoname_id,- where: e.city_geoname_id != 0,- limit: 24+ where: e.city_geoname_id != 0)+ |> Imported.merge_imported_city_suggestions(site, query)+ |> limit(24)|> ClickhouseRepo.all()|> Enum.map(fn c ->city = Location.get_city(c)@@ -101,9 +133,10 @@ defmodule Plausible.Stats.FilterSuggestions dogroup_by: e.city_geoname_id,order_by: [desc: fragment("count(*)")],select: e.city_geoname_id,- where: e.city_geoname_id != 0,- limit: 5000+ where: e.city_geoname_id != 0)+ |> Imported.merge_imported_city_suggestions(site, query)+ |> limit(5000)ClickhouseRepo.all(q)|> Enum.map(fn c -> Location.get_city(c) end)@@ -223,10 +256,16 @@ defmodule Plausible.Stats.FilterSuggestions dowhere: fragment("? ilike ?", field(e, ^filter_name), ^filter_query),select: field(e, ^filter_name),group_by: ^filter_name,- order_by: [desc: fragment("count(*)")],- limit: 25+ order_by: [desc: fragment("count(*)")])|> apply_additional_filters(filter_name, site)+ |> Imported.merge_imported_filter_suggestions(+ site,+ query,+ filter_name,+ filter_query+ )+ |> limit(25)|> ClickhouseRepo.all()|> Enum.filter(fn suggestion -> suggestion != "" end)|> wrap_suggestions()diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.exnew file mode 100644index 0000000000..e15e09d62c--- /dev/null+++ b/lib/plausible/stats/imported/base.ex@@ -0,0 +1,191 @@+defmodule Plausible.Stats.Imported.Base do+ @moduledoc """+ A module for building the base of an imported stats query+ """++ import Ecto.Query++ alias Plausible.Imported+ alias Plausible.Stats.Filters+ alias Plausible.Stats.Query++ @property_to_table_mappings %{+ "visit:source" => "imported_sources",+ "visit:referrer" => "imported_sources",+ "visit:utm_source" => "imported_sources",+ "visit:utm_medium" => "imported_sources",+ "visit:utm_campaign" => "imported_sources",+ "visit:utm_term" => "imported_sources",+ "visit:utm_content" => "imported_sources",+ "visit:entry_page" => "imported_entry_pages",+ "visit:exit_page" => "imported_exit_pages",+ "visit:country" => "imported_locations",+ "visit:region" => "imported_locations",+ "visit:city" => "imported_locations",+ "visit:device" => "imported_devices",+ "visit:browser" => "imported_browsers",+ "visit:browser_version" => "imported_browsers",+ "visit:os" => "imported_operating_systems",+ "visit:os_version" => "imported_operating_systems",+ "event:page" => "imported_pages",+ "event:name" => "imported_custom_events",+ "event:props:url" => "imported_custom_events",+ "event:props:path" => "imported_custom_events",++ # NOTE: these properties can be only filtered by+ "visit:screen" => "imported_devices",+ "event:hostname" => "imported_pages"+ }++ @imported_custom_props Imported.imported_custom_props()++ @db_field_mappings %{+ referrer_source: :source,+ screen_size: :device,+ screen: :device,+ os: :operating_system,+ os_version: :operating_system_version,+ country_code: :country,+ subdivision1_code: :region,+ city_geoname_id: :city,+ entry_page_hostname: :hostname,+ pathname: :page,+ url: :link_url+ }++ def property_to_table_mappings(), do: @property_to_table_mappings++ def query_imported(site, query) do+ query+ |> transform_filters()+ |> decide_table()+ |> query_imported(site, query)+ end++ def query_imported(table, site, query) do+ query = transform_filters(query)+ import_ids = site.complete_import_ids+ %{first: date_from, last: date_to} = query.date_range++ from(i in table,+ where: i.site_id == ^site.id,+ where: i.import_id in ^import_ids,+ where: i.date >= ^date_from,+ where: i.date <= ^date_to,+ select: %{}+ )+ |> apply_filter(query)+ end++ def decide_table(query) do+ query+ |> transform_filters()+ |> do_decide_table()+ end++ defp transform_filters(query) do+ new_filters =+ query.filters+ |> Enum.reject(fn+ [:is, "event:name", "pageview"] -> true+ _ -> false+ end)+ |> Enum.flat_map(fn filter ->+ case filter do+ [op, "event:goal", {:event, name}] ->+ [[op, "event:name", name]]++ [op, "event:goal", {:page, page}] ->+ [[op, "event:page", page]]++ [op, "event:goal", events] ->+ events+ |> Enum.group_by(&elem(&1, 0), &elem(&1, 1))+ |> Enum.map(fn+ {:event, names} -> [op, "event:name", names]+ {:page, pages} -> [op, "event:page", pages]+ end)++ filter ->+ [filter]+ end+ end)++ struct!(query, filters: new_filters)+ end++ defp do_decide_table(%Query{filters: [], property: nil}), do: "imported_visitors"++ defp do_decide_table(%Query{filters: filters, property: "event:props:" <> prop_key = property})+ when prop_key in @imported_custom_props do+ has_required_name_filter? =+ Enum.any?(filters, fn+ [:is, "event:name", name] -> name in special_goals_for(prop_key)+ _ -> false+ end)++ has_unsupported_filters? =+ Enum.any?(filters, fn [_, filtered_prop | _] ->+ filtered_prop not in [property, "event:name"]+ end)++ if has_required_name_filter? && not has_unsupported_filters? do+ "imported_custom_events"+ else+ nil+ end+ end++ defp do_decide_table(%Query{filters: [], property: "event:goal"}) do+ "imported_custom_events"+ end++ defp do_decide_table(%Query{filters: [], property: property}) do+ @property_to_table_mappings[property]+ end++ defp do_decide_table(%Query{filters: filters, property: "event:goal"}) do+ filter_props = Enum.map(filters, &Enum.at(&1, 1))++ any_event_name_filters? = "event:name" in filter_props+ any_page_filters? = "event:page" in filter_props+ any_other_filters? = Enum.any?(filter_props, &(&1 not in ["event:page", "event:name"]))++ cond do+ any_other_filters? -> nil+ any_event_name_filters? and not any_page_filters? -> "imported_custom_events"+ any_page_filters? and not any_event_name_filters? -> "imported_pages"+ true -> nil+ end+ end++ defp do_decide_table(%Query{filters: filters, property: property}) do+ table_candidates =+ filters+ |> Enum.map(fn [_, prop | _] -> prop end)+ |> Enum.concat(if property, do: [property], else: [])+ |> Enum.map(fn+ "visit:screen" -> "visit:device"+ prop -> prop+ end)+ |> Enum.map(&@property_to_table_mappings[&1])++ case Enum.uniq(table_candidates) do+ [candidate] -> candidate+ _ -> nil+ end+ end++ defp apply_filter(q, %Query{filters: filters}) do+ Enum.reduce(filters, q, fn [_, filtered_prop | _] = filter, q ->+ db_field = Plausible.Stats.Filters.without_prefix(filtered_prop)+ mapped_db_field = Map.get(@db_field_mappings, db_field, db_field)+ condition = Filters.WhereBuilder.build_condition(mapped_db_field, filter)++ where(q, ^condition)+ end)+ end++ def special_goals_for("url"), do: Imported.goals_with_url()+ def special_goals_for("path"), do: Imported.goals_with_path()+enddiff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported/imported.exsimilarity index 65%rename from lib/plausible/stats/imported.exrename to lib/plausible/stats/imported/imported.exindex b6623587aa..2589a23873 100644--- a/lib/plausible/stats/imported.ex+++ b/lib/plausible/stats/imported/imported.ex@@ -1,43 +1,29 @@defmodule Plausible.Stats.Imported douse Plausible.ClickhouseRepo- alias Plausible.Stats.{Query, Base}import Ecto.Queryimport Plausible.Stats.Fragments+ alias Plausible.Stats.Base+ alias Plausible.Stats.Imported+ alias Plausible.Stats.Query+@no_ref "Direct / None"@not_set "(not set)"@none "(none)"- @property_to_table_mappings %{- "visit:source" => "imported_sources",- "visit:referrer" => "imported_sources",- "visit:utm_source" => "imported_sources",- "visit:utm_medium" => "imported_sources",- "visit:utm_campaign" => "imported_sources",- "visit:utm_term" => "imported_sources",- "visit:utm_content" => "imported_sources",- "visit:entry_page" => "imported_entry_pages",- "visit:exit_page" => "imported_exit_pages",- "visit:country" => "imported_locations",- "visit:region" => "imported_locations",- "visit:city" => "imported_locations",- "visit:device" => "imported_devices",- "visit:browser" => "imported_browsers",- "visit:browser_version" => "imported_browsers",- "visit:os" => "imported_operating_systems",- "visit:os_version" => "imported_operating_systems",- "event:page" => "imported_pages",- "event:name" => "imported_custom_events",- "event:props:url" => "imported_custom_events",- "event:props:path" => "imported_custom_events"- }+ @property_to_table_mappings Imported.Base.property_to_table_mappings()@imported_properties Map.keys(@property_to_table_mappings)@goals_with_url Plausible.Imported.goals_with_url()++ def goals_with_url(), do: @goals_with_url+@goals_with_path Plausible.Imported.goals_with_path()+ def goals_with_path(), do: @goals_with_path+@doc """Returns a boolean indicating whether the combination of filters andbreakdown property is possible to query from the imported tables.@@ -49,17 +35,193 @@ defmodule Plausible.Stats.Imported do(see `@goals_with_url` and `@goals_with_path`)."""def schema_supports_query?(query) do- filter_count = length(query.filters)-- case {filter_count, query.property} do- {0, "event:props:" <> _} -> false- {0, _} -> true- {1, "event:props:url"} -> has_special_goal_filter?(query, @goals_with_url)- {1, "event:props:path"} -> has_special_goal_filter?(query, @goals_with_path)- {_, _} -> false+ not is_nil(Imported.Base.decide_table(query))+ end++ def merge_imported_country_suggestions(native_q, _site, %Plausible.Stats.Query{+ include_imported: false+ }) do+ native_q+ end++ def merge_imported_country_suggestions(native_q, site, query) do+ supports_filter_set? =+ Enum.all?(query.filters, fn filter ->+ [_, filtered_prop | _] = filter+ @property_to_table_mappings[filtered_prop] == "imported_locations"+ end)++ if supports_filter_set? do+ native_q =+ native_q+ |> exclude(:order_by)+ |> exclude(:select)+ |> select([e], %{country_code: e.country_code, count: fragment("count(*)")})++ imported_q =+ from i in Imported.Base.query_imported("imported_locations", site, query),+ group_by: i.country,+ select_merge: %{country_code: i.country, count: fragment("sum(?)", i.pageviews)}++ from(s in subquery(native_q),+ full_join: i in subquery(imported_q),+ on: s.country_code == i.country_code,+ select:+ fragment("if(not empty(?), ?, ?)", s.country_code, s.country_code, i.country_code),+ order_by: [desc: fragment("? + ?", s.count, i.count)]+ )+ else+ native_q+ end+ end++ def merge_imported_region_suggestions(native_q, _site, %Plausible.Stats.Query{+ include_imported: false+ }) do+ native_q+ end++ def merge_imported_region_suggestions(native_q, site, query) do+ supports_filter_set? =+ Enum.all?(query.filters, fn filter ->+ [_, filtered_prop | _] = filter+ @property_to_table_mappings[filtered_prop] == "imported_locations"+ end)++ if supports_filter_set? do+ native_q =+ native_q+ |> exclude(:order_by)+ |> exclude(:select)+ |> select([e], %{region_code: e.subdivision1_code, count: fragment("count(*)")})++ imported_q =+ from i in Imported.Base.query_imported("imported_locations", site, query),+ where: i.region != "",+ group_by: i.region,+ select_merge: %{region_code: i.region, count: fragment("sum(?)", i.pageviews)}++ from(s in subquery(native_q),+ full_join: i in subquery(imported_q),+ on: s.region_code == i.region_code,+ select: fragment("if(not empty(?), ?, ?)", s.region_code, s.region_code, i.region_code),+ order_by: [desc: fragment("? + ?", s.count, i.count)]+ )+ else+ native_q+ end+ end++ def merge_imported_city_suggestions(native_q, _site, %Plausible.Stats.Query{+ include_imported: false+ }) do+ native_q+ end++ def merge_imported_city_suggestions(native_q, site, query) do+ supports_filter_set? =+ Enum.all?(query.filters, fn filter ->+ [_, filtered_prop | _] = filter+ @property_to_table_mappings[filtered_prop] == "imported_locations"+ end)++ if supports_filter_set? do+ native_q =+ native_q+ |> exclude(:order_by)+ |> exclude(:select)+ |> select([e], %{city_id: e.city_geoname_id, count: fragment("count(*)")})++ imported_q =+ from i in Imported.Base.query_imported("imported_locations", site, query),+ where: i.city != 0,+ group_by: i.city,+ select_merge: %{city_id: i.city, count: fragment("sum(?)", i.pageviews)}++ from(s in subquery(native_q),+ full_join: i in subquery(imported_q),+ on: s.city_id == i.city_id,+ select: fragment("if(? > 0, ?, ?)", s.city_id, s.city_id, i.city_id),+ order_by: [desc: fragment("? + ?", s.count, i.count)]+ )+ else+ native_q+ end+ end++ def merge_imported_filter_suggestions(+ native_q,+ _site,+ %Plausible.Stats.Query{include_imported: false},+ _filter_name,+ _filter_search+ ) do+ native_q+ end++ def merge_imported_filter_suggestions(+ native_q,+ site,+ query,+ filter_name,+ filter_query+ ) do+ {table, db_field} = expand_suggestions_field(filter_name)++ supports_filter_set? =+ Enum.all?(query.filters, fn filter ->+ [_, filtered_prop | _] = filter+ @property_to_table_mappings[filtered_prop] == table+ end)++ if supports_filter_set? do+ native_q =+ native_q+ |> exclude(:order_by)+ |> exclude(:select)+ |> select([e], %{name: field(e, ^filter_name), count: fragment("count(*)")})++ imported_q =+ from i in Imported.Base.query_imported(table, site, query),+ where: fragment("? ilike ?", field(i, ^db_field), ^filter_query),+ group_by: field(i, ^db_field),+ select_merge: %{name: field(i, ^db_field), count: fragment("sum(?)", i.pageviews)}++ from(s in subquery(native_q),+ full_join: i in subquery(imported_q),+ on: s.name == i.name,+ select: fragment("if(not empty(?), ?, ?)", s.name, s.name, i.name),+ order_by: [desc: fragment("? + ?", s.count, i.count)],+ limit: 25+ )+ else+ native_qendend+ @filter_suggestions_mapping %{+ referrer_source: :source,+ screen_size: :device,+ pathname: :page+ }++ defp expand_suggestions_field(filter_name) do+ db_field = Map.get(@filter_suggestions_mapping, filter_name, filter_name)++ property =+ case db_field do+ :operating_system -> :os+ :operating_system_version -> :os_version+ other -> other+ end++ table_by_visit = Map.get(@property_to_table_mappings, "visit:#{property}")+ table_by_event = Map.get(@property_to_table_mappings, "event:#{property}")+ table = table_by_visit || table_by_event++ {table, db_field}+ end+def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{include_imported: false}, _),do: native_q@@ -69,15 +231,9 @@ defmodule Plausible.Stats.Imported doquery,metrics) do- import_ids = site.complete_import_ids-imported_q =- from(v in "imported_visitors",- where: v.site_id == ^site.id,- where: v.import_id in ^import_ids,- where: v.date >= ^query.date_range.first and v.date <= ^query.date_range.last,- select: %{}- )+ site+ |> Imported.Base.query_imported(query)|> select_imported_metrics(metrics)|> apply_interval(query, site)@@ -111,19 +267,12 @@ defmodule Plausible.Stats.Imported dodef merge_imported(q, site, %Query{property: property} = query, metrics)when property in @imported_properties do- table = Map.fetch!(@property_to_table_mappings, property)dim = Plausible.Stats.Filters.without_prefix(property)- import_ids = site.complete_import_idsimported_q =- from(- i in table,- where: i.site_id == ^site.id,- where: i.import_id in ^import_ids,- where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,- where: i.visitors > 0,- select: %{}- )+ site+ |> Imported.Base.query_imported(query)+ |> where([i], i.visitors > 0)|> maybe_apply_filter(query, property, dim)|> group_imported_by(dim)|> select_imported_metrics(metrics)@@ -155,7 +304,8 @@ defmodule Plausible.Stats.Imported dodef merge_imported(q, site, %Query{property: nil} = query, metrics) doimported_q =- imported_visitors(site, query)+ site+ |> Imported.Base.query_imported(query)|> select_imported_metrics(metrics)from(@@ -171,69 +321,40 @@ defmodule Plausible.Stats.Imported dodef merge_imported_pageview_goals(q, _, %Query{include_imported: false}, _, _), do: qdef merge_imported_pageview_goals(q, site, query, page_exprs, metrics) do- page_regexes = Enum.map(page_exprs, &Base.page_regex/1)-- imported_q =- from(- i in "imported_pages",- where: i.site_id == ^site.id,- where: i.import_id in ^site.complete_import_ids,- where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,- where: i.visitors > 0,- where:- fragment(- "notEmpty(multiMatchAllIndices(?, ?) as indices)",- i.page,- ^page_regexes- ),- array_join: index in fragment("indices"),- group_by: index,- select: %{+ if Imported.Base.decide_table(query) == "imported_pages" do+ page_regexes = Enum.map(page_exprs, &Base.page_regex/1)++ imported_q =+ "imported_pages"+ |> Imported.Base.query_imported(site, query)+ |> where([i], i.visitors > 0)+ |> where(+ [i],+ fragment("notEmpty(multiMatchAllIndices(?, ?) as indices)", i.page, ^page_regexes)+ )+ |> join(:array, index in fragment("indices"))+ |> group_by([_i, index], index)+ |> select_merge([_i, index], %{name: fragment("concat('Visit ', ?[?])", ^page_exprs, index)- }- )- |> select_imported_metrics(metrics)-- from(s in Ecto.Query.subquery(q),- full_join: i in subquery(imported_q),- on: s.name == i.name,- select: %{}- )- |> select_joined_dimension(:name)- |> select_joined_metrics(metrics)- end-- def total_imported_visitors(site, query) do- imported_visitors(site, query)- |> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)})- end-- defp imported_visitors(site, query) do- import_ids = site.complete_import_ids+ })+ |> select_imported_metrics(metrics)- from(- i in "imported_visitors",- where: i.site_id == ^site.id,- where: i.import_id in ^import_ids,- where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,- select: %{}- )- end-- defp maybe_apply_filter(q, query, "event:props:url", _) do- if name = find_special_goal_filter(query, @goals_with_url) do- where(q, [i], i.name == ^name)+ from(s in Ecto.Query.subquery(q),+ full_join: i in subquery(imported_q),+ on: s.name == i.name,+ select: %{}+ )+ |> select_joined_dimension(:name)+ |> select_joined_metrics(metrics)elseqendend- defp maybe_apply_filter(q, query, "event:props:path", _) do- if name = find_special_goal_filter(query, @goals_with_path) do- where(q, [i], i.name == ^name)- else- q- end+ def total_imported_visitors(site, query) do+ site+ |> Imported.Base.query_imported(query)+ |> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)})enddefp maybe_apply_filter(q, query, property, dim) do@@ -243,20 +364,6 @@ defmodule Plausible.Stats.Imported doendend- defp has_special_goal_filter?(query, event_names) do- not is_nil(find_special_goal_filter(query, event_names))- end-- defp find_special_goal_filter(query, event_names) do- case Query.get_filter(query, "event:goal") do- [:is, "event:goal", {:event, name}] ->- if name in event_names, do: name, else: nil-- _ ->- nil- end- end-defp select_imported_metrics(q, []), do: qdefp select_imported_metrics(q, [:visitors | rest]) do@@ -320,6 +427,18 @@ defmodule Plausible.Stats.Imported do|> select_imported_metrics(rest)end+ defp select_imported_metrics(+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_pages", _}}} = q,+ [:bounce_rate | rest]+ ) do+ q+ |> select_merge([i], %{+ bounces: 0,+ __internal_visits: 0+ })+ |> select_imported_metrics(rest)+ end+defp select_imported_metrics(%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,[:bounce_rate | rest]@@ -353,6 +472,18 @@ defmodule Plausible.Stats.Imported do|> select_imported_metrics(rest)end+ defp select_imported_metrics(+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_pages", _}}} = q,+ [:visit_duration | rest]+ ) do+ q+ |> select_merge([i], %{+ visit_duration: 0,+ __internal_visits: 0+ })+ |> select_imported_metrics(rest)+ end+defp select_imported_metrics(%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,[:visit_duration | rest]@@ -386,6 +517,32 @@ defmodule Plausible.Stats.Imported do|> select_imported_metrics(rest)end+ defp select_imported_metrics(+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,+ [:views_per_visit | rest]+ ) do+ q+ |> where([i], i.pageviews > 0)+ |> select_merge([i], %{+ pageviews: sum(i.pageviews),+ __internal_visits: sum(i.entrances)+ })+ |> select_imported_metrics(rest)+ end++ defp select_imported_metrics(+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,+ [:views_per_visit | rest]+ ) do+ q+ |> where([i], i.pageviews > 0)+ |> select_merge([i], %{+ pageviews: sum(i.pageviews),+ __internal_visits: sum(i.exits)+ })+ |> select_imported_metrics(rest)+ end+defp select_imported_metrics(q, [:views_per_visit | rest]) doq|> where([i], i.pageviews > 0)@@ -558,7 +715,7 @@ defmodule Plausible.Stats.Imported doenddefp select_joined_metrics(q, []), do: q- # TODO: Reverse-engineering the native data bounces and total visit+ # NOTE: Reverse-engineering the native data bounces and total visit# durations to combine with imported data is inefficient. Instead both# queries should fetch bounces/total_visit_duration and visits and be# used as subqueries to a main query that then find the bounce rate/avgdiff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex 90f347d61a..261840c831 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -12,13 +12,15 @@ defmodule Plausible.Stats.Query doskip_imported_reason: nil,now: nil,experimental_session_count?: false,- experimental_reduced_joins?: false+ experimental_reduced_joins?: false,+ latest_import_end_date: nilrequire OpenTelemetry.Tracer, as: Traceralias Plausible.Stats.{Filters, Interval, Imported}@type t :: %__MODULE__{}+ @spec from(Plausible.Site.t(), map()) :: t()def from(site, params) donow = NaiveDateTime.utc_now(:second)@@ -201,19 +203,43 @@ defmodule Plausible.Stats.Query dostruct!(query, filters: Filters.parse(params["filters"]))end+ @spec set_property(t(), String.t() | nil, Keyword.t()) :: t()+ def set_property(query, property, opts \\ []) do+ query = struct!(query, property: property)++ if Keyword.get(opts, :skip_refresh_imported_opts),+ do: query,+ else: refresh_imported_opts(query)+ end+def put_filter(query, filter) do- struct!(query,- filters: query.filters ++ [filter]- )+ query+ |> struct!(filters: query.filters ++ [filter])+ |> refresh_imported_opts()end- def remove_filters(query, prefixes) do+ def remove_filters(query, prefixes, opts \\ []) donew_filters =Enum.reject(query.filters, fn [_, filter_key | _rest] ->Enum.any?(prefixes, &String.starts_with?(filter_key, &1))end)- struct!(query, filters: new_filters)+ query = struct!(query, filters: new_filters)++ if Keyword.get(opts, :skip_refresh_imported_opts),+ do: query,+ else: refresh_imported_opts(query)+ end++ def exclude_imported(query) do+ struct!(query,+ include_imported: false,+ skip_imported_reason: :manual_exclusion+ )+ end++ defp refresh_imported_opts(query) do+ put_imported_opts(query, nil, %{})enddef has_event_filters?(query) do@@ -247,13 +273,22 @@ defmodule Plausible.Stats.Query doenddefp put_imported_opts(query, site, params) do- requested? = params["with_imported"] == "true"+ requested? = params["with_imported"] == "true" || query.imported_data_requested++ latest_import_end_date =+ if site do+ site.latest_import_end_date+ else+ query.latest_import_end_date+ end- case ensure_include_imported(query, site) do+ query = struct!(query, latest_import_end_date: latest_import_end_date)++ case ensure_include_imported(query, requested?) do:ok ->struct!(query,- imported_data_requested: requested?,- include_imported: requested?+ imported_data_requested: true,+ include_imported: true){:error, reason} ->@@ -265,12 +300,13 @@ defmodule Plausible.Stats.Query doendend- @spec ensure_include_imported(t(), Plausible.Site.t()) ::- :ok | {:error, :no_imported_data | :out_of_range | :unsupported_query}- def ensure_include_imported(query, site) do+ @spec ensure_include_imported(t(), boolean()) ::+ :ok | {:error, :not_requested | :no_imported_data | :out_of_range | :unsupported_query}+ def ensure_include_imported(query, requested?) docond do- is_nil(site.latest_import_end_date) -> {:error, :no_imported_data}- Date.after?(query.date_range.first, site.latest_import_end_date) -> {:error, :out_of_range}+ not requested? -> {:error, :not_requested}+ is_nil(query.latest_import_end_date) -> {:error, :no_imported_data}+ Date.after?(query.date_range.first, query.latest_import_end_date) -> {:error, :out_of_range}not Imported.schema_supports_query?(query) -> {:error, :unsupported_query}query.period == "realtime" -> {:error, :unsupported_query}true -> :okdiff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.exindex 08dfaf18bf..bab37c6e22 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/timeseries.ex@@ -1,7 +1,7 @@defmodule Plausible.Stats.Timeseries douse Plausible.ClickhouseRepouse Plausible- alias Plausible.Stats.{Query, Util}+ alias Plausible.Stats.{Query, Util, Imported}import Plausible.Stats.{Base}import Ecto.Queryuse Plausible.Stats.Fragments@@ -56,8 +56,8 @@ defmodule Plausible.Stats.Timeseries dofrom(e in base_event_query(site, query), select: ^select_event_metrics(metrics))|> select_bucket(:events, site, query)+ |> Imported.merge_imported_timeseries(site, query, metrics)|> maybe_add_timeseries_conversion_rate(site, query, metrics)- |> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics)|> ClickhouseRepo.all()end@@ -67,7 +67,7 @@ defmodule Plausible.Stats.Timeseries dofrom(e in query_sessions(site, query), select: ^select_session_metrics(metrics, query))|> filter_converted_sessions(site, query)|> select_bucket(:sessions, site, query)- |> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics)+ |> Imported.merge_imported_timeseries(site, query, metrics)|> ClickhouseRepo.all()|> Util.keep_requested_metrics(metrics)end@@ -314,13 +314,16 @@ defmodule Plausible.Stats.Timeseries dodefp maybe_add_timeseries_conversion_rate(q, site, query, metrics) doif :conversion_rate in metrics do- totals_query = query |> Query.remove_filters(["event:goal", "event:props"])+ totals_query =+ query+ |> Query.remove_filters(["event:goal", "event:props"], skip_refresh_imported_opts: true)totals_timeseries_q =from(e in base_event_query(site, totals_query),select: ^select_event_metrics([:visitors]))- |> select_bucket(:events, site, query)+ |> select_bucket(:events, site, totals_query)+ |> Imported.merge_imported_timeseries(site, totals_query, [:visitors])from(e in subquery(q),left_join: c in subquery(totals_timeseries_q),diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.exindex 77e3d51ca5..f8145b8ccf 100644--- a/lib/plausible_web/controllers/api/external_stats_controller.ex+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex@@ -380,7 +380,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController doenddefp maybe_add_warning(payload, %{skip_imported_reason: reason})- when reason in [nil, :no_imported_data, :out_of_range] do+ when reason in [nil, :not_requested, :no_imported_data, :out_of_range, :manual_exclusion] dopayloadenddiff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex bc9086ca84..868194ffa7 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -65,7 +65,7 @@ defmodule PlausibleWeb.Api.StatsController do* `interval` - the interval used for querying.- * `with_imported` - boolean indicating whether the Google Analytics data+ * `includes_imported` - boolean indicating whether imported datawas queried or not.* `imports_exist` - boolean indicating whether there are any completed@@ -92,7 +92,7 @@ defmodule PlausibleWeb.Api.StatsController do"labels" => ["2021-09-01", "2021-10-01", "2021-11-01", "2021-12-01"],"plot" => [0, 0, 0, 0],"present_index" => nil,- "with_imported" => false+ "includes_imported" => false}```@@ -137,7 +137,7 @@ defmodule PlausibleWeb.Api.StatsController docomparison_labels: comparison_result && label_timeseries(comparison_result, nil),present_index: present_index,interval: query.interval,- with_imported: with_imported?(query, comparison_query),+ includes_imported: includes_imported?(query, comparison_query),imports_exist: site.complete_import_ids != [],full_intervals: full_intervals})@@ -217,7 +217,7 @@ defmodule PlausibleWeb.Api.StatsController dotop_stats: top_stats,interval: query.interval,sample_percent: sample_percent,- with_imported: with_imported?(query, comparison_query),+ includes_imported: includes_imported?(query, comparison_query),imports_exist: site.complete_import_ids != [],comparing_from: comparison_query && comparison_query.date_range.first,comparing_to: comparison_query && comparison_query.date_range.last,@@ -381,26 +381,15 @@ defmodule PlausibleWeb.Api.StatsController doenddefp fetch_other_top_stats(site, query, comparison_query) do+ page_filter? = Query.get_filter(query, "event:page")++ metrics = [:visitors, :visits, :pageviews, :sample_percent]+metrics =- if Query.get_filter(query, "event:page") do- [- :visitors,- :visits,- :pageviews,- :bounce_rate,- :time_on_page,- :sample_percent- ]- else- [- :visitors,- :visits,- :pageviews,- :views_per_visit,- :bounce_rate,- :visit_duration,- :sample_percent- ]+ cond do+ page_filter? && query.include_imported -> metrics+ page_filter? -> metrics ++ [:bounce_rate, :time_on_page]+ true -> metrics ++ [:views_per_visit, :bounce_rate, :visit_duration]endcurrent_results = Stats.aggregate(site, query, metrics)@@ -492,7 +481,10 @@ defmodule PlausibleWeb.Api.StatsController dores |> to_csv([:name, :visitors, :bounce_rate, :visit_duration])endelse- json(conn, res)+ json(conn, %{+ results: res,+ skip_imported_reason: query.skip_imported_reason+ })endend@@ -570,7 +562,10 @@ defmodule PlausibleWeb.Api.StatsController dores |> to_csv([:name, :visitors, :bounce_rate, :visit_duration])endelse- json(conn, res)+ json(conn, %{+ results: res,+ skip_imported_reason: query.skip_imported_reason+ })endend@@ -594,7 +589,10 @@ defmodule PlausibleWeb.Api.StatsController dores |> to_csv([:name, :visitors, :bounce_rate, :visit_duration])endelse- json(conn, res)+ json(conn, %{+ results: res,+ skip_imported_reason: query.skip_imported_reason+ })endend@@ -618,7 +616,10 @@ defmodule PlausibleWeb.Api.StatsController dores |> to_csv([:name, :visitors, :bounce_rate, :visit_duration])endelse- json(conn, res)+ json(conn, %{+ results: res,+ skip_imported_reason: query.skip_imported_reason+ })endend@@ -642,7 +643,10 @@ defmodule PlausibleWeb.Api.StatsController dores |> to_csv([:name, :visitors, :bounce_rate, :visit_duration])endelse- json(conn, res)+ json(conn, %{+ results: res,+ skip_imported_reason: query.skip_imported_reason+ })endend@@ -666,7 +670,10 @@ defmodule PlausibleWeb.Api.StatsController dores |> to_csv([:name, :visitors, :bounce_rate, :visit_duration])endelse- json(conn, res)+ json(conn, %{+ results: res,+ skip_imported_reason: query.skip_imported_reason+ })endend@@ -690,7 +697,10 @@ defmodule PlausibleWeb.Api.StatsController dores |> to_csv([:name, :visitors, :bounce_rate, :visit_duration])endelse- json(conn, res)+ json(conn, %{+ results: res,+ skip_imported_reason: query.skip_imported_reason+ })endend@@ -743,7 +753,10 @@ defmodule PlausibleWeb.Api.StatsController doStats.breakdown(site, query, metrics, pagination)|> transform_keys(%{referrer: :name})- json(conn, referrers)+ json(conn, %{+ results: referrers,+ skip_imported_reason: query.skip_imported_reason+ })enddef pages(conn, params) do@@ -772,7 +785,10 @@ defmodule PlausibleWeb.Api.StatsController dopages |> to_csv([:name, :visitors, :pageviews, :bounce_rate, :time_on_page])endelse- json(conn, pages)+ json(conn, %{+ results: pages,+ skip_imported_reason: query.skip_imported_reason+ })endend@@ -803,7 +819,10 @@ defmodule PlausibleWeb.Api.StatsController do])endelse- json(conn, entry_pages)+ json(conn, %{+ results: entry_pages,+ skip_imported_reason: query.skip_imported_reason+ })endend@@ -835,7 +854,10 @@ defmodule PlausibleWeb.Api.StatsController do])endelse- json(conn, exit_pages)+ json(conn, %{+ results: exit_pages,+ skip_imported_reason: query.skip_imported_reason+ })endend@@ -845,14 +867,15 @@ defmodule PlausibleWeb.Api.StatsController doelsepages = Enum.map(breakdown_results, & &1[:exit_page])- total_visits_query =+ total_pageviews_query =query+ |> Query.remove_filters(["visit:exit_page"])|> Query.put_filter([:member, "event:page", pages])|> Query.put_filter([:is, "event:name", "pageview"])- |> struct!(property: "event:page")+ |> Query.set_property("event:page")total_pageviews =- Stats.breakdown(site, total_visits_query, [:pageviews], {limit, 1})+ Stats.breakdown(site, total_pageviews_query, [:pageviews], {limit, 1})Enum.map(breakdown_results, fn result ->exit_rate =@@ -917,7 +940,10 @@ defmodule PlausibleWeb.Api.StatsController doendend)- json(conn, countries)+ json(conn, %{+ results: countries,+ skip_imported_reason: query.skip_imported_reason+ })endend@@ -952,7 +978,10 @@ defmodule PlausibleWeb.Api.StatsController doregions |> to_csv([:name, :visitors])endelse- json(conn, regions)+ json(conn, %{+ results: regions,+ skip_imported_reason: query.skip_imported_reason+ })endend@@ -992,7 +1021,10 @@ defmodule PlausibleWeb.Api.StatsController docities |> to_csv([:name, :visitors])endelse- json(conn, cities)+ json(conn, %{+ results: cities,+ skip_imported_reason: query.skip_imported_reason+ })endend@@ -1016,7 +1048,10 @@ defmodule PlausibleWeb.Api.StatsController dobrowsers |> to_csv([:name, :visitors])endelse- json(conn, browsers)+ json(conn, %{+ results: browsers,+ skip_imported_reason: query.skip_imported_reason+ })endend@@ -1046,7 +1081,10 @@ defmodule PlausibleWeb.Api.StatsController do|> to_csv([:name, :version, :visitors])endelse- json(conn, versions)+ json(conn, %{+ results: versions,+ skip_imported_reason: query.skip_imported_reason+ })endend@@ -1070,7 +1108,10 @@ defmodule PlausibleWeb.Api.StatsController dosystems |> to_csv([:name, :visitors])endelse- json(conn, systems)+ json(conn, %{+ results: systems,+ skip_imported_reason: query.skip_imported_reason+ })endend@@ -1096,7 +1137,10 @@ defmodule PlausibleWeb.Api.StatsController do|> to_csv([:name, :version, :visitors])endelse- json(conn, versions)+ json(conn, %{+ results: versions,+ skip_imported_reason: query.skip_imported_reason+ })endend@@ -1120,7 +1164,10 @@ defmodule PlausibleWeb.Api.StatsController dosizes |> to_csv([:name, :visitors])endelse- json(conn, sizes)+ json(conn, %{+ results: sizes,+ skip_imported_reason: query.skip_imported_reason+ })endend@@ -1156,7 +1203,10 @@ defmodule PlausibleWeb.Api.StatsController do:total_conversions])else- json(conn, conversions)+ json(conn, %{+ results: conversions,+ skip_imported_reason: query.skip_imported_reason+ })endend@@ -1166,8 +1216,7 @@ defmodule PlausibleWeb.Api.StatsController docase Plausible.Props.ensure_prop_key_accessible(prop_key, site.owner) do:ok ->- props = breakdown_custom_prop_values(site, params)- json(conn, props)+ json(conn, breakdown_custom_prop_values(site, params)){:error, :upgrade_required} ->H.payment_required(@@ -1195,6 +1244,7 @@ defmodule PlausibleWeb.Api.StatsController doprop_names|> Enum.map(fn prop_key ->breakdown_custom_prop_values(site, Map.put(params, "prop_key", prop_key))+ |> Map.get(:results)|> Enum.map(&Map.put(&1, :property, prop_key))|> transform_keys(%{:name => :value})end)@@ -1224,12 +1274,15 @@ defmodule PlausibleWeb.Api.StatsController do[:visitors, :events, :percentage] ++ @revenue_metricsend- Stats.breakdown(site, query, metrics, pagination)- |> transform_keys(%{prop_key => :name})- |> Enum.map(fn entry ->- Enum.map(entry, &format_revenue_metric/1)- |> Map.new()- end)+ props =+ Stats.breakdown(site, query, metrics, pagination)+ |> transform_keys(%{prop_key => :name})+ |> Enum.map(fn entry ->+ Enum.map(entry, &format_revenue_metric/1)+ |> Map.new()+ end)++ %{results: props, skip_imported_reason: query.skip_imported_reason}enddef current_visitors(conn, _) do@@ -1401,7 +1454,7 @@ defmodule PlausibleWeb.Api.StatsController do]end- defp with_imported?(source_query, comparison_query) do+ defp includes_imported?(source_query, comparison_query) docond dosource_query.include_imported -> truecomparison_query && comparison_query.include_imported -> truediff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exsindex 5dcbe84601..68cfefd859 100644--- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs@@ -561,14 +561,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do}end- test "ignores imported data when filters are applied", %{+ test "includes imported data in comparison when filter applied", %{conn: conn,site: site,site_import: site_import} dopopulate_stats(site, site_import.id, [build(:imported_visitors, date: ~D[2023-01-01]),- build(:imported_sources, date: ~D[2023-01-01]),+ build(:imported_sources, source: "Google", date: ~D[2023-01-01], visitors: 3),build(:pageview,referrer_source: "Google",timestamp: ~N[2023-01-02 00:10:00]@@ -587,7 +587,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do})assert json_response(conn, 200)["results"] == %{- "visitors" => %{"value" => 1, "change" => 100}+ "visitors" => %{"value" => 1, "change" => -67}}end@@ -681,6 +681,60 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest dorefute json_response(conn, 200)["warning"]end++ test "excludes imported data from conversion rate computation when query filters by non-imported props",+ %{conn: conn, site: site, site_import: site_import} do+ insert(:goal, site: site, event_name: "Purchase")++ populate_stats(site, site_import.id, [+ build(:event,+ name: "Purchase",+ "meta.key": ["package"],+ "meta.value": ["large"]+ ),+ build(:imported_visitors, visitors: 9)+ ])++ conn =+ get(conn, "/api/v1/stats/aggregate", %{+ "site_id" => site.domain,+ "period" => "day",+ "metrics" => "visitors,conversion_rate",+ "filters" => "event:goal==Purchase;event:props:package==large",+ "with_imported" => "true"+ })++ assert json_response(conn, 200)["results"] == %{+ "visitors" => %{"value" => 1},+ "conversion_rate" => %{"value" => 100.0}+ }+ end++ test "returns stats with page + pageview goal filter",+ %{conn: conn, site: site, site_import: site_import} do+ insert(:goal, site: site, page_path: "/blog/**")++ populate_stats(site, site_import.id, [+ build(:imported_pages, page: "/blog/1", visitors: 1, pageviews: 1),+ build(:imported_pages, page: "/blog/2", visitors: 2, pageviews: 2),+ build(:imported_pages, visitors: 3)+ ])++ conn =+ get(conn, "/api/v1/stats/aggregate", %{+ "site_id" => site.domain,+ "period" => "day",+ "metrics" => "visitors,events,conversion_rate",+ "filters" => "event:page==/blog/2;event:goal==Visit /blog/**",+ "with_imported" => "true"+ })++ assert json_response(conn, 200)["results"] == %{+ "visitors" => %{"value" => 2},+ "events" => %{"value" => 2},+ "conversion_rate" => %{"value" => 100.0}+ }+ endenddescribe "filters" dodiff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exsindex 1125722da2..8be42e3187 100644--- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs@@ -3158,6 +3158,37 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doenddescribe "imported data" do+ test "returns screen sizes breakdown when filtering by screen size", %{conn: conn, site: site} do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:pageview,+ timestamp: ~N[2021-01-01 00:00:01],+ screen_size: "Mobile"+ ),+ build(:imported_devices,+ device: "Mobile",+ visitors: 3,+ pageviews: 5,+ date: ~D[2021-01-01]+ )+ ])++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "period" => "day",+ "date" => "2021-01-01",+ "property" => "visit:device",+ "filters" => "visit:device==Mobile",+ "metrics" => "visitors,pageviews",+ "with_imported" => "true"+ })++ assert [%{"pageviews" => 6, "visitors" => 4, "device" => "Mobile"}] =+ json_response(conn, 200)["results"]+ end+test "returns custom event goals and pageview goals", %{conn: conn, site: site} doinsert(:goal, site: site, event_name: "Purchase")insert(:goal, site: site, page_path: "/test")@@ -3478,5 +3509,64 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest dorefute json_response(conn, 200)["warning"]end++ test "applies multiple filters if the properties belong to the same table", %{+ conn: conn,+ site: site+ } do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:imported_sources, source: "Google", utm_medium: "organic", utm_term: "one"),+ build(:imported_sources, source: "Twitter", utm_medium: "organic", utm_term: "two"),+ build(:imported_sources,+ source: "Facebook",+ utm_medium: "something_else",+ utm_term: "one"+ )+ ])++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "period" => "day",+ "property" => "visit:source",+ "filters" => "visit:utm_medium==organic;visit:utm_term==one",+ "with_imported" => "true"+ })++ assert json_response(conn, 200) == %{+ "results" => [%{"source" => "Google", "visitors" => 1}]+ }+ end++ test "ignores imported data if filtered property belongs to a different table than the breakdown property",+ %{+ conn: conn,+ site: site+ } do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:imported_sources, source: "Google"),+ build(:imported_devices, device: "Desktop")+ ])++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "period" => "day",+ "property" => "visit:source",+ "filters" => "visit:device==Desktop",+ "with_imported" => "true"+ })++ assert %{+ "results" => [],+ "warning" => warning+ } = json_response(conn, 200)++ assert warning =~ "Imported stats are not included in the results"+ endendenddiff --git a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exsindex d1ecccfa32..9149711aa3 100644--- a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs@@ -1647,5 +1647,240 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest dorefute json_response(conn, 200)["warning"]end++ test "returns all metrics based on imported/native data when filtering by browser", %{+ conn: conn,+ site: site+ } do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:pageview, browser: "Chrome", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, browser: "Chrome", user_id: 1, timestamp: ~N[2021-01-01 00:03:00]),+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),+ build(:imported_browsers, browser: "Firefox", date: ~D[2021-01-02]),+ build(:imported_browsers,+ browser: "Chrome",+ visitors: 1,+ pageviews: 1,+ bounces: 1,+ visit_duration: 3,+ visits: 1,+ date: ~D[2021-01-03]+ ),+ build(:pageview, browser: "Chrome", user_id: 2, timestamp: ~N[2021-01-04 00:00:00]),+ build(:event,+ name: "Signup",+ browser: "Chrome",+ user_id: 2,+ timestamp: ~N[2021-01-04 00:10:00]+ ),+ build(:imported_browsers,+ browser: "Chrome",+ visitors: 4,+ pageviews: 6,+ bounces: 1,+ visit_duration: 300,+ visits: 5,+ date: ~D[2021-01-04]+ )+ ])++ results =+ conn+ |> get("/api/v1/stats/timeseries", %{+ "site_id" => site.domain,+ "period" => "custom",+ "date" => "2021-01-01,2021-01-04",+ "metrics" =>+ "visitors,pageviews,events,visits,views_per_visit,bounce_rate,visit_duration",+ "filters" => "visit:browser==Chrome",+ "with_imported" => "true"+ })+ |> json_response(200)+ |> Map.get("results")++ assert results == [+ %{+ "bounce_rate" => 0.0,+ "date" => "2021-01-01",+ "events" => 2,+ "pageviews" => 2,+ "views_per_visit" => 2.0,+ "visit_duration" => 180.0,+ "visitors" => 1,+ "visits" => 1+ },+ %{+ "bounce_rate" => nil,+ "date" => "2021-01-02",+ "events" => 0,+ "pageviews" => 0,+ "views_per_visit" => 0.0,+ "visit_duration" => nil,+ "visitors" => 0,+ "visits" => 0+ },+ %{+ "bounce_rate" => 100,+ "date" => "2021-01-03",+ "events" => 1,+ "pageviews" => 1,+ "views_per_visit" => 1.0,+ "visit_duration" => 3,+ "visitors" => 1,+ "visits" => 1+ },+ %{+ "bounce_rate" => 17.0,+ "date" => "2021-01-04",+ "events" => 8,+ "pageviews" => 7,+ "views_per_visit" => 1.17,+ "visit_duration" => 150,+ "visitors" => 5,+ "visits" => 6+ }+ ]+ end++ test "returns conversion rate timeseries with a goal filter", %{+ conn: conn,+ site: site+ } do+ site_import = insert(:site_import, site: site)++ insert(:goal, site: site, event_name: "Outbound Link: Click")++ populate_stats(site, site_import.id, [+ # 2021-01-01+ build(:event, name: "Outbound Link: Click", timestamp: ~N[2021-01-01 00:00:00]),+ build(:imported_custom_events, name: "Outbound Link: Click", date: ~D[2021-01-01]),+ build(:imported_visitors, date: ~D[2021-01-01], visitors: 4),+ # 2021-01-02+ build(:event, name: "Outbound Link: Click", timestamp: ~N[2021-01-02 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-02 00:00:00]),+ # 2021-01-03+ build(:imported_custom_events, name: "Outbound Link: Click", date: ~D[2021-01-03]),+ build(:imported_visitors, date: ~D[2021-01-03]),+ # 2021-01-04+ build(:event, name: "Outbound Link: Click", timestamp: ~N[2021-01-04 00:00:00]),+ build(:imported_visitors, date: ~D[2021-01-04], visitors: 2)+ ])++ results =+ conn+ |> get("/api/v1/stats/timeseries", %{+ "site_id" => site.domain,+ "period" => "custom",+ "date" => "2021-01-01,2021-01-04",+ "metrics" => "conversion_rate",+ "filters" => "event:goal==Outbound Link: Click",+ "with_imported" => "true"+ })+ |> json_response(200)+ |> Map.get("results")++ assert results == [+ %{+ "date" => "2021-01-01",+ "conversion_rate" => 40.0+ },+ %{+ "date" => "2021-01-02",+ "conversion_rate" => 50.0+ },+ %{+ "date" => "2021-01-03",+ "conversion_rate" => 100.0+ },+ %{+ "date" => "2021-01-04",+ "conversion_rate" => 33.3+ }+ ]+ end++ test "returns conversion rate timeseries with a goal + custom prop filter", %{+ conn: conn,+ site: site+ } do+ site_import = insert(:site_import, site: site)++ insert(:goal, site: site, event_name: "Outbound Link: Click")++ populate_stats(site, site_import.id, [+ # 2021-01-01+ build(:event,+ name: "Outbound Link: Click",+ "meta.key": ["url"],+ "meta.value": ["https://site.com"],+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:imported_custom_events,+ name: "Outbound Link: Click",+ link_url: "https://site.com",+ date: ~D[2021-01-01]+ ),+ build(:imported_custom_events,+ name: "File Download",+ link_url: "https://site.com",+ date: ~D[2021-01-01]+ ),+ build(:imported_custom_events,+ name: "Outbound Link: Click",+ link_url: "https://notthis.com",+ date: ~D[2021-01-01]+ ),+ build(:imported_visitors, date: ~D[2021-01-01], visitors: 4),+ # 2021-01-03+ build(:imported_custom_events,+ name: "Outbound Link: Click",+ link_url: "https://site.com",+ date: ~D[2021-01-03]+ ),+ build(:imported_visitors, date: ~D[2021-01-03]),+ # 2021-01-04+ build(:event,+ name: "Outbound Link: Click",+ "meta.key": ["url"],+ "meta.value": ["https://site.com"],+ timestamp: ~N[2021-01-04 00:00:00]+ ),+ build(:imported_visitors, date: ~D[2021-01-04], visitors: 2)+ ])++ results =+ conn+ |> get("/api/v1/stats/timeseries", %{+ "site_id" => site.domain,+ "period" => "custom",+ "date" => "2021-01-01,2021-01-04",+ "metrics" => "conversion_rate",+ "filters" => "event:goal==Outbound Link: Click;event:props:url==https://site.com",+ "with_imported" => "true"+ })+ |> json_response(200)+ |> Map.get("results")++ assert results == [+ %{+ "date" => "2021-01-01",+ "conversion_rate" => 40.0+ },+ %{+ "date" => "2021-01-02",+ "conversion_rate" => 0.0+ },+ %{+ "date" => "2021-01-03",+ "conversion_rate" => 100.0+ },+ %{+ "date" => "2021-01-04",+ "conversion_rate" => 33.3+ }+ ]+ endendenddiff --git a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs b/test/plausible_web/controllers/api/stats_controller/browsers_test.exsindex bd504c50ba..1c2d8fc806 100644--- a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs@@ -13,7 +13,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest doconn = get(conn, "/api/stats/#{site.domain}/browsers?period=day")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Chrome", "visitors" => 2, "percentage" => 66.7},%{"name" => "Firefox", "visitors" => 1, "percentage" => 33.3}]@@ -47,7 +47,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest dofilters = Jason.encode!(%{props: %{"author" => "John Doe"}})conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Chrome", "visitors" => 1, "percentage" => 100}]end@@ -82,7 +82,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest dofilters = Jason.encode!(%{props: %{"author" => "!John Doe"}})conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Firefox", "visitors" => 1, "percentage" => 50},%{"name" => "Safari", "visitors" => 1, "percentage" => 50}]@@ -99,7 +99,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest doconn = get(conn, "/api/stats/#{site.domain}/browsers?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Chrome","total_visitors" => 2,@@ -123,13 +123,13 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest doconn = get(conn, "/api/stats/#{site.domain}/browsers?period=day")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Chrome", "visitors" => 1, "percentage" => 100}]conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Chrome", "visitors" => 2, "percentage" => 66.7},%{"name" => "Firefox", "visitors" => 1, "percentage" => 33.3}]@@ -154,7 +154,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest doconn = get(conn, "/api/stats/#{site.domain}/browsers?period=day&with_imported=true")- assert json_response(conn, 200) == []+ assert json_response(conn, 200)["results"] == []endtest "returns (not set) when appropriate", %{conn: conn, site: site} do@@ -167,7 +167,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest doconn = get(conn, "/api/stats/#{site.domain}/browsers?period=day")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "(not set)", "visitors" => 1, "percentage" => 100.0}]end@@ -185,7 +185,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest doconn = get(conn, "/api/stats/#{site.domain}/browsers?period=day&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "(not set)", "visitors" => 2, "percentage" => 100.0}]end@@ -220,6 +220,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do"/api/stats/#{site.domain}/browser-versions?period=day&filters=#{filters}")|> json_response(200)+ |> Map.get("results")assert %{"browser" => "Chrome",@@ -254,7 +255,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do"/api/stats/#{site.domain}/browser-versions?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "78.0", "visitors" => 2, "percentage" => 66.7, "browser" => "Chrome"},%{"name" => "77.0", "visitors" => 1, "percentage" => 33.3, "browser" => "Chrome"}]@@ -273,7 +274,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do"/api/stats/#{site.domain}/browser-versions?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "(not set)","visitors" => 1,@@ -317,7 +318,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do"/api/stats/#{site.domain}/browser-versions?period=day&date=2021-01-01&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"browser" => "(not set)","name" => "(not set)",diff --git a/test/plausible_web/controllers/api/stats_controller/cities_test.exs b/test/plausible_web/controllers/api/stats_controller/cities_test.exsindex 903d6d625e..0f97188f23 100644--- a/test/plausible_web/controllers/api/stats_controller/cities_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/cities_test.exs@@ -37,7 +37,7 @@ defmodule PlausibleWeb.Api.StatsController.CitiesTest dotest "returns top cities by new visitors", %{conn: conn, site: site} doconn = get(conn, "/api/stats/#{site.domain}/cities?period=day")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"code" => 588_409, "country_flag" => "🇪🇪", "name" => "Tallinn", "visitors" => 3},%{"code" => 591_632, "country_flag" => "🇪🇪", "name" => "Kärdla", "visitors" => 2}]@@ -47,7 +47,7 @@ defmodule PlausibleWeb.Api.StatsController.CitiesTest dofilters = Jason.encode!(%{city: "591632"})conn = get(conn, "/api/stats/#{site.domain}/cities?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"code" => 591_632, "country_flag" => "🇪🇪", "name" => "Kärdla", "visitors" => 2}]end@@ -61,7 +61,7 @@ defmodule PlausibleWeb.Api.StatsController.CitiesTest doconn = get(conn, "/api/stats/#{site.domain}/cities?period=day&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"code" => 588_409, "country_flag" => "🇪🇪", "name" => "Tallinn", "visitors" => 4},%{"code" => 591_632, "country_flag" => "🇪🇪", "name" => "Kärdla", "visitors" => 2}]diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exsindex f932456dfb..24e18ed2da 100644--- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs@@ -32,7 +32,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest doconn = get(conn, "/api/stats/#{site.domain}/conversions?period=day")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Signup","visitors" => 2,@@ -79,7 +79,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest dofilters = Jason.encode!(%{props: %{"logged_in" => "true"}})conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Payment","visitors" => 1,@@ -119,7 +119,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest dofilters = Jason.encode!(%{props: %{"logged_in" => "!true"}})conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Payment","visitors" => 2,@@ -157,7 +157,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest dofilters = Jason.encode!(%{props: %{"logged_in" => "(none)"}})conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Payment","visitors" => 2,@@ -197,7 +197,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest dofilters = Jason.encode!(%{props: %{"logged_in" => "!(none)"}})conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Payment","visitors" => 2,@@ -215,6 +215,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest doconn|> get("/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}")|> json_response(200)+ |> Map.get("results")assert resp == []end@@ -249,7 +250,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest dofilters = Jason.encode!(%{browser: "Firefox"})conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Payment","visitors" => 1,@@ -294,7 +295,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest doconn = get(conn, "/api/stats/#{site.domain}/conversions?period=day")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Payment","visitors" => 5,@@ -340,7 +341,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest doconn = get(conn, "/api/stats/#{site.domain}/conversions?period=day")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Payment","visitors" => 5,@@ -372,7 +373,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest doinsert(:goal, %{site: site, event_name: "Payment", currency: :EUR})conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day")- response = json_response(conn, 200)+ response = json_response(conn, 200)["results"]assert [%{@@ -414,7 +415,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest doconn = get(conn, "/api/stats/#{site.domain}/conversions?period=day")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Signup","visitors" => 1,@@ -447,6 +448,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest doget(conn, path <> query)|> json_response(200)+ |> Map.get("results")endexpected = [@@ -488,6 +490,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest doget(conn, path <> query)|> json_response(200)+ |> Map.get("results")endexpected = [@@ -539,7 +542,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do"/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Signup","visitors" => 2,@@ -573,7 +576,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do"/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Visit /blog/**","visitors" => 2,@@ -611,7 +614,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do"/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Visit /blog**","visitors" => 2,@@ -649,7 +652,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do"/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Signup","visitors" => 1,@@ -713,7 +716,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do"/api/stats/#{site.domain}/conversions?period=day&date=2019-07-01")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"conversion_rate" => 100.0,"visitors" => 8,@@ -801,7 +804,291 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do"events" => 3,"conversion_rate" => 37.5}- ] = json_response(conn, 200)+ ] = json_response(conn, 200)["results"]+ end++ test "returns only custom event goals with a custom event goal filter", %{+ conn: conn,+ site: site+ } do+ insert(:goal, site: site, event_name: "Purchase")+ insert(:goal, site: site, event_name: "Activation")+ insert(:goal, site: site, page_path: "/test")++ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:pageview,+ timestamp: ~N[2021-01-01 00:00:01],+ pathname: "/test"+ ),+ build(:event,+ name: "Purchase",+ timestamp: ~N[2021-01-01 00:00:03]+ ),+ build(:event,+ name: "Purchase",+ timestamp: ~N[2021-01-01 00:00:03]+ ),+ build(:imported_custom_events,+ name: "Purchase",+ visitors: 3,+ events: 5,+ date: ~D[2021-01-01]+ ),+ build(:imported_pages,+ page: "/test",+ visitors: 2,+ pageviews: 2,+ date: ~D[2021-01-01]+ ),+ build(:imported_visitors, visitors: 5, date: ~D[2021-01-01])+ ])++ filters = Jason.encode!(%{goal: "Purchase"})+ url_query_params = "?filters=#{filters}&period=day&date=2021-01-01&with_imported=true"+ conn = get(conn, "/api/stats/#{site.domain}/conversions#{url_query_params}")++ assert [+ %{+ "name" => "Purchase",+ "visitors" => 5,+ "events" => 7,+ "conversion_rate" => 62.5+ }+ ] = json_response(conn, 200)["results"]+ end++ test "returns custom event goals with more than one option in goal filter", %{+ conn: conn,+ site: site+ } do+ insert(:goal, site: site, event_name: "Purchase")+ insert(:goal, site: site, event_name: "Activation")+ insert(:goal, site: site, page_path: "/test")++ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:pageview,+ timestamp: ~N[2021-01-01 00:00:01],+ pathname: "/test"+ ),+ build(:event,+ name: "Purchase",+ timestamp: ~N[2021-01-01 00:00:03]+ ),+ build(:event,+ name: "Purchase",+ timestamp: ~N[2021-01-01 00:00:03]+ ),+ build(:event,+ name: "Activation",+ timestamp: ~N[2021-01-01 00:00:03]+ ),+ build(:imported_custom_events,+ name: "Purchase",+ visitors: 3,+ events: 5,+ date: ~D[2021-01-01]+ ),+ build(:imported_custom_events,+ name: "Activation",+ visitors: 2,+ events: 4,+ date: ~D[2021-01-01]+ ),+ build(:imported_pages,+ page: "/test",+ visitors: 2,+ pageviews: 2,+ date: ~D[2021-01-01]+ ),+ build(:imported_visitors, visitors: 5, date: ~D[2021-01-01])+ ])++ filters = Jason.encode!(%{goal: "Purchase|Activation"})+ url_query_params = "?filters=#{filters}&period=day&date=2021-01-01&with_imported=true"+ conn = get(conn, "/api/stats/#{site.domain}/conversions#{url_query_params}")++ assert [+ %{+ "name" => "Purchase",+ "visitors" => 5,+ "events" => 7,+ "conversion_rate" => 55.6+ },+ %{+ "name" => "Activation",+ "visitors" => 3,+ "events" => 5,+ "conversion_rate" => 33.3+ }+ ] = json_response(conn, 200)["results"]+ end++ test "returns only pageview goals with a pageview goal filter", %{+ conn: conn,+ site: site+ } do+ insert(:goal, site: site, event_name: "Purchase")+ insert(:goal, site: site, event_name: "Activation")+ insert(:goal, site: site, page_path: "/test")++ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:pageview,+ timestamp: ~N[2021-01-01 00:00:01],+ pathname: "/test"+ ),+ build(:event,+ name: "Purchase",+ timestamp: ~N[2021-01-01 00:00:03]+ ),+ build(:event,+ name: "Purchase",+ timestamp: ~N[2021-01-01 00:00:03]+ ),+ build(:imported_custom_events,+ name: "Purchase",+ visitors: 3,+ events: 5,+ date: ~D[2021-01-01]+ ),+ build(:imported_pages,+ page: "/test",+ visitors: 2,+ pageviews: 2,+ date: ~D[2021-01-01]+ ),+ build(:imported_visitors, visitors: 5, date: ~D[2021-01-01])+ ])++ filters = Jason.encode!(%{goal: "Visit /test"})+ url_query_params = "?filters=#{filters}&period=day&date=2021-01-01&with_imported=true"+ conn = get(conn, "/api/stats/#{site.domain}/conversions#{url_query_params}")++ assert [+ %{+ "name" => "Visit /test",+ "visitors" => 3,+ "events" => 3,+ "conversion_rate" => 37.5+ }+ ] = json_response(conn, 200)["results"]+ end++ test "returns pageview goals with more than one option in pageview goal filter", %{+ conn: conn,+ site: site+ } do+ insert(:goal, site: site, event_name: "Purchase")+ insert(:goal, site: site, event_name: "Activation")+ insert(:goal, site: site, page_path: "/test")+ insert(:goal, site: site, page_path: "/blog")++ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:pageview,+ timestamp: ~N[2021-01-01 00:00:01],+ pathname: "/test"+ ),+ build(:pageview,+ timestamp: ~N[2021-01-01 00:00:01],+ pathname: "/blog"+ ),+ build(:event,+ name: "Purchase",+ timestamp: ~N[2021-01-01 00:00:03]+ ),+ build(:event,+ name: "Purchase",+ timestamp: ~N[2021-01-01 00:00:03]+ ),+ build(:imported_custom_events,+ name: "Purchase",+ visitors: 3,+ events: 5,+ date: ~D[2021-01-01]+ ),+ build(:imported_pages,+ page: "/test",+ visitors: 2,+ pageviews: 2,+ date: ~D[2021-01-01]+ ),+ build(:imported_pages,+ page: "/blog",+ visitors: 1,+ pageviews: 1,+ date: ~D[2021-01-01]+ ),+ build(:imported_visitors, visitors: 5, date: ~D[2021-01-01])+ ])++ filters = Jason.encode!(%{goal: "Visit /test|Visit /blog"})+ url_query_params = "?filters=#{filters}&period=day&date=2021-01-01&with_imported=true"+ conn = get(conn, "/api/stats/#{site.domain}/conversions#{url_query_params}")++ assert [+ %{+ "name" => "Visit /test",+ "visitors" => 3,+ "events" => 3,+ "conversion_rate" => 33.3+ },+ %{+ "name" => "Visit /blog",+ "visitors" => 2,+ "events" => 2,+ "conversion_rate" => 22.2+ }+ ] = json_response(conn, 200)["results"]+ end++ test "returns pageview goals with a page filter", %{+ conn: conn,+ site: site+ } do+ insert(:goal, site: site, page_path: "/blog/two")+ insert(:goal, site: site, page_path: "/blog/thr**")+ insert(:goal, site: site, page_path: "/blog/*")++ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:imported_pages, page: "/", visitors: 1, pageviews: 1, date: ~D[2021-01-01]),+ build(:imported_pages, page: "/blog/one", visitors: 2, pageviews: 2, date: ~D[2021-01-01]),+ build(:imported_pages, page: "/blog/two", visitors: 3, pageviews: 3, date: ~D[2021-01-01]),+ build(:imported_pages,+ page: "/blog/three",+ visitors: 4,+ pageviews: 4,+ date: ~D[2021-01-01]+ ),+ build(:imported_visitors, visitors: 10, date: ~D[2021-01-01])+ ])++ filters = Jason.encode!(%{page: "/blog/one|/blog/two"})+ q = "?filters=#{filters}&period=day&date=2021-01-01&with_imported=true"+ conn = get(conn, "/api/stats/#{site.domain}/conversions#{q}")++ assert [+ %{+ "name" => "Visit /blog/*",+ "visitors" => 5,+ "events" => 5,+ "conversion_rate" => 100.0+ },+ %{+ "name" => "Visit /blog/two",+ "visitors" => 3,+ "events" => 3,+ "conversion_rate" => 60.0+ }+ ] = json_response(conn, 200)["results"]endtest "calculates conversion_rate for goals with glob pattern with imported data", %{@@ -832,7 +1119,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do"/api/stats/#{site.domain}/conversions?period=day")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Visit /blog**","visitors" => 2,diff --git a/test/plausible_web/controllers/api/stats_controller/countries_test.exs b/test/plausible_web/controllers/api/stats_controller/countries_test.exsindex de616c28d1..5875487820 100644--- a/test/plausible_web/controllers/api/stats_controller/countries_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/countries_test.exs@@ -16,7 +16,7 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest doconn = get(conn, "/api/stats/#{site.domain}/countries?period=day")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"code" => "EE","alpha_3" => "EST",@@ -37,7 +37,7 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest doconn = get(conn, "/api/stats/#{site.domain}/countries?period=day&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"code" => "EE","alpha_3" => "EST",@@ -65,7 +65,7 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest doconn = get(conn, "/api/stats/#{site.domain}/countries?period=day&with_imported=true")- assert json_response(conn, 200) == []+ assert json_response(conn, 200)["results"] == []endtest "calculates conversion_rate when filtering for goal", %{conn: conn, site: site} do@@ -90,7 +90,7 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest doconn = get(conn, "/api/stats/#{site.domain}/countries?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"code" => "EE","alpha_3" => "EST",@@ -140,7 +140,7 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest dofilters = Jason.encode!(%{props: %{"author" => "John Doe"}})conn = get(conn, "/api/stats/#{site.domain}/countries?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"code" => "EE","alpha_3" => "EST",@@ -182,7 +182,7 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest dofilters = Jason.encode!(%{props: %{"author" => "!John Doe"}})conn = get(conn, "/api/stats/#{site.domain}/countries?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"code" => "GB","alpha_3" => "GBR",@@ -217,7 +217,7 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest dofilters = Jason.encode!(%{props: %{"author" => "(none)"}})conn = get(conn, "/api/stats/#{site.domain}/countries?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"code" => "GB","alpha_3" => "GBR",@@ -257,7 +257,7 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest dofilters = Jason.encode!(%{props: %{"author" => "!(none)"}})conn = get(conn, "/api/stats/#{site.domain}/countries?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"code" => "EE","alpha_3" => "EST",@@ -279,7 +279,7 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest dofilters = Jason.encode!(%{country: "GB"})conn = get(conn, "/api/stats/#{site.domain}/countries?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"code" => "GB","alpha_3" => "GBR",diff --git a/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs b/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exsindex 9c1898a02c..d916347897 100644--- a/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs@@ -21,7 +21,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 2,"name" => "K2sna Kalle",@@ -57,7 +57,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 1,"name" => "K2sna Kalle",@@ -65,6 +65,8 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"percentage" => 100.0}]++ refute json_response(conn, 200)["warning"]endtest "returns (none) values in the breakdown", %{conn: conn, site: site} do@@ -82,7 +84,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 2,"name" => "K2sna Kalle",@@ -122,7 +124,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&limit=2&page=2")- assert json_response(conn1, 200) == [+ assert json_response(conn1, 200)["results"] == [%{"visitors" => 3,"name" => "Tiit",@@ -137,7 +139,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do}]- assert json_response(conn2, 200) == [+ assert json_response(conn2, 200)["results"] == [%{"visitors" => 1,"name" => "(none)",@@ -171,7 +173,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 2,"name" => "B",@@ -207,7 +209,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 2,"name" => "(none)",@@ -250,7 +252,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/cost?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "0","visitors" => 1,@@ -287,7 +289,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/cost?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "(none)","visitors" => 1,@@ -334,7 +336,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/cost?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "20","visitors" => 2,@@ -377,7 +379,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/cost?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "0","visitors" => 1,@@ -424,7 +426,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/cost?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "1","visitors" => 2,@@ -475,7 +477,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/cost?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "1","visitors" => 2,@@ -533,7 +535,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/cost?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "20","visitors" => 2,@@ -584,7 +586,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/cost?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "20","visitors" => 2,@@ -611,7 +613,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/variant?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 2,"name" => "A",@@ -645,7 +647,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 1,"name" => "B",@@ -690,7 +692,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "A","visitors" => 1,@@ -735,7 +737,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "A","visitors" => 1,@@ -783,7 +785,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 2,"name" => "true",@@ -844,7 +846,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 2,"name" => "true",@@ -889,6 +891,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest doreturned_metrics =json_response(conn, 200)+ |> Map.get("results")|> List.first()|> Map.keys()@@ -916,7 +919,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 1,"name" => "Sipsik",@@ -946,7 +949,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 1,"name" => "Sipsik",@@ -973,7 +976,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 1,"name" => "Sipsik",@@ -1004,7 +1007,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/#{prop_key}?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 2,"name" => "K2sna Kalle",@@ -1040,7 +1043,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/key?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 2,"name" => "bar",@@ -1078,7 +1081,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/key?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 1,"name" => "bar",@@ -1116,11 +1119,13 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest doconn|> get("/api/stats/#{site.domain}/custom-prop-values/url?period=day")|> json_response(200)+ |> Map.get("results")[%{"visitors" => 1, "name" => "two"}] =conn|> get("/api/stats/#{site.domain}/custom-prop-values/path?period=day")|> json_response(200)+ |> Map.get("results")endtest "returns 402 'upgrade required' for any other prop key", %{conn: conn, site: site} do@@ -1136,7 +1141,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest dodescribe "with imported data" dosetup [:create_user, :log_in, :create_new_site]- for goal_name <- ["Outbound Link: Click", "File Download"] do+ for goal_name <- ["Outbound Link: Click", "File Download", "Cloaked Link: Click"] dotest "returns url breakdown for #{goal_name} goal", %{conn: conn, site: site} doinsert(:goal, event_name: unquote(goal_name), site: site)site_import = insert(:site_import, site: site)@@ -1175,7 +1180,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do"/api/stats/#{site.domain}/custom-prop-values/url?period=day&with_imported=true&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 5,"name" => "https://two.com",@@ -1191,5 +1196,58 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do]endend++ for goal_name <- ["Outbound Link: Click", "File Download", "Cloaked Link: Click"] do+ test "returns url breakdown for #{goal_name} goal with a url filter", %{+ conn: conn,+ site: site+ } do+ insert(:goal, event_name: unquote(goal_name), site: site)+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:event,+ name: unquote(goal_name),+ "meta.key": ["url"],+ "meta.value": ["https://one.com"]+ ),+ build(:imported_custom_events,+ name: unquote(goal_name),+ visitors: 2,+ events: 5,+ link_url: "https://one.com"+ ),+ build(:imported_custom_events,+ name: unquote(goal_name),+ visitors: 5,+ events: 10,+ link_url: "https://two.com"+ ),+ build(:imported_custom_events,+ name: "view_search_results",+ visitors: 100,+ events: 200+ ),+ build(:imported_visitors, visitors: 9)+ ])++ filters = Jason.encode!(%{goal: unquote(goal_name), props: %{url: "https://two.com"}})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/custom-prop-values/url?period=day&with_imported=true&filters=#{filters}"+ )++ assert json_response(conn, 200)["results"] == [+ %{+ "visitors" => 5,+ "name" => "https://two.com",+ "events" => 10,+ "conversion_rate" => 50.0+ }+ ]+ end+ endendenddiff --git a/test/plausible_web/controllers/api/stats_controller/imported_test.exs b/test/plausible_web/controllers/api/stats_controller/imported_test.exsindex 42eafec869..370eb0ecf6 100644--- a/test/plausible_web/controllers/api/stats_controller/imported_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/imported_test.exs@@ -261,13 +261,16 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do"imported_sources")- conn =- get(- conn,+ results =+ conn+ |> get("/api/stats/#{site.domain}/sources?period=month&date=2021-01-01&with_imported=true")+ |> json_response(200)+ |> Map.get("results")+ |> Enum.sort()- assert conn |> json_response(200) |> Enum.sort() == [+ assert results == [%{"name" => "A Nice Newsletter", "visitors" => 1},%{"name" => "Direct / None", "visitors" => 1},%{"name" => "DuckDuckGo", "visitors" => 2},@@ -338,7 +341,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do"/api/stats/#{site.domain}/utm_mediums?period=day&date=2021-01-01&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"bounce_rate" => 100.0,"name" => "social",@@ -420,7 +423,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do"/api/stats/#{site.domain}/utm_campaigns?period=day&date=2021-01-01&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "august","visitors" => 2,@@ -509,7 +512,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do"/api/stats/#{site.domain}/utm_terms?period=day&date=2021-01-01&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Sweden","visitors" => 3,@@ -597,7 +600,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do"/api/stats/#{site.domain}/utm_contents?period=day&date=2021-01-01&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "blog","visitors" => 2,@@ -703,7 +706,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do"/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"bounce_rate" => nil,"time_on_page" => 60,@@ -786,7 +789,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do"/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "/page2","visitors" => 3,@@ -859,7 +862,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do"/api/stats/#{site.domain}/cities?period=day&date=2021-01-01&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"code" => 588_335, "name" => "Tartu", "visitors" => 1, "country_flag" => "🇪🇪"},%{"code" => 2_650_225,@@ -933,7 +936,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do"/api/stats/#{site.domain}/countries?period=day&date=2021-01-01&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"code" => "EE","alpha_3" => "EST",@@ -997,7 +1000,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do"/api/stats/#{site.domain}/screen-sizes?period=day&date=2021-01-01&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Desktop", "visitors" => 2, "percentage" => 40},%{"name" => "Laptop", "visitors" => 2, "percentage" => 40},%{"name" => "Mobile", "visitors" => 1, "percentage" => 20}@@ -1050,7 +1053,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do"/api/stats/#{site.domain}/browsers?period=day&date=2021-01-01&with_imported=true")- assert stats = json_response(conn, 200)+ assert stats = json_response(conn, 200)["results"]assert length(stats) == 3assert %{"name" => "Firefox", "visitors" => 2, "percentage" => 50.0} in statsassert %{"name" => "Mobile App", "visitors" => 1, "percentage" => 25.0} in stats@@ -1104,7 +1107,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do"/api/stats/#{site.domain}/operating-systems?period=day&date=2021-01-01&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Mac", "visitors" => 3, "percentage" => 60},%{"name" => "GNU/Linux", "visitors" => 2, "percentage" => 40}]diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exsindex b88fae0fa5..773bc8cbf5 100644--- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs@@ -53,7 +53,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do)zeroes = List.duplicate(0, 30)- assert %{"plot" => ^zeroes, "with_imported" => false} = json_response(conn, 200)+ assert %{"plot" => ^zeroes, "includes_imported" => false} = json_response(conn, 200)endtest "displays visitors for a day with imported data", %{conn: conn, site: site} do@@ -70,7 +70,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do"/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&with_imported=true")- assert %{"plot" => plot, "imports_exist" => true, "with_imported" => true} =+ assert %{"plot" => plot, "imports_exist" => true, "includes_imported" => true} =json_response(conn, 200)assert plot == [2] ++ List.duplicate(0, 23)@@ -137,7 +137,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do"/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true")- assert %{"plot" => plot, "imports_exist" => true, "with_imported" => true} =+ assert %{"plot" => plot, "imports_exist" => true, "includes_imported" => true} =json_response(conn, 200)assert Enum.count(plot) == 31@@ -158,7 +158,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do"/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true")- assert %{"plot" => plot, "imports_exist" => true, "with_imported" => true} =+ assert %{"plot" => plot, "imports_exist" => true, "includes_imported" => true} =json_response(conn, 200)assert Enum.count(plot) == 31@@ -1157,7 +1157,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do"plot" => plot,"comparison_plot" => comparison_plot,"imports_exist" => true,- "with_imported" => true+ "includes_imported" => true} = json_response(conn, 200)assert 4 == Enum.sum(plot)@@ -1203,7 +1203,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do"plot" => plot,"comparison_plot" => comparison_plot,"imports_exist" => true,- "with_imported" => false+ "includes_imported" => false} = json_response(conn, 200)assert 4 == Enum.sum(plot)@@ -1233,7 +1233,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do"plot" => this_week_plot,"comparison_plot" => last_week_plot,"imports_exist" => true,- "with_imported" => false+ "includes_imported" => false} = json_response(conn, 200)assert this_week_plot == [50.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]diff --git a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exsindex d383fca24b..ea9f8990a2 100644--- a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs@@ -13,7 +13,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest doconn = get(conn, "/api/stats/#{site.domain}/operating-systems?period=day")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Mac", "visitors" => 2, "percentage" => 66.7},%{"name" => "Android", "visitors" => 1, "percentage" => 33.3}]@@ -31,7 +31,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest doconn = get(conn, "/api/stats/#{site.domain}/operating-systems?period=day")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "(not set)", "visitors" => 1, "percentage" => 50},%{"name" => "Linux", "visitors" => 1, "percentage" => 50}]@@ -41,7 +41,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest doconn =get(conn, "/api/stats/#{site.domain}/operating-systems?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "(not set)", "visitors" => 1, "percentage" => 100}]end@@ -57,7 +57,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest doconn =get(conn, "/api/stats/#{site.domain}/operating-systems?period=day&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "(not set)", "visitors" => 2, "percentage" => 100.0}]end@@ -74,7 +74,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest doconn =get(conn, "/api/stats/#{site.domain}/operating-systems?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Mac","total_visitors" => 2,@@ -114,7 +114,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest doconn =get(conn, "/api/stats/#{site.domain}/operating-systems?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Mac", "visitors" => 1, "percentage" => 100}]end@@ -151,7 +151,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest doconn =get(conn, "/api/stats/#{site.domain}/operating-systems?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Android", "visitors" => 1, "percentage" => 50},%{"name" => "Mac", "visitors" => 1, "percentage" => 50}]@@ -172,7 +172,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest doconn = get(conn, "/api/stats/#{site.domain}/operating-systems?period=day")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Mac", "visitors" => 2, "percentage" => 66.7},%{"name" => "Android", "visitors" => 1, "percentage" => 33.3}]@@ -180,7 +180,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest doconn =get(conn, "/api/stats/#{site.domain}/operating-systems?period=day&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Mac", "visitors" => 3, "percentage" => 60},%{"name" => "Android", "visitors" => 2, "percentage" => 40}]@@ -199,7 +199,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest doconn =get(conn, "/api/stats/#{site.domain}/operating-systems?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Mac","total_visitors" => 2,@@ -241,7 +241,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do"/api/stats/#{site.domain}/operating-system-versions?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "10.16", "visitors" => 2, "percentage" => 66.7, "os" => "Mac"},%{"name" => "10.15", "visitors" => 1, "percentage" => 33.3, "os" => "Mac"}]@@ -281,7 +281,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do"/api/stats/#{site.domain}/operating-system-versions?period=day&date=2021-01-01&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"os" => "(not set)","name" => "(not set)",diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exsindex ea9ccfe6ae..4af5c5cd0b 100644--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs@@ -18,7 +18,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest doconn = get(conn, "/api/stats/#{site.domain}/pages?period=day")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 3, "name" => "/"},%{"visitors" => 2, "name" => "/register"},%{"visitors" => 1, "name" => "/contact"}@@ -40,7 +40,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest dofilters = Jason.encode!(%{"hostname" => "*.example.com"})conn = get(conn1, "/api/stats/#{site.domain}/pages?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 3, "name" => "/"},%{"visitors" => 2, "name" => "/register"},%{"visitors" => 1, "name" => "/contact"},@@ -50,7 +50,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest dofilters = Jason.encode!(%{"hostname" => "d.example.com"})conn = get(conn1, "/api/stats/#{site.domain}/pages?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 2, "name" => "/register"},%{"visitors" => 1, "name" => "/"}]@@ -74,7 +74,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest dofilters = Jason.encode!(%{props: %{"author" => "John Doe"}})conn = get(conn, "/api/stats/#{site.domain}/pages?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 1, "name" => "/blog/john-1"}]end@@ -100,7 +100,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest dofilters = Jason.encode!(%{props: %{"author" => "!John Doe"}})conn = get(conn, "/api/stats/#{site.domain}/pages?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 1, "name" => "/"},%{"visitors" => 1, "name" => "/blog/other-post"}]@@ -137,7 +137,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest dofilters = Jason.encode!(%{props: %{"prop" => "~bar"}})conn = get(conn, "/api/stats/#{site.domain}/pages?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 1, "name" => "/1"},%{"visitors" => 1, "name" => "/2"}]@@ -179,7 +179,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest dofilters = Jason.encode!(%{props: %{"prop" => "~bar|nea"}})conn = get(conn, "/api/stats/#{site.domain}/pages?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 1, "name" => "/1"},%{"visitors" => 1, "name" => "/2"},%{"visitors" => 1, "name" => "/6"}@@ -217,7 +217,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest dofilters = Jason.encode!(%{props: %{"prop" => "bar", "number" => "1"}})conn = get(conn, "/api/stats/#{site.domain}/pages?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 1, "name" => "/1"}]end@@ -266,7 +266,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "/blog/john-2","visitors" => 2,@@ -328,7 +328,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "/blog","visitors" => 2,@@ -380,7 +380,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "/blog","visitors" => 2,@@ -436,7 +436,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "/blog/other-post","visitors" => 2,@@ -494,7 +494,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest doconn =get(conn, "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "/firefox","visitors" => 2@@ -534,7 +534,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest doconn =get(conn, "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "/safari","visitors" => 1@@ -578,7 +578,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "/","visitors" => 2,@@ -625,7 +625,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "/","visitors" => 2,@@ -679,7 +679,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "/","visitors" => 2,@@ -725,7 +725,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "/articles/post-1","visitors" => 2,@@ -777,7 +777,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "/blog/(/post-1","visitors" => 1,@@ -830,7 +830,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "/","visitors" => 2,@@ -862,7 +862,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest doconn = get(conn, "/api/stats/#{site.domain}/pages?period=day")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 3, "name" => "/"},%{"visitors" => 2, "name" => "/register"},%{"visitors" => 1, "name" => "/contact"}@@ -870,13 +870,43 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest doconn = get(conn, "/api/stats/#{site.domain}/pages?period=day&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 4, "name" => "/"},%{"visitors" => 3, "name" => "/register"},%{"visitors" => 1, "name" => "/contact"}]end+ test "returns imported pages with a pageview goal filter", %{conn: conn, site: site} do+ insert(:goal, site: site, page_path: "/blog**")++ populate_stats(site, [+ build(:imported_pages, page: "/blog"),+ build(:imported_pages, page: "/not-this"),+ build(:imported_pages, page: "/blog/post-1", visitors: 2),+ build(:imported_visitors, visitors: 4)+ ])++ filters = Jason.encode!(%{goal: "Visit /blog**"})+ q = "?period=day&filters=#{filters}&with_imported=true"+ conn = get(conn, "/api/stats/#{site.domain}/pages#{q}")++ assert json_response(conn, 200)["results"] == [+ %{+ "visitors" => 2,+ "name" => "/blog/post-1",+ "conversion_rate" => 100.0,+ "total_visitors" => 2+ },+ %{+ "visitors" => 1,+ "name" => "/blog",+ "conversion_rate" => 100.0,+ "total_visitors" => 1+ }+ ]+ end+test "calculates bounce rate and time on page for pages", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview,@@ -901,7 +931,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"bounce_rate" => 50.0,"time_on_page" => 900.0,@@ -948,7 +978,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"bounce_rate" => 50,"name" => "/about",@@ -1027,7 +1057,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"bounce_rate" => 50,"name" => "/about-blog",@@ -1055,6 +1085,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest doconn|> get("/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true")|> json_response(200)+ |> Map.get("results")endtest "ignores page refresh when calculating time on page", %{conn: conn, site: site} do@@ -1072,6 +1103,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest doconn|> get("/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true")|> json_response(200)+ |> Map.get("results")endtest "calculates time on page per unique transition within session", %{conn: conn, site: site} do@@ -1105,6 +1137,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest doconn|> get("/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true")|> json_response(200)+ |> Map.get("results")endtest "calculates bounce rate and time on page for pages with imported data", %{@@ -1150,7 +1183,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"bounce_rate" => 40.0,"time_on_page" => 800.0,@@ -1177,7 +1210,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest doconn = get(conn, "/api/stats/#{site.domain}/pages?period=realtime")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 2, "name" => "/page1"},%{"visitors" => 1, "name" => "/page2"}]@@ -1195,10 +1228,160 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest doconn = get(conn, "/api/stats/#{site.domain}/pages?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"total_visitors" => 3, "visitors" => 1, "name" => "/", "conversion_rate" => 33.3}]end++ test "filter by :is page with imported data", %{conn: conn, site: site} do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:pageview, user_id: 1, pathname: "/", timestamp: ~N[2021-01-01 12:00:00]),+ build(:pageview, user_id: 1, pathname: "/ignored", timestamp: ~N[2021-01-01 12:01:00]),+ build(:imported_entry_pages,+ entry_page: "/",+ visitors: 1,+ bounces: 1,+ date: ~D[2021-01-01]+ ),+ build(:imported_pages,+ page: "/",+ visitors: 3,+ pageviews: 3,+ time_on_page: 300,+ date: ~D[2021-01-01]+ ),+ build(:imported_pages, page: "/ignored", visitors: 10, date: ~D[2021-01-01])+ ])++ filters = Jason.encode!(%{"page" => "/"})+ q = "?period=day&date=2021-01-01&filters=#{filters}&detailed=true&with_imported=true"++ conn = get(conn, "/api/stats/#{site.domain}/pages#{q}")++ assert json_response(conn, 200)["results"] == [+ %{+ "bounce_rate" => 50,+ "name" => "/",+ "pageviews" => 4,+ "time_on_page" => 90.0,+ "visitors" => 4+ }+ ]+ end++ test "filter by :member page with imported data", %{conn: conn, site: site} do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:pageview, user_id: 1, pathname: "/", timestamp: ~N[2021-01-01 12:00:00]),+ build(:pageview, user_id: 1, pathname: "/ignored", timestamp: ~N[2021-01-01 12:01:00]),+ build(:imported_entry_pages,+ entry_page: "/",+ visitors: 1,+ bounces: 1,+ date: ~D[2021-01-01]+ ),+ build(:imported_entry_pages,+ entry_page: "/a",+ visitors: 1,+ bounces: 1,+ date: ~D[2021-01-01]+ ),+ build(:imported_pages,+ page: "/",+ visitors: 3,+ pageviews: 3,+ time_on_page: 300,+ date: ~D[2021-01-01]+ ),+ build(:imported_pages,+ page: "/a",+ visitors: 1,+ date: ~D[2021-01-01]+ ),+ build(:imported_pages, page: "/ignored", visitors: 10, date: ~D[2021-01-01])+ ])++ filters = Jason.encode!(%{"page" => "/|/a"})+ q = "?period=day&date=2021-01-01&filters=#{filters}&detailed=true&with_imported=true"++ conn = get(conn, "/api/stats/#{site.domain}/pages#{q}")++ assert json_response(conn, 200)["results"] == [+ %{+ "bounce_rate" => 50,+ "name" => "/",+ "pageviews" => 4,+ "time_on_page" => 90.0,+ "visitors" => 4+ },+ %{+ "bounce_rate" => 100,+ "name" => "/a",+ "pageviews" => 1,+ "time_on_page" => 10.0,+ "visitors" => 1+ }+ ]+ end++ test "filter by :matches page with imported data", %{conn: conn, site: site} do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:pageview, user_id: 1, pathname: "/aaa", timestamp: ~N[2021-01-01 12:00:00]),+ build(:pageview, user_id: 1, pathname: "/ignored", timestamp: ~N[2021-01-01 12:01:00]),+ build(:imported_entry_pages,+ entry_page: "/aaa",+ visitors: 1,+ bounces: 1,+ date: ~D[2021-01-01]+ ),+ build(:imported_entry_pages,+ entry_page: "/a",+ visitors: 1,+ bounces: 1,+ date: ~D[2021-01-01]+ ),+ build(:imported_pages,+ page: "/aaa",+ visitors: 3,+ pageviews: 3,+ time_on_page: 300,+ date: ~D[2021-01-01]+ ),+ build(:imported_pages,+ page: "/a",+ visitors: 1,+ date: ~D[2021-01-01]+ ),+ build(:imported_pages, page: "/ignored", visitors: 10, date: ~D[2021-01-01])+ ])++ filters = Jason.encode!(%{"page" => "/a**"})+ q = "?period=day&date=2021-01-01&filters=#{filters}&detailed=true&with_imported=true"++ conn = get(conn, "/api/stats/#{site.domain}/pages#{q}")++ assert json_response(conn, 200)["results"] == [+ %{+ "bounce_rate" => 50,+ "name" => "/aaa",+ "pageviews" => 4,+ "time_on_page" => 90.0,+ "visitors" => 4+ },+ %{+ "bounce_rate" => 100,+ "name" => "/a",+ "pageviews" => 1,+ "time_on_page" => 10.0,+ "visitors" => 1+ }+ ]+ endenddescribe "GET /api/stats/:domain/entry-pages" do@@ -1236,7 +1419,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest doconn = get(conn, "/api/stats/#{site.domain}/entry-pages?period=day&date=2021-01-01")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 2,"visits" => 2,@@ -1292,7 +1475,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/entry-pages?period=day&date=2021-01-01&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 1,"visits" => 1,@@ -1347,7 +1530,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest doconn = get(conn, "/api/stats/#{site.domain}/entry-pages?period=day&date=2021-01-01")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 2,"visits" => 2,@@ -1368,7 +1551,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/entry-pages?period=day&date=2021-01-01&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"visitors" => 3,"visits" => 5,@@ -1431,7 +1614,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do)# We're going to only join sessions where the exit hostname matches the filter- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "/page1", "visit_duration" => 0, "visitors" => 1, "visits" => 1},%{"name" => "/page2", "visit_duration" => 0, "visitors" => 1, "visits" => 1}]@@ -1460,6 +1643,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/pages?date=2021-01-01&period=day&filters=#{filters}&limit=#{limit}&page=#{page}")|> json_response(200)+ |> Map.get("results")|> Enum.map(fn %{"name" => "/signup/" <> seq} ->seqend)@@ -1519,7 +1703,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/entry-pages?period=day&date=2021-01-01&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"total_visitors" => 2,"visitors" => 1,@@ -1550,7 +1734,57 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/entry-pages?period=day&date=2021-01-01")- assert json_response(conn, 200) == []+ assert json_response(conn, 200)["results"] == []+ end++ test "filter by :matches_member entry_page with imported data", %{conn: conn, site: site} do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:pageview, pathname: "/aaa", timestamp: ~N[2021-01-01 12:00:00]),+ build(:pageview, pathname: "/a", timestamp: ~N[2021-01-01 12:00:00]),+ build(:pageview, pathname: "/ignored", timestamp: ~N[2021-01-01 12:01:00]),+ build(:imported_entry_pages,+ entry_page: "/a",+ visitors: 5,+ entrances: 9,+ visit_duration: 1000,+ date: ~D[2021-01-01]+ ),+ build(:imported_entry_pages,+ entry_page: "/bbb",+ visitors: 2,+ entrances: 2,+ visit_duration: 100,+ date: ~D[2021-01-01]+ )+ ])++ filters = Jason.encode!(%{"entry_page" => "/a**|/b**"})+ q = "?period=day&date=2021-01-01&filters=#{filters}&detailed=true&with_imported=true"++ conn = get(conn, "/api/stats/#{site.domain}/entry-pages#{q}")++ assert json_response(conn, 200)["results"] == [+ %{+ "visit_duration" => 100.0,+ "name" => "/a",+ "visits" => 10,+ "visitors" => 6+ },+ %{+ "visit_duration" => 50.0,+ "name" => "/bbb",+ "visits" => 2,+ "visitors" => 2+ },+ %{+ "visit_duration" => 0,+ "name" => "/aaa",+ "visits" => 1,+ "visitors" => 1+ }+ ]endend@@ -1581,7 +1815,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest doconn = get(conn, "/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "/page1", "visitors" => 2, "visits" => 2, "exit_rate" => 66},%{"name" => "/page2", "visitors" => 1, "visits" => 1, "exit_rate" => 100}]@@ -1629,7 +1863,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do)# We're going to only join sessions where the entry hostname matches the filter- assert json_response(conn, 200) ==+ assert json_response(conn, 200)["results"] ==[%{"name" => "/page1", "visitors" => 1, "visits" => 1}]end@@ -1667,7 +1901,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "/", "visitors" => 1, "visits" => 1}]end@@ -1711,7 +1945,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest doconn = get(conn, "/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "/page1", "visitors" => 2, "visits" => 2, "exit_rate" => 66},%{"name" => "/page2", "visitors" => 1, "visits" => 1, "exit_rate" => 100}]@@ -1722,7 +1956,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "/page2","visitors" => 3,@@ -1773,7 +2007,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "/exit1","visitors" => 1,@@ -1826,7 +2060,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "/exit1", "visitors" => 1, "visits" => 1},%{"name" => "/exit2", "visitors" => 1, "visits" => 1}]@@ -1847,7 +2081,59 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01")- assert json_response(conn, 200) == []+ assert json_response(conn, 200)["results"] == []+ end++ test "filter by :is_not exit_page with imported data", %{conn: conn, site: site} do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:pageview, pathname: "/aaa", timestamp: ~N[2021-01-01 12:00:00]),+ build(:pageview, pathname: "/a", timestamp: ~N[2021-01-01 12:00:00]),+ build(:pageview, pathname: "/ignored", timestamp: ~N[2021-01-01 12:01:00]),+ build(:imported_exit_pages,+ exit_page: "/a",+ visitors: 5,+ exits: 9,+ visit_duration: 1000,+ date: ~D[2021-01-01]+ ),+ build(:imported_exit_pages,+ exit_page: "/bbb",+ visitors: 2,+ exits: 2,+ visit_duration: 100,+ date: ~D[2021-01-01]+ ),+ build(:imported_pages, page: "/a", pageviews: 19, date: ~D[2021-01-01]),+ build(:imported_pages, page: "/bbb", pageviews: 2, date: ~D[2021-01-01])+ ])++ filters = Jason.encode!(%{"exit_page" => "!/ignored"})+ q = "?period=day&date=2021-01-01&filters=#{filters}&detailed=true&with_imported=true"++ conn = get(conn, "/api/stats/#{site.domain}/exit-pages#{q}")++ assert json_response(conn, 200)["results"] == [+ %{+ "exit_rate" => 50.0,+ "name" => "/a",+ "visits" => 10,+ "visitors" => 6+ },+ %{+ "exit_rate" => 100.0,+ "name" => "/bbb",+ "visits" => 2,+ "visitors" => 2+ },+ %{+ "exit_rate" => 100.0,+ "name" => "/aaa",+ "visits" => 1,+ "visitors" => 1+ }+ ]endendenddiff --git a/test/plausible_web/controllers/api/stats_controller/regions_test.exs b/test/plausible_web/controllers/api/stats_controller/regions_test.exsindex 23ac063bab..711ae957f8 100644--- a/test/plausible_web/controllers/api/stats_controller/regions_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/regions_test.exs@@ -37,7 +37,7 @@ defmodule PlausibleWeb.Api.StatsController.RegionsTest dotest "returns top cities by new visitors", %{conn: conn, site: site} doconn = get(conn, "/api/stats/#{site.domain}/regions?period=day")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"code" => "EE-37", "country_flag" => "🇪🇪", "name" => "Harjumaa", "visitors" => 3},%{"code" => "EE-39", "country_flag" => "🇪🇪", "name" => "Hiiumaa", "visitors" => 2}]@@ -47,7 +47,7 @@ defmodule PlausibleWeb.Api.StatsController.RegionsTest dofilters = Jason.encode!(%{region: "EE-39"})conn = get(conn, "/api/stats/#{site.domain}/regions?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"code" => "EE-39", "country_flag" => "🇪🇪", "name" => "Hiiumaa", "visitors" => 2}]enddiff --git a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exsindex 128d1293df..49c94c6e7d 100644--- a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs@@ -13,7 +13,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest doconn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Desktop", "visitors" => 2, "percentage" => 66.7},%{"name" => "Laptop", "visitors" => 1, "percentage" => 33.3}]@@ -39,7 +39,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do"date" => "2021-01-01"})- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Desktop", "visitors" => 1, "percentage" => 100},%{"name" => "Laptop", "visitors" => 1, "percentage" => 100}]@@ -57,7 +57,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest doconn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "(not set)", "visitors" => 1, "percentage" => 50},%{"name" => "Desktop", "visitors" => 1, "percentage" => 50}]@@ -67,7 +67,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest dofilters = Jason.encode!(%{screen: "(not set)"})conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "(not set)", "visitors" => 1, "percentage" => 100}]end@@ -84,7 +84,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest doconn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "(not set)", "visitors" => 2, "percentage" => 100.0}]end@@ -117,7 +117,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest dofilters = Jason.encode!(%{props: %{"author" => "John Doe"}})conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Desktop", "visitors" => 1, "percentage" => 100}]end@@ -152,7 +152,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest dofilters = Jason.encode!(%{props: %{"author" => "!John Doe"}})conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Mobile", "visitors" => 1, "percentage" => 50},%{"name" => "Tablet", "visitors" => 1, "percentage" => 50}]@@ -173,20 +173,41 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest doconn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Desktop", "visitors" => 2, "percentage" => 66.7},%{"name" => "Laptop", "visitors" => 1, "percentage" => 33.3}]conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Desktop", "visitors" => 2, "percentage" => 40},%{"name" => "Laptop", "visitors" => 2, "percentage" => 40},%{"name" => "Mobile", "visitors" => 1, "percentage" => 20}]end+ test "returns screen sizes when filtering by imported screen size", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, screen_size: "Desktop"),+ build(:imported_devices, device: "Desktop"),+ build(:imported_devices, device: "Laptop"),+ build(:imported_visitors, visitors: 2)+ ])++ filters = Jason.encode!(%{screen: "Desktop"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/screen-sizes?filters=#{filters}&period=day&with_imported=true"+ )++ assert json_response(conn, 200)["results"] == [+ %{"name" => "Desktop", "visitors" => 2, "percentage" => 100.0}+ ]+ end+test "returns screen sizes for user making multiple sessions by no of visitors with imported data",%{conn: conn, site: site} dopopulate_stats(site, [@@ -215,7 +236,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do"with_imported" => "true"})- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Desktop", "visitors" => 2, "percentage" => 100},%{"name" => "Laptop", "visitors" => 2, "percentage" => 100}]@@ -232,7 +253,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest doconn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Desktop","total_visitors" => 2,@@ -258,7 +279,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest doconn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Desktop", "visitors" => 2, "percentage" => 66.7},%{"name" => "Mobile", "visitors" => 1, "percentage" => 33.3}]diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exsindex dfa9480c02..a76b1da274 100644--- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs@@ -33,7 +33,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest doconn = get(conn, "/api/stats/#{site.domain}/sources")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Google", "visitors" => 3},%{"name" => "DuckDuckGo", "visitors" => 2},%{"name" => "Direct / None", "visitors" => 1}@@ -83,7 +83,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Google", "visitors" => 2},%{"name" => "DuckDuckGo", "visitors" => 1}]@@ -137,7 +137,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Google", "visitors" => 2},%{"name" => "DuckDuckGo", "visitors" => 1}]@@ -187,7 +187,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Facebook", "visitors" => 2},%{"name" => "DuckDuckGo", "visitors" => 1}]@@ -241,7 +241,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Google", "visitors" => 2},%{"name" => "DuckDuckGo", "visitors" => 1}]@@ -270,14 +270,14 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest doconn = get(conn, "/api/stats/#{site.domain}/sources")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Google", "visitors" => 2},%{"name" => "DuckDuckGo", "visitors" => 1}]conn = get(conn, "/api/stats/#{site.domain}/sources?with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Google", "visitors" => 4},%{"name" => "DuckDuckGo", "visitors" => 2}]@@ -310,7 +310,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&detailed=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "DuckDuckGo","visitors" => 1,@@ -375,7 +375,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&detailed=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "DuckDuckGo","visitors" => 1,@@ -396,7 +396,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&detailed=true&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Google","visitors" => 3,@@ -433,7 +433,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest doconn = get(conn, "/api/stats/#{site.domain}/sources?period=realtime")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Google", "visitors" => 2},%{"name" => "DuckDuckGo", "visitors" => 1}]@@ -460,13 +460,13 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest doconn = get(conn, "/api/stats/#{site.domain}/sources?limit=1&page=2")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "DuckDuckGo", "visitors" => 1}]conn = get(conn, "/api/stats/#{site.domain}/sources?limit=1&page=2&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "DuckDuckGo", "visitors" => 2}]end@@ -490,7 +490,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest dofilters = Jason.encode!(%{"page" => "/page1"})conn = get(conn, "/api/stats/#{site.domain}/sources?filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Google", "visitors" => 2},%{"name" => "DuckDuckGo", "visitors" => 1}]@@ -538,7 +538,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do)# nobody landed on one.example.com from utm_param=ad- assert json_response(conn, 200) == []+ assert json_response(conn, 200)["results"] == []endendend@@ -589,7 +589,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/utm_mediums?period=day&date=2021-01-01")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "social","visitors" => 1,@@ -610,7 +610,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/utm_mediums?period=day&date=2021-01-01&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "social","visitors" => 2,@@ -669,7 +669,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/utm_mediums?period=day&date=2021-01-01")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "social","visitors" => 1,@@ -684,7 +684,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/utm_mediums?period=day&date=2021-01-01&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "social","visitors" => 2,@@ -745,7 +745,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/utm_campaigns?period=day&date=2021-01-01")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "august","visitors" => 2,@@ -766,7 +766,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/utm_campaigns?period=day&date=2021-01-01&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "august","visitors" => 3,@@ -829,7 +829,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/utm_campaigns?period=day&date=2021-01-01")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "profile","visitors" => 1,@@ -844,7 +844,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/utm_campaigns?period=day&date=2021-01-01&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "profile","visitors" => 2,@@ -886,7 +886,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/utm_sources?period=day&date=2021-01-01")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "newsletter","visitors" => 2,@@ -953,7 +953,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/utm_terms?period=day&date=2021-01-01")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Sweden","visitors" => 2,@@ -974,7 +974,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/utm_terms?period=day&date=2021-01-01&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Sweden","visitors" => 3,@@ -1037,7 +1037,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/utm_terms?period=day&date=2021-01-01")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "oat milk","visitors" => 1,@@ -1052,7 +1052,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/utm_terms?period=day&date=2021-01-01&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "oat milk","visitors" => 2,@@ -1113,7 +1113,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/utm_contents?period=day&date=2021-01-01")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "blog","visitors" => 2,@@ -1134,7 +1134,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/utm_contents?period=day&date=2021-01-01&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "blog","visitors" => 3,@@ -1197,7 +1197,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/utm_contents?period=day&date=2021-01-01")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "ad","visitors" => 1,@@ -1212,7 +1212,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/utm_contents?period=day&date=2021-01-01&with_imported=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "ad","visitors" => 2,@@ -1257,7 +1257,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/sources?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Twitter","total_visitors" => 2,@@ -1299,7 +1299,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/sources?period=day&filters=#{filters}")- assert json_response(conn, 200) == []+ assert json_response(conn, 200)["results"] == []endtest "returns top referrers for a custom goal and filtered by hostname (2)",@@ -1330,7 +1330,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/sources?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"conversion_rate" => 100.0,"name" => "Facebook",@@ -1380,7 +1380,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "DuckDuckGo","visitors" => 1,@@ -1431,7 +1431,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "DuckDuckGo","visitors" => 1,@@ -1473,7 +1473,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/sources?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "Twitter","total_visitors" => 2,@@ -1513,7 +1513,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/referrers/10words?period=day")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "10words.com", "visitors" => 2},%{"name" => "10words.com/page1", "visitors" => 1}]@@ -1559,7 +1559,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/referrers/example?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "example.com/page1", "visitors" => 1}]end@@ -1596,7 +1596,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/referrers/10words?period=day&date=2021-01-01&detailed=true")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "10words.com","visitors" => 2,@@ -1649,13 +1649,13 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest doconn = get(conn, "/api/stats/#{site.domain}/referrers/!Google?period=day")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "duckduckgo.com", "visitors" => 1}]conn = get(conn, "/api/stats/#{site.domain}/referrers/Google|DuckDuckGo?period=day")- assert [entry1, entry2] = json_response(conn, 200)+ assert [entry1, entry2] = json_response(conn, 200)["results"]assert %{"name" => "google.com", "visitors" => 2} in [entry1, entry2]assert %{"name" => "duckduckgo.com", "visitors" => 1} in [entry1, entry2]end@@ -1688,7 +1688,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/referrers/10words?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "10words.com","total_visitors" => 2,@@ -1726,7 +1726,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do"/api/stats/#{site.domain}/referrers/10words?period=day&filters=#{filters}")- assert json_response(conn, 200) == [+ assert json_response(conn, 200)["results"] == [%{"name" => "10words.com","total_visitors" => 2,diff --git a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exsindex fd77e5279d..bfe7ae16d4 100644--- a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs@@ -634,4 +634,611 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do}endend++ describe "imported data" do+ setup [:create_user, :log_in, :create_site, :create_site_import]++ test "merges country suggestions from native and imported data", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], country_code: "US"),+ build(:pageview, timestamp: ~N[2019-01-01 23:30:01], country_code: "US"),+ build(:pageview, timestamp: ~N[2019-01-01 23:40:01], country_code: "US"),+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], country_code: "GB"),+ build(:imported_locations, date: ~D[2019-01-01], country: "GB", pageviews: 3)+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/country?period=month&date=2019-01-01&q=Unit&with_imported=true"+ )++ assert json_response(conn, 200) == [+ %{"value" => "GB", "label" => "United Kingdom"},+ %{"value" => "US", "label" => "United States"}+ ]+ end++ test "ignores imported data in country suggestions when a different property is filtered by",+ %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:pageview, country_code: "EE", referrer_source: "Bing"),+ build(:imported_locations, country: "GB")+ ])++ filters = Jason.encode!(%{source: "Bing"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/country?filters=#{filters}&q=&with_imported=true"+ )++ assert json_response(conn, 200) == [%{"value" => "EE", "label" => "Estonia"}]+ end++ test "queries imported countries when filtering by country", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:imported_locations, date: ~D[2019-01-01], country: "EE")+ ])++ filters = Jason.encode!(%{country: "EE"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/country?period=month&date=2019-01-01&filters=#{filters}&q=&with_imported=true"+ )++ assert json_response(conn, 200) == [%{"value" => "EE", "label" => "Estonia"}]+ end++ test "ignores imported country data when not requested", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:imported_locations, date: ~D[2019-01-01], country: "GB", pageviews: 3)+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/country?period=month&date=2019-01-01&q="+ )++ assert json_response(conn, 200) == []+ end++ for {q, label} <- [{"", "without filter"}, {"H", "with filter"}] do+ test "merges region suggestions from native and imported data #{label}", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:pageview, country_code: "EE", subdivision1_code: "EE-37"),+ build(:pageview, country_code: "EE", subdivision1_code: "EE-39"),+ build(:pageview, country_code: "EE", subdivision1_code: "EE-39"),+ build(:imported_locations, country: "EE", region: "EE-37", pageviews: 2)+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/region?q=#{unquote(q)}&with_imported=true"+ )++ assert json_response(conn, 200) == [+ %{"value" => "EE-37", "label" => "Harjumaa"},+ %{"value" => "EE-39", "label" => "Hiiumaa"}+ ]+ end+ end++ test "handles invalid region codes in imported data gracefully (GA4)", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ # NOTE: Currently, the regions imported from GA4 do not conform to region code standard+ # we are using. Instead, literal region names are persisted. Those names often do not+ # match the names from our region databases either. Regardless of that, we still consider+ # them when filtering suggestions.++ populate_stats(site, site_import.id, [+ build(:imported_locations, country: "EE", region: "EE-37", pageviews: 2),+ build(:imported_locations, country: "EE", region: "Hiiumaa", pageviews: 1)+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/region?q=&with_imported=true"+ )++ assert json_response(conn, 200) == [+ %{"value" => "EE-37", "label" => "Harjumaa"},+ %{"value" => "Hiiumaa", "label" => "Hiiumaa"}+ ]++ conn2 =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/region?q=H&with_imported=true"+ )++ assert json_response(conn2, 200) == [+ %{"value" => "EE-37", "label" => "Harjumaa"},+ %{"value" => "Hiiumaa", "label" => "Hiiumaa"}+ ]+ end++ test "ignores imported data in region suggestions when a different property is filtered by",+ %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:pageview,+ country_code: "EE",+ subdivision1_code: "EE-39",+ referrer_source: "Bing"+ ),+ build(:imported_locations, country: "EE", region: "EE-37")+ ])++ filters = Jason.encode!(%{source: "Bing"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/region?filters=#{filters}&q=&with_imported=true"+ )++ assert json_response(conn, 200) == [%{"value" => "EE-39", "label" => "Hiiumaa"}]+ end++ test "queries imported regions when filtering by region", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:imported_locations, date: ~D[2019-01-01], region: "EE-39")+ ])++ filters = Jason.encode!(%{region: "EE-39"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/region?period=month&date=2019-01-01&filters=#{filters}&q=&with_imported=true"+ )++ assert json_response(conn, 200) == [%{"value" => "EE-39", "label" => "Hiiumaa"}]+ end++ test "ignores imported region data when not requested", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:imported_locations, country: "EE", region: "EE-37", pageviews: 2)+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/region?q="+ )++ assert json_response(conn, 200) == []+ end++ for {q, label} <- [{"", "without filter"}, {"l", "with filter"}] do+ test "merges city suggestions from native and imported data #{label}", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:pageview,+ country_code: "EE",+ subdivision1_code: "EE-37",+ city_geoname_id: 588_409+ ),+ build(:pageview,+ country_code: "EE",+ subdivision1_code: "EE-39",+ city_geoname_id: 591_632+ ),+ build(:pageview,+ country_code: "EE",+ subdivision1_code: "EE-39",+ city_geoname_id: 591_632+ ),+ build(:imported_locations, country: "EE", region: "EE-37", city: 588_409, pageviews: 2)+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/city?q=#{unquote(q)}&with_imported=true"+ )++ assert json_response(conn, 200) == [+ %{"value" => "588409", "label" => "Tallinn"},+ %{"value" => "591632", "label" => "Kärdla"}+ ]+ end+ end++ test "ignores imported data in city suggestions when a different property is filtered by", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:pageview,+ country_code: "EE",+ subdivision1_code: "EE-39",+ city_geoname_id: 591_632,+ referrer_source: "Bing"+ ),+ build(:imported_locations, country: "EE", region: "EE-37", city: 588_409)+ ])++ filters = Jason.encode!(%{source: "Bing"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/city?filters=#{filters}&q=&with_imported=true"+ )++ assert json_response(conn, 200) == [%{"value" => "591632", "label" => "Kärdla"}]+ end++ test "queries imported cities when filtering by city", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:imported_locations, date: ~D[2019-01-01], city: 591_632)+ ])++ filters = Jason.encode!(%{city: "591632"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/city?period=month&date=2019-01-01&filters=#{filters}&q=&with_imported=true"+ )++ assert json_response(conn, 200) == [%{"value" => "591632", "label" => "Kärdla"}]+ end++ test "ignores imported city data when not requested", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:imported_locations, country: "EE", region: "EE-37", city: 588_409, pageviews: 2)+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/city?q="+ )++ assert json_response(conn, 200) == []+ end++ test "ignores imported data when asking for prop key and value suggestions", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:pageview,+ "meta.key": ["url"],+ "meta.value": ["http://example1.com"],+ timestamp: ~N[2022-01-01 00:00:00]+ ),+ build(:imported_custom_events,+ date: ~D[2022-01-01],+ name: "Outbound Link: Click",+ link_url: "http://example2.com"+ ),+ build(:imported_custom_events,+ date: ~D[2022-01-01],+ name: "404",+ path: "/dev/null"+ )+ ])++ key_conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/prop_key?period=day&date=2022-01-01&with_imported=true"+ )++ assert json_response(key_conn, 200) == [%{"label" => "url", "value" => "url"}]++ filters = Jason.encode!(%{props: %{url: "!(none)"}})++ value_conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/prop_value?period=day&date=2022-01-01&with_imported=true&filters=#{filters}"+ )++ assert json_response(value_conn, 200) == [+ %{"label" => "http://example1.com", "value" => "http://example1.com"}+ ]+ end++ for {q, label} <- [{"", "without filter"}, {"g", "with filter"}] do+ test "merges source suggestions from native and imported data #{label}", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], referrer_source: "Bing"),+ build(:pageview, timestamp: ~N[2019-01-01 23:30:01], referrer_source: "Bing"),+ build(:pageview, timestamp: ~N[2019-01-01 23:40:01], referrer_source: "Bing"),+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], referrer_source: "Google"),+ build(:imported_sources, date: ~D[2019-01-01], source: "Google", pageviews: 3)+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/source?period=month&date=2019-01-01&q=#{unquote(q)}&with_imported=true"+ )++ assert json_response(conn, 200) == [+ %{"value" => "Google", "label" => "Google"},+ %{"value" => "Bing", "label" => "Bing"}+ ]+ end+ end++ for {q, label} <- [{"", "without filter"}, {"o", "with filter"}] do+ test "merges screen suggestions from native and imported data #{label}", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], screen_size: "Mobile"),+ build(:pageview, timestamp: ~N[2019-01-01 23:30:01], screen_size: "Mobile"),+ build(:pageview, timestamp: ~N[2019-01-01 23:40:01], screen_size: "Mobile"),+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], screen_size: "Desktop"),+ build(:imported_devices, date: ~D[2019-01-01], device: "Desktop", pageviews: 3)+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/screen?period=month&date=2019-01-01&q=#{unquote(q)}&with_imported=true"+ )++ assert json_response(conn, 200) == [+ %{"value" => "Desktop", "label" => "Desktop"},+ %{"value" => "Mobile", "label" => "Mobile"}+ ]+ end+ end++ for {q, label} <- [{"", "without filter"}, {"o", "with filter"}] do+ test "merges page suggestions from native and imported data #{label}", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/welcome"),+ build(:pageview, timestamp: ~N[2019-01-01 23:30:01], pathname: "/welcome"),+ build(:pageview, timestamp: ~N[2019-01-01 23:40:01], pathname: "/welcome"),+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/blog"),+ build(:imported_pages, date: ~D[2019-01-01], page: "/blog", pageviews: 3)+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/page?period=month&date=2019-01-01&q=#{unquote(q)}&with_imported=true"+ )++ assert json_response(conn, 200) == [+ %{"value" => "/blog", "label" => "/blog"},+ %{"value" => "/welcome", "label" => "/welcome"}+ ]+ end+ end++ for {q, label} <- [{"", "without filter"}, {"o", "with filter"}] do+ test "merges entry page suggestions from native and imported data #{label}", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/welcome"),+ build(:pageview, timestamp: ~N[2019-01-01 23:30:01], pathname: "/welcome"),+ build(:pageview, timestamp: ~N[2019-01-01 23:40:01], pathname: "/welcome"),+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/blog"),+ build(:imported_entry_pages, date: ~D[2019-01-01], entry_page: "/blog", pageviews: 3)+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/entry_page?period=month&date=2019-01-01&q=#{unquote(q)}&with_imported=true"+ )++ assert json_response(conn, 200) == [+ %{"value" => "/blog", "label" => "/blog"},+ %{"value" => "/welcome", "label" => "/welcome"}+ ]+ end+ end++ for {q, label} <- [{"", "without filter"}, {"o", "with filter"}] do+ test "merges exit page suggestions from native and imported data #{label}", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/welcome"),+ build(:pageview, timestamp: ~N[2019-01-01 23:30:01], pathname: "/welcome"),+ build(:pageview, timestamp: ~N[2019-01-01 23:40:01], pathname: "/welcome"),+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/blog"),+ build(:imported_exit_pages, date: ~D[2019-01-01], exit_page: "/blog", pageviews: 3)+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/exit_page?period=month&date=2019-01-01&q=#{unquote(q)}&with_imported=true"+ )++ assert json_response(conn, 200) == [+ %{"value" => "/blog", "label" => "/blog"},+ %{"value" => "/welcome", "label" => "/welcome"}+ ]+ end+ end++ for {q, label} <- [{"", "without filter"}, {"o", "with filter"}] do+ test "merges browser suggestions from native and imported data #{label}", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], browser: "Chrome"),+ build(:pageview, timestamp: ~N[2019-01-01 23:30:01], browser: "Chrome"),+ build(:pageview, timestamp: ~N[2019-01-01 23:40:01], browser: "Chrome"),+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], browser: "Firefox"),+ build(:imported_browsers, date: ~D[2019-01-01], browser: "Firefox", pageviews: 3)+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/browser?period=month&date=2019-01-01&q=#{unquote(q)}&with_imported=true"+ )++ assert json_response(conn, 200) == [+ %{"value" => "Firefox", "label" => "Firefox"},+ %{"value" => "Chrome", "label" => "Chrome"}+ ]+ end+ end++ for {q, label} <- [{"", "without filter"}, {"i", "with filter"}] do+ test "merges operating system suggestions from native and imported data #{label}", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], operating_system: "Linux"),+ build(:pageview, timestamp: ~N[2019-01-01 23:30:01], operating_system: "Linux"),+ build(:pageview, timestamp: ~N[2019-01-01 23:40:01], operating_system: "Linux"),+ build(:pageview, timestamp: ~N[2019-01-01 23:00:01], operating_system: "Windows"),+ build(:imported_operating_systems,+ date: ~D[2019-01-01],+ operating_system: "Windows",+ pageviews: 3+ )+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/operating_system?period=month&date=2019-01-01&q=#{unquote(q)}&with_imported=true"+ )++ assert json_response(conn, 200) == [+ %{"value" => "Windows", "label" => "Windows"},+ %{"value" => "Linux", "label" => "Linux"}+ ]+ end+ end++ test "does not query imported data when a different property is filtered by", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:pageview,+ timestamp: ~N[2019-01-01 23:00:01],+ pathname: "/blog",+ operating_system: "Linux"+ ),+ build(:imported_operating_systems, date: ~D[2019-01-01], operating_system: "Windows")+ ])++ filters = Jason.encode!(%{page: "/blog"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/operating_system?period=month&date=2019-01-01&filters=#{filters}&q=&with_imported=true"+ )++ assert json_response(conn, 200) == [%{"value" => "Linux", "label" => "Linux"}]+ end++ test "queries imported data when filtering by the same property", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:pageview,+ timestamp: ~N[2019-01-01 23:00:01],+ pathname: "/blog",+ operating_system: "Linux"+ ),+ build(:imported_operating_systems, date: ~D[2019-01-01], operating_system: "Windows"),+ build(:imported_operating_systems, date: ~D[2019-01-01], operating_system: "Linux")+ ])++ filters = Jason.encode!(%{os: "!Linux"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/suggestions/operating_system?period=month&date=2019-01-01&filters=#{filters}&q=&with_imported=true"+ )++ assert json_response(conn, 200) == [%{"value" => "Windows", "label" => "Windows"}]+ end+ endenddiff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exsindex 4ec122086c..6483b6619e 100644--- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs@@ -530,6 +530,102 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do%{"name" => "Visit duration", "value" => 303, "graph_metric" => "visit_duration"}]end++ test ":member filter on country", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview,+ country_code: "EE",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ country_code: "EE",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:01:00]+ ),+ build(:imported_locations,+ country: "EE",+ date: ~D[2021-01-01],+ visitors: 1,+ visits: 3,+ pageviews: 34,+ bounces: 0,+ visit_duration: 420+ ),+ build(:imported_locations,+ country: "FR",+ date: ~D[2021-01-01],+ visitors: 3,+ visits: 7,+ pageviews: 65,+ bounces: 1,+ visit_duration: 300+ ),+ build(:imported_locations, country: "US", date: ~D[2021-01-01], visitors: 999)+ ])++ filters = Jason.encode!(%{country: "EE|FR"})+ q = "?period=day&date=2021-01-01&with_imported=true&filters=#{filters}"++ conn = get(conn, "/api/stats/#{site.domain}/top-stats#{q}")++ res = json_response(conn, 200)++ assert res["top_stats"] == [+ %{"name" => "Unique visitors", "value" => 5, "graph_metric" => "visitors"},+ %{"name" => "Total visits", "value" => 11, "graph_metric" => "visits"},+ %{"name" => "Total pageviews", "value" => 101, "graph_metric" => "pageviews"},+ %{+ "name" => "Views per visit",+ "value" => 9.18,+ "graph_metric" => "views_per_visit"+ },+ %{"name" => "Bounce rate", "value" => 9, "graph_metric" => "bounce_rate"},+ %{"name" => "Visit duration", "value" => 71, "graph_metric" => "visit_duration"}+ ]+ end++ test ":is filter on page returns only visitors, visits and pageviews", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview,+ pathname: "/",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:01:00]+ ),+ build(:imported_pages,+ page: "/",+ date: ~D[2021-01-01],+ visitors: 1,+ visits: 3,+ pageviews: 34+ ),+ build(:imported_pages, page: "/ignored", date: ~D[2021-01-01], visitors: 999)+ ])++ filters = Jason.encode!(%{page: "/"})+ q = "?period=day&date=2021-01-01&with_imported=true&filters=#{filters}"++ conn = get(conn, "/api/stats/#{site.domain}/top-stats#{q}")++ res = json_response(conn, 200)++ assert res["top_stats"] == [+ %{"name" => "Unique visitors", "value" => 2, "graph_metric" => "visitors"},+ %{"name" => "Total visits", "value" => 4, "graph_metric" => "visits"},+ %{"name" => "Total pageviews", "value" => 36, "graph_metric" => "pageviews"}+ ]+ endenddescribe "GET /api/stats/top-stats - realtime" do@@ -1358,7 +1454,7 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do"/api/stats/#{site.domain}/top-stats?period=month&date=2021-01-01&with_imported=true&comparison=year_over_year")- assert %{"top_stats" => top_stats, "with_imported" => true} = json_response(conn, 200)+ assert %{"top_stats" => top_stats, "includes_imported" => true} = json_response(conn, 200)assert %{"change" => 100,@@ -1388,7 +1484,7 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do"/api/stats/#{site.domain}/top-stats?period=month&date=2021-01-01&with_imported=false&comparison=year_over_year")- assert %{"top_stats" => top_stats, "with_imported" => false} = json_response(conn, 200)+ assert %{"top_stats" => top_stats, "includes_imported" => false} = json_response(conn, 200)assert %{"change" => 100,