:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(heatmaps): normalize and validate heatmap url patterns (#113821)
Jordan Mryyan973f1534da05
diff --git a/frontend/src/toolbar/elements/heatmapToolbarMenuLogic.test.ts b/frontend/src/toolbar/elements/heatmapToolbarMenuLogic.test.tsindex be3d25010002..c824fd9b59bd 100644--- a/frontend/src/toolbar/elements/heatmapToolbarMenuLogic.test.ts+++ b/frontend/src/toolbar/elements/heatmapToolbarMenuLogic.test.ts@@ -1,5 +1,7 @@import { expectLogic } from 'kea-test-utils'+import { heatmapDataLogic } from 'lib/components/heatmaps/heatmapDataLogic'+import { resumeKeaLoadersErrors, silenceKeaLoadersErrors } from '~/initKea'import { initKeaTests } from '~/test/init'import { currentPageLogic } from '~/toolbar/stats/currentPageLogic'@@ -100,6 +102,13 @@ describe('heatmapToolbarMenuLogic', () => {},],],+ [+ 'an exact url filter on the normalized url when the wildcard href is a url without a wildcard',+ 'https://example.com/page',+ 'https://example.com',+ null,+ [{ key: '$current_url', value: 'https://example.com/', operator: 'exact', type: 'event' }],+ ],['an element selector filter when an area is chosen','https://example.com/page',@@ -345,6 +354,77 @@ describe('heatmapToolbarMenuLogic', () => {])})+ it.each([+ [+ 'setHref',+ 'https://example.com/other?filter=[a-b]&count={2,1}',+ 'exact',+ 'https://example.com/other?filter=[a-b]&count={2,1}',+ ],+ [+ 'setWildcardHref',+ 'https://example.com/other?filter=[a-b]&count={2,1}',+ 'exact',+ 'https://example.com/other?filter=[a-b]&count={2,1}',+ ],+ ['setHref', 'https://example.com/search?q=*', 'exact', 'https://example.com/search?q=*'],+ [+ 'setWildcardHref',+ 'https://example.com/products/*?filter=[a-b]',+ 'pattern',+ String.raw`https\:\/\/example\.com\/products\/*\?filter\=\[a-b\]`,+ ],+ ['setWildcardHref', 'not a URL', 'exact', ''],+ ] as const)('normalizes %s input %s for heatmap data', async (action, input, matchType, href) => {+ await expectLogic(logic, () => logic.actions.enableHeatmap()).toDispatchActions(['getElementStatsSuccess'])+ await expectLogic(heatmapDataLogic({ context: 'toolbar' }), () =>+ logic.actions[action](input)+ ).toMatchValues({+ href,+ hrefMatchType: matchType,+ })+ expect(currentPageLogic.values.wildcardHref).toEqual(input)+ })++ it.each([+ [+ 'setWildcardHref',+ 'https://example.com/products/*',+ false,+ 'pattern',+ String.raw`https\:\/\/example\.com\/products\/*`,+ ],+ [+ 'setWildcardHref',+ 'https://example.com/products/123?lang=en',+ true,+ 'pattern',+ String.raw`https\:\/\/example\.com\/products\/*\?lang\=en`,+ ],+ [+ 'setWildcardHref',+ 'https://example.com/products/latest?lang=en',+ true,+ 'exact',+ 'https://example.com/products/latest?lang=en',+ ],+ ['setHref', 'https://example.com/search?q=*', false, 'exact', 'https://example.com/search?q=*'],+ ] as const)(+ 'preserves matching for %s %s when reopening the menu',+ async (action, input, automatic, matchType, href) => {+ logic.actions[action](input)+ if (automatic) {+ currentPageLogic.actions.autoWildcardHref()+ }+ await expectLogic(heatmapDataLogic({ context: 'toolbar' }), () =>+ logic.actions.enableHeatmap()+ ).toMatchValues({+ href,+ hrefMatchType: matchType,+ })+ }+ )+it('does not fetch element stats when clickmaps are toggled off while the request is debouncing', async () => {await expectLogic(logic, () => {logic.actions.enableHeatmap()diff --git a/frontend/src/toolbar/elements/heatmapToolbarMenuLogic.ts b/frontend/src/toolbar/elements/heatmapToolbarMenuLogic.tsindex 5652a61a89d8..b5d0ffb00a1e 100644--- a/frontend/src/toolbar/elements/heatmapToolbarMenuLogic.ts+++ b/frontend/src/toolbar/elements/heatmapToolbarMenuLogic.ts@@ -18,7 +18,11 @@ import { collectAllElementsDeep } from 'query-selector-shadow-dom'import type { PaginatedResponse } from 'lib/api'import { heatmapDataLogic } from 'lib/components/heatmaps/heatmapDataLogic'-import { escapeUnescapedRegex, heatmapUrlPatternToRegex } from 'lib/components/heatmaps/heatmapUrlMatch'+import {+ escapeUnescapedRegex,+ heatmapUrlPatternToRegex,+ resolveHeatmapUrlFilter,+} from 'lib/components/heatmaps/heatmapUrlMatch'import { HeatmapBoundsFilter } from 'lib/components/heatmaps/types'import { FEATURE_FLAGS } from 'lib/constants'import { createSliceYielder } from 'lib/utils/async'@@ -171,17 +175,20 @@ export function buildElementStatsProperties(wildcardHref: string,areaSelector: string | null): AnyPropertyFilter[] {+ const urlFilter = wildcardHref === href ? null : resolveHeatmapUrlFilter(wildcardHref)const properties: AnyPropertyFilter[] = [- wildcardHref === href+ wildcardHref === href || urlFilter?.matchType === 'exact'? {key: '$current_url',- value: href,+ value: urlFilter?.href ?? href,operator: PropertyOperator.Exact,type: PropertyFilterType.Event,}: {key: '$current_url',- value: heatmapUrlPatternToRegex(wildcardHref.split('*').map(escapeUnescapedRegex).join('*')),+ value:+ urlFilter?.regex ??+ heatmapUrlPatternToRegex(wildcardHref.split('*').map(escapeUnescapedRegex).join('*')),operator: PropertyOperator.Regex,type: PropertyFilterType.Event,},@@ -1219,7 +1226,14 @@ export const heatmapToolbarMenuLogic = kea<heatmapToolbarMenuLogicType>([},enableHeatmap: () => {- actions.setDataHref(values.href)+ if (values.wildcardHref === values.href) {+ actions.setHrefMatchType('exact')+ actions.setDataHref(values.href)+ } else {+ const urlFilter = resolveHeatmapUrlFilter(values.wildcardHref)+ actions.setHrefMatchType(urlFilter?.matchType ?? 'exact')+ actions.setDataHref(urlFilter?.href ?? '')+ }actions.loadAllEnabled()toolbarPosthogJS.capture('toolbar mode triggered', { mode: 'heatmap', enabled: true })},@@ -1359,7 +1373,7 @@ export const heatmapToolbarMenuLogic = kea<heatmapToolbarMenuLogicType>([setHref: ({ href }) => {if (values.heatmapEnabled) {- actions.setHrefMatchType(href === window.location.href ? 'exact' : 'pattern')+ actions.setHrefMatchType('exact')actions.setDataHref(href)}// the area filter describes an element on the page it was picked on; carrying it@@ -1375,8 +1389,9 @@ export const heatmapToolbarMenuLogic = kea<heatmapToolbarMenuLogicType>([setWildcardHref: ({ href }) => {if (values.heatmapEnabled) {- actions.setHrefMatchType(href === window.location.href ? 'exact' : 'pattern')- actions.setDataHref(href)+ const urlFilter = resolveHeatmapUrlFilter(href)+ actions.setHrefMatchType(urlFilter?.matchType ?? 'exact')+ actions.setDataHref(urlFilter?.href ?? '')}actions.maybeLoadClickmap()},diff --git a/frontend/src/toolbar/stats/currentPageLogic.ts b/frontend/src/toolbar/stats/currentPageLogic.tsindex 02c1359c4995..bec30b62f8ad 100644--- a/frontend/src/toolbar/stats/currentPageLogic.ts+++ b/frontend/src/toolbar/stats/currentPageLogic.ts@@ -38,7 +38,7 @@ const autoWildcardHref = (url: string): string => {queryParams[i] = `${key}=${replaceWithWildcard(value)}`}- url = `${url}\\?${queryParams.join('&')}`+ url = `${url}?${queryParams.join('&')}`}return urldiff --git a/products/web_analytics/backend/api/heatmaps_api.py b/products/web_analytics/backend/api/heatmaps_api.pyindex 1498f07a32ba..87c7997fea91 100644--- a/products/web_analytics/backend/api/heatmaps_api.py+++ b/products/web_analytics/backend/api/heatmaps_api.py@@ -14,6 +14,7 @@ from django.http import HttpResponsefrom django.utils import timezonefrom django.utils.cache import get_conditional_response, patch_cache_control, patch_vary_headers+import re2import structlogimport posthoganalyticsfrom drf_spectacular.types import OpenApiTypes@@ -499,7 +500,14 @@ class HeatmapsRequestSerializer(serializers.Serializer):def validate_url_pattern(self, value: str | None) -> str | None:if value is None:return None- return anchor_url_pattern(value)+ pattern = anchor_url_pattern(value)+ options = re2.Options()+ options.log_errors = False+ try:+ re2.compile(pattern, options=options)+ except re2.error as error:+ raise serializers.ValidationError("Enter a valid URL pattern.") from error+ return patterndef validate(self, values) -> dict:if ("timestamp_from" in values) != ("timestamp_to" in values):diff --git a/products/web_analytics/backend/test/test_heatmaps_api.py b/products/web_analytics/backend/test/test_heatmaps_api.pyindex 0d582f55402c..2cdecb62ec1a 100644--- a/products/web_analytics/backend/test/test_heatmaps_api.py+++ b/products/web_analytics/backend/test/test_heatmaps_api.py@@ -988,6 +988,29 @@ class TestSessionRecordings(APIBaseTest, ClickhouseTestMixin, QueryMatchingTest))+class TestHeatmapUrlPatternValidation(SimpleTestCase):+ @parameterized.expand(+ [+ ("bare_wildcard", "https://example.com/*", "^https://example.com/.+$"),+ ("regex_wildcard", r"https://example\.com/.*", r"^https://example\.com/.*$"),+ ("repeated_wildcards", "https://example.com/**", "^https://example.com/.+.+$"),+ ("escaped_punctuation", r"https://example\.com/a\[b\]\?x=1", r"^https://example\.com/a\[b\]\?x=1$"),+ ("page_pattern", r"^https://example\.com\/?(\?.*)?(#.*)?$", r"^https://example\.com\/?(\?.*)?(#.*)?$"),+ ("unclosed_bracket", "https://example.com/[", None),+ ("bad_repetition", "https://example.com/a{2,1}", None),+ ("unsupported_lookahead", "https://example.com/(?=a)", None),+ ]+ )+ def test_validates_transformed_re2_pattern(self, _name: str, pattern: str, expected: str | None) -> None:+ serializer = HeatmapsRequestSerializer(data={"url_pattern": pattern}, context={"team": Team()})+ self.assertEqual(serializer.is_valid(), expected is not None)+ if expected is None:+ self.assertEqual(set(serializer.errors), {"url_pattern"})+ self.assertEqual(serializer.errors["url_pattern"][0], "Enter a valid URL pattern.")+ else:+ self.assertEqual(serializer.validated_data["url_pattern"], expected)++class TestHeatmapsRequestIntervals(SimpleTestCase):@parameterized.expand([