:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(events): parse elements chains in linear time (#119842)
Eli Reisman1b7df8a2d56c
diff --git a/frontend/src/lib/utils/elements-chain.test.ts b/frontend/src/lib/utils/elements-chain.test.tsnew file mode 100644index 000000000000..ed26e9cd1eb5--- /dev/null+++ b/frontend/src/lib/utils/elements-chain.test.ts@@ -0,0 +1,55 @@+import { chainToElements, parseElementAttributes, splitElementsChain } from './elements-chain'++describe('chainToElements', () => {+ it('parses a well-formed chain', () => {+ const elements = chainToElements(+ 'a.small:href="/a-url"nth-child="1"nth-of-type="0"text="say \\"hi\\"";button:nth-child="0"nth-of-type="0"'+ )+ expect(elements).toMatchObject([+ { tag_name: 'a', attr_class: ['small'], href: '/a-url', nth_child: 1, text: 'say \\"hi\\"', order: 0 },+ { tag_name: 'button', nth_child: 0, nth_of_type: 0, order: 1 },+ ])+ })++ it('keeps a colon in a class name and an empty attribute value', () => {+ const elements = chainToElements('div.sm:flex.x:attr__alt=""href="/a"nth-child="2"')+ expect(elements).toMatchObject([+ { tag_name: 'div', attr_class: ['sm:flex', 'x'], href: '/a', nth_child: 2, attributes: { attr__alt: '' } },+ ])+ })++ it.each([+ ['a long backslash run', 'a:text="' + '\\'.repeat(5000), ['a', '\\'.repeat(5000)]],+ ['a long run of escaped quotes', '"\\'.repeat(5000), Array(5000).fill('\\')],+ ['a quote at the end of the chain', 'button:nth-child="1"text="unclosed', ['button', 'unclosed']],+ ])('ends the element at a quote that is never closed: %s', (_name, chain, expectedTagNames) => {+ const elements = chainToElements(chain)+ expect(elements.map((element) => element.tag_name)).toEqual(expectedTagNames)+ expect(elements.every((element) => element.text === undefined)).toBe(true)+ })++ it('drops a value that ends in a backslash at the end of the chain', () => {+ const elements = chainToElements('button:nth-child="2";a:nth-child="1"href="C:\\"')+ expect(elements).toMatchObject([{ tag_name: 'button' }, { tag_name: 'a', nth_child: 1 }])+ expect(elements[1].href).toBeUndefined()+ })++ it.each([+ ['a value that never closes', 'x="' + 'a'.repeat(80_000), []],+ ['a never-closing value after a closed one', 'href="/a"text="' + '\\"'.repeat(2000), [['href', '/a']]],+ [+ 'a backslash before an escaped quote',+ 'text="foo\\\\"bar"nth-child="1"',+ [+ ['text', 'foo\\\\"bar'],+ ['nth-child', '1'],+ ],+ ],+ ])('parses attributes with %s', (_name, attributes, expected) => {+ expect(parseElementAttributes(attributes)).toEqual(expected)+ })++ it('splits on whitespace and semicolons outside quotes only', () => {+ expect(splitElementsChain('a:text="x; y" ;b\tc')).toEqual(['a:text="x; y"', 'b', 'c'])+ })+})diff --git a/frontend/src/lib/utils/elements-chain.ts b/frontend/src/lib/utils/elements-chain.tsindex 278f41c26a91..c4eb13bd76ce 100644--- a/frontend/src/lib/utils/elements-chain.ts+++ b/frontend/src/lib/utils/elements-chain.ts@@ -1,67 +1,151 @@import { ElementType } from '~/types'-// NOTE: This function should not be edited directly but rather copied from nodejs/src/common/utils/elements-chain.ts-export function chainToElements(chain: string, options: { throwOnError?: boolean } = {}): ElementType[] {- const elements: ElementType[] = []+// nodejs/src/common/utils/elements-chain.ts reads the same grammar under RE2. The browser has no+// RE2, so every scan here is a single pass over client-supplied input that cannot backtrack.- // Below splits all elements by ;, while ignoring escaped quotes and semicolons within quotes- const splitChainRegex = /(?:[^\s;"]|"(?:\\.|[^"])*")+/g+const QUOTE = '"'+const ESCAPE = '\\'+const ELEMENT_SEPARATOR = ';'+const ATTRIBUTE_SEPARATOR = ':'+const WHITESPACE = /\s/+const ATTRIBUTE_KEY_CHARACTER = /[a-zA-Z0-9_-]/- // Below splits the tag/classes from attributes- // Needs a regex because classes can have : too- const splitClassAttributes = /(.*?)($|:([a-zA-Z\-_0-9]*=.*))/g- const parseAttributesRegex = /((.*?)="((?:\\"|[^"])*)")/gm+/** Serializers escape only quotes, so a quote preceded by a backslash never closes a value. */+function findClosingQuote(text: string, valueStart: number): number {+ let position = valueStart+ while (true) {+ const closing = text.indexOf(QUOTE, position)+ if (closing === -1) {+ return -1+ }+ if (text[closing - 1] !== ESCAPE) {+ return closing+ }+ position = closing + 1+ }+}++/** A quote with no closing quote ends the element, unless the chain ends with an escaped quote. */+export function splitElementsChain(chain: string): string[] {+ const elements: string[] = []+ let tokenStart = -1+ let position = 0+ // A failed closing-quote search skipped every later quote as escaped, so later searches fail too.+ let closingQuotesRemain = true++ const endToken = (end: number): void => {+ if (tokenStart !== -1 && end > tokenStart) {+ elements.push(chain.slice(tokenStart, end))+ }+ tokenStart = -1+ }++ while (position < chain.length) {+ const character = chain[position]+ if (character === QUOTE) {+ let closing = closingQuotesRemain ? findClosingQuote(chain, position + 1) : -1+ if (closing === -1 && chain.length - 1 > position && chain.endsWith(ESCAPE + QUOTE)) {+ closing = chain.length - 1+ }+ if (closing === -1) {+ closingQuotesRemain = false+ endToken(position)+ position += 1+ continue+ }+ if (tokenStart === -1) {+ tokenStart = position+ }+ position = closing + 1+ } else if (character === ELEMENT_SEPARATOR || WHITESPACE.test(character)) {+ endToken(position)+ position += 1+ } else {+ if (tokenStart === -1) {+ tokenStart = position+ }+ position += 1+ }+ }+ endToken(chain.length)+ return elements+}++function startsAttributeKey(elString: string, position: number): boolean {+ while (position < elString.length && ATTRIBUTE_KEY_CHARACTER.test(elString[position])) {+ position += 1+ }+ return elString[position] === '='+}+/** The attributes start at the first colon followed by key=, so a class name may contain a colon. */+function splitTagAndAttributes(elString: string): { tag: string; attributes: string | null } {+ let colon = elString.indexOf(ATTRIBUTE_SEPARATOR)+ while (colon !== -1) {+ if (startsAttributeKey(elString, colon + 1)) {+ return { tag: elString.slice(0, colon), attributes: elString.slice(colon + 1) }+ }+ colon = elString.indexOf(ATTRIBUTE_SEPARATOR, colon + 1)+ }+ return { tag: elString, attributes: null }+}++export function parseElementAttributes(attributes: string): [string, string][] {+ const pairs: [string, string][] = []+ let start = 0+ while (true) {+ const separator = attributes.indexOf('="', start)+ if (separator === -1) {+ return pairs+ }+ const valueStart = separator + 2+ const closing = findClosingQuote(attributes, valueStart)+ if (closing === -1) {+ return pairs+ }+ pairs.push([attributes.slice(start, separator), attributes.slice(valueStart, closing)])+ start = closing + 1+ }+}++export function chainToElements(chain: string): ElementType[] {chain = chain.replace(/\n/g, '')- try {- Array.from(chain.matchAll(splitChainRegex))- .map((r) => r[0])- .forEach((elString, index) => {- const elStringSplit = Array.from(elString.matchAll(splitClassAttributes))[0]- const attributes =- elStringSplit.length > 3 && elStringSplit[3]- ? Array.from(elStringSplit[3].matchAll(parseAttributesRegex)).map((a) => [a[2], a[3]])- : []+ return splitElementsChain(chain).map((elString, index) => {+ const { tag, attributes } = splitTagAndAttributes(elString)- const element: ElementType = {- attributes: {},- order: index,- tag_name: '',- }+ const element: ElementType = {+ attributes: {},+ order: index,+ tag_name: '',+ }- if (elStringSplit[1]) {- const tagAndClass = elStringSplit[1].split('.')- element.tag_name = tagAndClass[0]- if (tagAndClass.length > 1) {- element.attr_class = tagAndClass.slice(1).filter(Boolean)- }- }+ if (tag) {+ const tagAndClass = tag.split('.')+ element.tag_name = tagAndClass[0]+ if (tagAndClass.length > 1) {+ element.attr_class = tagAndClass.slice(1).filter(Boolean)+ }+ }- for (const [key, value] of attributes) {- if (key == 'href') {- element.href = value- } else if (key == 'nth-child') {- element.nth_child = parseInt(value)- } else if (key == 'nth-of-type') {- element.nth_of_type = parseInt(value)- } else if (key == 'text') {- element.text = value- } else if (key == 'attr_id') {- element.attr_id = value- } else if (key) {- if (!element.attributes) {- element.attributes = {}- }- element.attributes[key] = value- }+ for (const [key, value] of attributes ? parseElementAttributes(attributes) : []) {+ if (key == 'href') {+ element.href = value+ } else if (key == 'nth-child') {+ element.nth_child = parseInt(value)+ } else if (key == 'nth-of-type') {+ element.nth_of_type = parseInt(value)+ } else if (key == 'text') {+ element.text = value+ } else if (key == 'attr_id') {+ element.attr_id = value+ } else if (key) {+ if (!element.attributes) {+ element.attributes = {}}- elements.push(element)- })- } catch (error) {- if (options.throwOnError) {- throw error+ element.attributes[key] = value+ }}- }- return elements+ return element+ })}diff --git a/posthog/models/element/element.py b/posthog/models/element/element.pyindex 2a269cfe4877..937c466e9da6 100644--- a/posthog/models/element/element.py+++ b/posthog/models/element/element.py@@ -1,9 +1,11 @@import re-from collections.abc import Callable+from collections.abc import Callable, Iteratorfrom django.contrib.postgres.fields import ArrayFieldfrom django.db import models+from posthog.dataclasses import frozen+class Element(models.Model):USEFUL_ELEMENTS = ["a", "button", "input", "select", "textarea", "label"]@@ -20,14 +22,142 @@ class Element(models.Model):group = models.ForeignKey("ElementGroup", on_delete=models.CASCADE, null=True, blank=True)-parse_attributes_regex = re.compile(r"(?P<attribute>(?P<key>.*?)\=\"(?P<value>.*?[^\\])\")", re.MULTILINE)+_QUOTE = '"'+_ESCAPE = "\\"+_NEWLINE = "\n"+_ATTRIBUTE_SEPARATOR = ":"+# The chain is client-supplied, so every scan below must stay linear in its length. These+# patterns are a single character class or a class run before one literal, which cannot+# backtrack, and the scanners move past each match so no character is searched twice.+_BARE_RUN_END = re.compile(r'[\s;"]')+_ATTRIBUTE_KEY = re.compile(r"[a-zA-Z0-9_-]*=")+-# Below splits all elements by ;, while ignoring escaped quotes and semicolons within quotes-split_chain_regex = re.compile(r'(?:[^\s;"]|"(?:\\.|[^"])*")+')+@frozen+class ElementAttribute:+ key: str+ value: str-# Below splits the tag/classes from attributes-# Needs a regex because classes can have : too-split_class_attributes = re.compile(r"(.*?)($|:([a-zA-Z\-\_0-9]*=.*))")++def _find_closing_quote(text: str, value_start: int) -> int:+ """Returns the index of the quote that closes the value starting at `value_start`, or -1.++ Serializers escape a quote inside a value with a backslash and escape nothing else, so+ a quote preceded by a backslash never closes the value. The value may be empty.+ """+ position = value_start+ while True:+ closing = text.find(_QUOTE, position)+ if closing == -1:+ return -1+ if text[closing - 1] != _ESCAPE:+ return closing+ position = closing + 1+++def split_elements_chain(chain: str) -> list[str]:+ """Splits an elements chain into one string per element.++ Elements are separated by whitespace or semicolons outside quoted attribute values.+ Inside a quoted value both are literal and a quote preceded by a backslash is escaped.+ A quote with no closing quote ends the current element and is dropped, unless the+ chain ends with an escaped quote, which then closes the value. This is a+ single left-to-right pass over client-supplied input: no input shape can make it+ scan a character twice.+ """+ elements: list[str] = []+ token_start = -1+ position = 0+ length = len(chain)+ # A failed search for a closing quote skipped every later quote as escaped, so the+ # same search from any later quote fails too and must not scan the chain again.+ closing_quotes_remain = True++ def end_token(end: int) -> None:+ nonlocal token_start+ if token_start != -1 and end > token_start:+ elements.append(chain[token_start:end])+ token_start = -1++ while position < length:+ boundary = _BARE_RUN_END.search(chain, position)+ if boundary is None:+ if token_start == -1:+ token_start = position+ break+ boundary_start = boundary.start()+ if boundary_start > position and token_start == -1:+ token_start = position+ if boundary.group() == _QUOTE:+ closing = _find_closing_quote(chain, boundary_start + 1) if closing_quotes_remain else -1+ if closing == -1 and length - 1 > boundary_start and chain.endswith(_ESCAPE + _QUOTE):+ # Serializers escape quotes but not backslashes, so a value that ends in a+ # backslash at the end of the chain reads as unterminated. It closes there.+ closing = length - 1+ if closing == -1:+ closing_quotes_remain = False+ end_token(boundary_start)+ position = boundary_start + 1+ continue+ if token_start == -1:+ token_start = boundary_start+ position = closing + 1+ else:+ end_token(boundary_start)+ position = boundary_start + 1+ end_token(length)+ return elements+++@frozen+class _ElementParts:+ tag: str+ attributes: str | None+++def _split_tag_and_attributes(el_string: str) -> _ElementParts:+ """Splits one element string into its tag part and its attributes part.++ The attributes start at the first colon followed by a key and an equals sign, so a+ class name may contain a colon. A line that holds no such colon is skipped unless it+ is the last line, which then forms the tag part on its own. Each colon only scans the+ key characters after it, which the next colon cannot share, so the scan stays linear.+ """+ line_start = 0+ length = len(el_string)+ while True:+ line_end = el_string.find(_NEWLINE, line_start)+ if line_end == -1:+ line_end = length+ colon = el_string.find(_ATTRIBUTE_SEPARATOR, line_start, line_end)+ while colon != -1:+ if _ATTRIBUTE_KEY.match(el_string, colon + 1, line_end) is not None:+ return _ElementParts(tag=el_string[line_start:colon], attributes=el_string[colon + 1 : line_end])+ colon = el_string.find(_ATTRIBUTE_SEPARATOR, colon + 1, line_end)+ if line_end >= length - 1:+ return _ElementParts(tag=el_string[line_start:line_end], attributes=None)+ line_start = line_end + 1+++def iter_element_attributes(attrs: str) -> Iterator[ElementAttribute]:+ """Yields the attributes found in the attributes part of one element string.++ Each attribute is key="value". The key runs from the end of the previous attribute to+ the next =". A failed search for a closing quote skipped every later quote as escaped,+ so the same search from any later =" fails too and the scan ends. This keeps the scan+ to a single pass over client-supplied input.+ """+ start = 0+ while True:+ separator = attrs.find('="', start)+ if separator == -1:+ return+ value_start = separator + 2+ closing = _find_closing_quote(attrs, value_start)+ if closing == -1:+ return+ yield ElementAttribute(key=attrs[start:separator], value=attrs[value_start:closing])+ start = closing + 1def _escape(input: str) -> str:@@ -111,19 +241,26 @@ def build_attributes_filter(wanted_data_attributes: list[str]) -> Callable[[str]return matches+def _int_or_none(value: str) -> int | None:+ try:+ return int(value)+ except ValueError:+ return None++_PROMOTED_ATTRIBUTES: dict[str, tuple[str, Callable[[str], object]]] = {"text": ("text", str),"href": ("href", str),"attr_id": ("attr_id", str),- "nth-child": ("nth_child", int),- "nth-of-type": ("nth_of_type", int),+ "nth-child": ("nth_child", _int_or_none),+ "nth-of-type": ("nth_of_type", _int_or_none),}def _parse_element(el_string: str, order: int, attributes_filter: Callable[[str], bool] | None) -> dict:- match = split_class_attributes.search(el_string)- tag_part = match.group(1) if match else ""- attrs_part = match.group(3) if match else None+ parts = _split_tag_and_attributes(el_string)+ tag_part = parts.tag+ attrs_part = parts.attributeselement: dict = {"text": None,@@ -144,15 +281,13 @@ def _parse_element(el_string: str, order: int, attributes_filter: Callable[[str]element["attr_class"] = [cl for cl in tag_and_class[1].split(".") if cl != ""]if attrs_part:- for attribute_match in parse_attributes_regex.finditer(attrs_part):- key = attribute_match.group("key")- value = attribute_match.group("value")- promoted = _PROMOTED_ATTRIBUTES.get(key)+ for attribute in iter_element_attributes(attrs_part):+ promoted = _PROMOTED_ATTRIBUTES.get(attribute.key)if promoted:field, convert = promoted- element[field] = convert(value)- elif key and (attributes_filter is None or attributes_filter(key)):- element["attributes"][key] = value+ element[field] = convert(attribute.value)+ elif attribute.key and (attributes_filter is None or attributes_filter(attribute.key)):+ element["attributes"][attribute.key] = attribute.valuereturn element@@ -165,8 +300,7 @@ def chain_to_element_dicts(chain: str, attributes_filter: Callable[[str], bool]map to matching keys (see build_attributes_filter)."""return [- _parse_element(el_string, idx, attributes_filter)- for idx, el_string in enumerate(split_chain_regex.findall(chain))+ _parse_element(el_string, idx, attributes_filter) for idx, el_string in enumerate(split_elements_chain(chain))]diff --git a/posthog/models/property/util.py b/posthog/models/property/util.pyindex 28a90d4b9c60..a2de4b366349 100644--- a/posthog/models/property/util.py+++ b/posthog/models/property/util.py@@ -150,7 +150,7 @@ def _chain_attribute_order(key: str) -> str:# A semicolon separates elements only outside a quoted attribute value — an inline# style="display: flex; gap: 4px" carries its own. Quotes inside a value are escaped# as \" (see _escape in posthog/models/element/element.py), so an escaped quote must-# not close the span. split_chain_regex draws the boundary the same way. A backslash+# not close the span. split_elements_chain draws the boundary the same way. A backslash# is consumed only together with the character after it, so a long run of backslashes# has one way to match and cannot cause catastrophic backtracking._QUOTED_VALUE = r'"(?:\\.|[^"\\])*"'diff --git a/posthog/test/test_element_model.py b/posthog/test/test_element_model.pyindex 04484af12805..0b1550b3c0ad 100644--- a/posthog/test/test_element_model.py+++ b/posthog/test/test_element_model.py@@ -8,7 +8,12 @@ from parameterized import parameterizedfrom posthog.api.element import ElementSerializerfrom posthog.models.element import Element, chain_to_elements, elements_to_string-from posthog.models.element.element import build_attributes_filter, chain_to_element_dicts+from posthog.models.element.element import (+ ElementAttribute,+ build_attributes_filter,+ chain_to_element_dicts,+ iter_element_attributes,+)class TestElement(ClickhouseTestMixin, SimpleTestCase):@@ -127,6 +132,71 @@ class TestElement(ClickhouseTestMixin, SimpleTestCase):assert element_dicts[0]["attributes"] == {}assert element_dicts[0]["nth_child"] == 0+ @parameterized.expand(+ [+ ("long backslash run", 'a:text="' + "\\" * 5000, ["a", "\\" * 5000]),+ ("quote at the end of the chain", 'button:nth-child="1"text="unclosed', ["button", "unclosed"]),+ ("long run of escaped quotes", '"\\' * 100_000, ["\\"] * 100_000),+ ("escaped quote with no value before it", 'a\\"', ["a\\"]),+ ]+ )+ def test_chain_to_element_dicts_ends_element_at_unterminated_quote(+ self, _name: str, chain: str, expected_tag_names: list[str]+ ) -> None:+ element_dicts = chain_to_element_dicts(chain)+ assert [element["tag_name"] for element in element_dicts] == expected_tag_names+ assert all(element["text"] is None for element in element_dicts)++ @parameterized.expand(+ [+ ("href", 'button:nth-child="2";a:href="C:\\"', ["button", "a"], {"href": None, "nth_child": None}),+ ("text", 'a:nth-child="1"text="ends with \\"', ["a"], {"text": None, "nth_child": 1}),+ ]+ )+ def test_chain_to_element_dicts_drops_a_value_that_ends_in_a_backslash_at_the_end_of_the_chain(+ self, _name: str, chain: str, expected_tag_names: list[str], expected_last: dict+ ) -> None:+ element_dicts = chain_to_element_dicts(chain)+ assert [element["tag_name"] for element in element_dicts] == expected_tag_names+ assert {field: element_dicts[-1][field] for field in expected_last} == expected_last++ def test_chain_to_element_dicts_finds_the_tag_past_a_newline_in_a_value(self) -> None:+ element_dicts = chain_to_element_dicts('text="' + "a" * 200_000 + '\nb:href="/a"')+ assert [element["tag_name"] for element in element_dicts] == ["b"]++ @parameterized.expand(+ [+ ("value never closes", 'href="/a"text="' + '\\"' * 2000, [ElementAttribute(key="href", value="/a")]),+ (+ "empty value",+ 'alt=""href="/a"',+ [ElementAttribute(key="alt", value=""), ElementAttribute(key="href", value="/a")],+ ),+ (+ "escaped quote inside value",+ 'text="say \\"hi\\""',+ [ElementAttribute(key="text", value='say \\"hi\\"')],+ ),+ ]+ )+ def test_iter_element_attributes(self, _name: str, attrs: str, expected: list[ElementAttribute]) -> None:+ assert list(iter_element_attributes(attrs)) == expected++ @parameterized.expand(+ [+ ("empty nth-child", 'a:nth-child=""nth-of-type="2"', {"nth_child": None, "nth_of_type": 2}),+ ("non-numeric nth-of-type", 'a:nth-child="1"nth-of-type="x"', {"nth_child": 1, "nth_of_type": None}),+ (+ "empty value before other attributes",+ 'a:attr__title=""href="/a-url"nth-child="1"',+ {"href": "/a-url", "nth_child": 1, "attributes": {"attr__title": ""}},+ ),+ ]+ )+ def test_chain_to_element_dicts_tolerates_malformed_values(self, _name: str, chain: str, expected: dict) -> None:+ element = chain_to_element_dicts(chain)[0]+ assert {field: element[field] for field in expected} == expected+def test_build_attributes_filter_caps_entry_count(self) -> None:many_attrs = [f"data-attr-{i}" for i in range(100)]matcher = build_attributes_filter(many_attrs)