:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(hubspot): lower property URL cap to avoid 431 errors (#124892)
Tom Owers020e24b8beb6
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hubspot/hubspot.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hubspot/hubspot.pyindex 99f342d2d8ae..8a5be7fc690a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hubspot/hubspot.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hubspot/hubspot.py@@ -15,6 +15,7 @@ Two fetch paths:Backfills associations per page via v4 batch-read."""+import reimport dataclassesimport urllib.parsefrom collections.abc import Iterator, Sequence@@ -58,7 +59,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.hubspot.seapply_crm_api_version,)-PROPERTY_LENGTH_LIMIT = 16_000 # Empirically determined rough limit for the HubSpot API+# HubSpot's practical URL ceiling sits around 16_300 chars, and this bounds only the properties+# value, not the rest of the URL (base, path, limit, associations) — nor the fact that HubSpot's+# own paginated `next` link re-serializes `associations` as repeated query keys instead of the+# compact comma-joined form the initial request sends, adding bytes on later pages. 16_000 left too+# little margin and produced a production 431 (Request Header Fields Too Large) on a resumed page.+PROPERTY_LENGTH_LIMIT = 10_000# Cap on the number of properties requested via the search path. The search API has no URL-length# concern (POST body), but each extra property multiplies backfill work and response payload size.SEARCH_PROPERTIES_LIMIT = 250@@ -200,6 +206,24 @@ def _build_initial_url(path: str, associations: list[str], properties: str, limireturn f"{BASE_URL.rstrip('/')}{path}?{'&'.join(parts)}"+_PROPERTIES_PARAM_RE = re.compile(r"(?<=[?&])properties=[^&]*")+++def _rebuild_resume_url_with_current_properties(next_url: str, props_str: str) -> str:+ """Replace a saved resume URL's `properties` value with the one built under today's cap.++ A URL saved by an older sync can carry a `properties` value sized against a since-lowered+ `PROPERTY_LENGTH_LIMIT`; HubSpot's 431 doesn't care when the URL was built, only how large it+ is now. Everything else in `next_url` — including HubSpot's opaque pagination cursor — is+ preserved untouched.+ """+ replacement = f"properties={props_str}"+ if _PROPERTIES_PARAM_RE.search(next_url):+ return _PROPERTIES_PARAM_RE.sub(replacement, next_url, count=1)+ separator = "&" if "?" in next_url else "?"+ return f"{next_url}{separator}{replacement}"++def _backfill_missing_properties(row: dict[str, Any], expected_properties: list[str]) -> None:"""HubSpot omits properties with null values; PyArrow drops absent columns during schema inference."""for prop in expected_properties:@@ -333,7 +357,7 @@ def get_rows(resume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else Noneif resume_config is not None and resume_config.next_url is not None:- url = resume_config.next_url+ url = _rebuild_resume_url_with_current_properties(resume_config.next_url, props_str)logger.debug(f"Hubspot: resuming from URL: {url}")else:url = _build_initial_url(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hubspot/test/test_hubspot_search.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hubspot/test/test_hubspot_search.pyindex a36867cd34d1..03369eb14670 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hubspot/test/test_hubspot_search.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hubspot/test/test_hubspot_search.py@@ -1,3 +1,4 @@+import urllib.parsefrom collections.abc import Callablefrom dataclasses import replacefrom datetime import UTC, datetime@@ -9,9 +10,11 @@ from unittest.mock import MagicMock, patchfrom requests.exceptions import JSONDecodeError as RequestsJSONDecodeErrorfrom products.warehouse_sources.backend.temporal.data_imports.sources.hubspot.hubspot import (+ PROPERTY_LENGTH_LIMIT,HubspotResumeConfig,_backfill_associations_into_results,_batch_read_associations,+ _get_properties_str,_iso_to_ms,_resolve_search_properties,get_rows_via_search,@@ -153,6 +156,31 @@ class TestResolveSearchProperties:logger.warning.assert_called()+class TestGetPropertiesStr:+ def test_truncates_with_headroom_below_hubspots_url_ceiling(self) -> None:+ # Regression: HubSpot's own paginated `next` link re-serializes `associations` as+ # repeated query keys instead of the compact comma-joined form the initial request+ # sends, adding bytes on a resumed page. A cap that only just fit the initial URL (the+ # old 16_000) left no room for that and produced a production 431 (Request Header+ # Fields Too Large). The properties value alone must leave real headroom, not just stay+ # under HubSpot's ~16_300-char ceiling.+ many_props = [f"custom_property_name_{i}" for i in range(2000)]+ logger = MagicMock()++ props_str = _get_properties_str(+ props=many_props,+ api_key="k",+ refresh_token="r",+ object_type="contact",+ logger=logger,+ include_custom_props=False,+ )++ assert len(urllib.parse.quote(props_str)) <= 12_000+ assert len(props_str.split(",")) < len(many_props)+ logger.warning.assert_called_once()++class TestBatchReadAssociations:def test_empty_ids_returns_empty(self) -> None:result = _batch_read_associations(@@ -722,10 +750,19 @@ class TestGetRowsFullRefresh:assert len(captured_urls) == 2assert captured_urls[1] == "https://api.hubapi.com/page2"- def test_resume_from_next_url(self) -> None:+ def test_resume_replaces_oversized_properties_but_keeps_cursor(self) -> None:+ # Regression: a URL saved by an older sync (or before a PROPERTY_LENGTH_LIMIT reduction)+ # can carry a `properties` value too large for the current cap. HubSpot's 431 doesn't+ # care when the URL was built, only how large it is now, so resuming must rebuild the+ # properties query rather than replaying the saved one verbatim — while still resuming+ # from (not restarting past) HubSpot's opaque pagination cursor.from products.warehouse_sources.backend.temporal.data_imports.sources.hubspot.hubspot import get_rows- resume = HubspotResumeConfig(next_url="https://api.hubapi.com/resume-here")+ oversized_properties = ",".join(f"custom_property_name_{i}" for i in range(2000))+ stale_url = (+ f"https://api.hubapi.com/crm/v3/objects/deals?properties={oversized_properties}&limit=100&after=NTI1cGFnZTI"+ )+ resume = HubspotResumeConfig(next_url=stale_url)manager = _make_manager(can_resume=True, resume_state=resume)logger = MagicMock()@@ -751,7 +788,11 @@ class TestGetRowsFullRefresh:))- assert captured_urls[0] == "https://api.hubapi.com/resume-here"+ requested_url = captured_urls[0]+ assert "after=NTI1cGFnZTI" in requested_url+ properties_value = requested_url.split("properties=")[1].split("&")[0]+ assert properties_value != oversized_properties+ assert len(urllib.parse.quote(properties_value)) <= PROPERTY_LENGTH_LIMITdef test_truncated_json_body_is_retried(self) -> None:# Regression: a truncated/partial page body raises requests' JSONDecodeError