Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(hubspot): lower property URL cap to avoid 431 errors (#124892)

Tom Owers020e24b8beb6
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hubspot/hubspot.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hubspot/hubspot.py
index 99f342d2d8ae..8a5be7fc690a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hubspot/hubspot.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hubspot/hubspot.py
@@ -15,6 +15,7 @@ Two fetch paths:
Backfills associations per page via v4 batch-read.
"""
+import re
import dataclasses
import urllib.parse
from collections.abc import Iterator, Sequence
@@ -58,7 +59,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.hubspot.se
apply_crm_api_version,
)
-PROPERTY_LENGTH_LIMIT = 16_000 # Empirically determined rough limit for the HubSpot API
+# HubSpot's practical URL ceiling sits around 16_300 chars, and this bounds only the properties
+# value, not the rest of the URL (base, path, limit, associations) — nor the fact that HubSpot's
+# own paginated `next` link re-serializes `associations` as repeated query keys instead of the
+# compact comma-joined form the initial request sends, adding bytes on later pages. 16_000 left too
+# little margin and produced a production 431 (Request Header Fields Too Large) on a resumed page.
+PROPERTY_LENGTH_LIMIT = 10_000
# Cap on the number of properties requested via the search path. The search API has no URL-length
# concern (POST body), but each extra property multiplies backfill work and response payload size.
SEARCH_PROPERTIES_LIMIT = 250
@@ -200,6 +206,24 @@ def _build_initial_url(path: str, associations: list[str], properties: str, limi
return f"{BASE_URL.rstrip('/')}{path}?{'&'.join(parts)}"
+_PROPERTIES_PARAM_RE = re.compile(r"(?<=[?&])properties=[^&]*")
+
+
+def _rebuild_resume_url_with_current_properties(next_url: str, props_str: str) -> str:
+ """Replace a saved resume URL's `properties` value with the one built under today's cap.
+
+ A URL saved by an older sync can carry a `properties` value sized against a since-lowered
+ `PROPERTY_LENGTH_LIMIT`; HubSpot's 431 doesn't care when the URL was built, only how large it
+ is now. Everything else in `next_url` — including HubSpot's opaque pagination cursor — is
+ preserved untouched.
+ """
+ replacement = f"properties={props_str}"
+ if _PROPERTIES_PARAM_RE.search(next_url):
+ return _PROPERTIES_PARAM_RE.sub(replacement, next_url, count=1)
+ separator = "&" if "?" in next_url else "?"
+ return f"{next_url}{separator}{replacement}"
+
+
def _backfill_missing_properties(row: dict[str, Any], expected_properties: list[str]) -> None:
"""HubSpot omits properties with null values; PyArrow drops absent columns during schema inference."""
for prop in expected_properties:
@@ -333,7 +357,7 @@ def get_rows(
resume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
if resume_config is not None and resume_config.next_url is not None:
- url = resume_config.next_url
+ url = _rebuild_resume_url_with_current_properties(resume_config.next_url, props_str)
logger.debug(f"Hubspot: resuming from URL: {url}")
else:
url = _build_initial_url(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hubspot/test/test_hubspot_search.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hubspot/test/test_hubspot_search.py
index a36867cd34d1..03369eb14670 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hubspot/test/test_hubspot_search.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hubspot/test/test_hubspot_search.py
@@ -1,3 +1,4 @@
+import urllib.parse
from collections.abc import Callable
from dataclasses import replace
from datetime import UTC, datetime
@@ -9,9 +10,11 @@ from unittest.mock import MagicMock, patch
from requests.exceptions import JSONDecodeError as RequestsJSONDecodeError
from products.warehouse_sources.backend.temporal.data_imports.sources.hubspot.hubspot import (
+ PROPERTY_LENGTH_LIMIT,
HubspotResumeConfig,
_backfill_associations_into_results,
_batch_read_associations,
+ _get_properties_str,
_iso_to_ms,
_resolve_search_properties,
get_rows_via_search,
@@ -153,6 +156,31 @@ class TestResolveSearchProperties:
logger.warning.assert_called()
+class TestGetPropertiesStr:
+ def test_truncates_with_headroom_below_hubspots_url_ceiling(self) -> None:
+ # Regression: HubSpot's own paginated `next` link re-serializes `associations` as
+ # repeated query keys instead of the compact comma-joined form the initial request
+ # sends, adding bytes on a resumed page. A cap that only just fit the initial URL (the
+ # old 16_000) left no room for that and produced a production 431 (Request Header
+ # Fields Too Large). The properties value alone must leave real headroom, not just stay
+ # under HubSpot's ~16_300-char ceiling.
+ many_props = [f"custom_property_name_{i}" for i in range(2000)]
+ logger = MagicMock()
+
+ props_str = _get_properties_str(
+ props=many_props,
+ api_key="k",
+ refresh_token="r",
+ object_type="contact",
+ logger=logger,
+ include_custom_props=False,
+ )
+
+ assert len(urllib.parse.quote(props_str)) <= 12_000
+ assert len(props_str.split(",")) < len(many_props)
+ logger.warning.assert_called_once()
+
+
class TestBatchReadAssociations:
def test_empty_ids_returns_empty(self) -> None:
result = _batch_read_associations(
@@ -722,10 +750,19 @@ class TestGetRowsFullRefresh:
assert len(captured_urls) == 2
assert captured_urls[1] == "https://api.hubapi.com/page2"
- def test_resume_from_next_url(self) -> None:
+ def test_resume_replaces_oversized_properties_but_keeps_cursor(self) -> None:
+ # Regression: a URL saved by an older sync (or before a PROPERTY_LENGTH_LIMIT reduction)
+ # can carry a `properties` value too large for the current cap. HubSpot's 431 doesn't
+ # care when the URL was built, only how large it is now, so resuming must rebuild the
+ # properties query rather than replaying the saved one verbatim — while still resuming
+ # from (not restarting past) HubSpot's opaque pagination cursor.
from products.warehouse_sources.backend.temporal.data_imports.sources.hubspot.hubspot import get_rows
- resume = HubspotResumeConfig(next_url="https://api.hubapi.com/resume-here")
+ oversized_properties = ",".join(f"custom_property_name_{i}" for i in range(2000))
+ stale_url = (
+ f"https://api.hubapi.com/crm/v3/objects/deals?properties={oversized_properties}&limit=100&after=NTI1cGFnZTI"
+ )
+ resume = HubspotResumeConfig(next_url=stale_url)
manager = _make_manager(can_resume=True, resume_state=resume)
logger = MagicMock()
@@ -751,7 +788,11 @@ class TestGetRowsFullRefresh:
)
)
- assert captured_urls[0] == "https://api.hubapi.com/resume-here"
+ requested_url = captured_urls[0]
+ assert "after=NTI1cGFnZTI" in requested_url
+ properties_value = requested_url.split("properties=")[1].split("&")[0]
+ assert properties_value != oversized_properties
+ assert len(urllib.parse.quote(properties_value)) <= PROPERTY_LENGTH_LIMIT
def test_truncated_json_body_is_retried(self) -> None:
# Regression: a truncated/partial page body raises requests' JSONDecodeError