:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(billing): read salesforce enrichment chunks from a redis list (#116397)
Matt Brooker276ecd48d7f3
diff --git a/ee/billing/salesforce_enrichment/constants.py b/ee/billing/salesforce_enrichment/constants.pyindex 2c1928781353..b81b8a05ffe7 100644--- a/ee/billing/salesforce_enrichment/constants.py+++ b/ee/billing/salesforce_enrichment/constants.py@@ -1,5 +1,5 @@REDIS_TTL_SECONDS: int = 12 * 60 * 60 # 12h-SALESFORCE_ACCOUNTS_CACHE_KEY: str = "salesforce-enrichment:global:all_accounts"+SALESFORCE_ACCOUNTS_CACHE_KEY: str = "salesforce-enrichment:global:accounts"SALESFORCE_ORG_MAPPINGS_CACHE_KEY: str = "salesforce-enrichment:global:org_mappings"SALESFORCE_STRIPE_ENRICHMENT_WATERMARK_KEY: str = "salesforce-enrichment:stripe:last_watermark"ORG_MAPPINGS_CACHE_MISSING_ERROR_TYPE: str = "OrgMappingsCacheMissing"diff --git a/ee/billing/salesforce_enrichment/enrichment.py b/ee/billing/salesforce_enrichment/enrichment.pyindex 63f256255888..c7792985e39f 100644--- a/ee/billing/salesforce_enrichment/enrichment.py+++ b/ee/billing/salesforce_enrichment/enrichment.py@@ -23,12 +23,13 @@ from .constants import (HARMONIC_BATCH_SIZE,METRIC_PERIODS,PERSONAL_EMAIL_DOMAINS,+ SALESFORCE_ACCOUNTS_QUERY,SALESFORCE_UPDATE_BATCH_SIZE,YC_INVESTOR_NAME,)from .harmonic_client import AsyncHarmonicClientfrom .harmonic_company import is_sub_entity-from .redis_cache import get_accounts_from_redis+from .redis_cache import get_accounts_from_redis, store_accounts_in_redisfrom .salesforce_client import get_salesforce_clientif TYPE_CHECKING:@@ -550,6 +551,7 @@ async def query_salesforce_accounts_chunk_async(sf, offset=0, limit=5000):logger = LOGGER.bind(function="query_salesforce_accounts_chunk_async", offset=offset, limit=limit)# Try Redis cache first+ cache_missing = Falsecache_start = time.time()try:cached_accounts = await get_accounts_from_redis(offset, limit)@@ -558,36 +560,29 @@ async def query_salesforce_accounts_chunk_async(sf, offset=0, limit=5000):if cached_accounts is not None:logger.info("Redis cache hit", accounts_count=len(cached_accounts), cache_time=round(cache_time, 3))return cached_accounts+ cache_missing = Trueexcept Exception as e:cache_time = time.time() - cache_startlogger.exception("Redis cache error", error=str(e), cache_time=round(cache_time, 3))capture_exception(e)- # Fallback to Salesforce query- query = """- SELECT Id, Name, Website, CreatedDate- FROM Account- WHERE Website != null- ORDER BY CreatedDate DESC- """-try:- accounts = sf.query_all(query)-- start_idx = offset- end_idx = min(offset + limit, len(accounts["records"]))-- if start_idx >= len(accounts["records"]):- return []-- chunk_records = accounts["records"][start_idx:end_idx]-- return chunk_records+ accounts_result = await asyncio.to_thread(sf.query_all, SALESFORCE_ACCOUNTS_QUERY)+ all_accounts = accounts_result["records"]except Exception as e:logger.exception("Salesforce query failed", error=str(e))capture_exception(e)return []+ if cache_missing:+ try:+ await store_accounts_in_redis(all_accounts)+ except Exception as e:+ logger.exception("Failed to rebuild the accounts cache", error=str(e))+ capture_exception(e)++ return all_accounts[offset : offset + limit]+def _build_result(chunk_number: int,diff --git a/ee/billing/salesforce_enrichment/redis_cache.py b/ee/billing/salesforce_enrichment/redis_cache.pyindex 25ebd507b3a0..1f030716ed67 100644--- a/ee/billing/salesforce_enrichment/redis_cache.py+++ b/ee/billing/salesforce_enrichment/redis_cache.py@@ -1,6 +1,4 @@-import gzipimport json-import asyncioimport datetime as dtfrom typing import Any@@ -14,36 +12,44 @@ from .constants import (SALESFORCE_STRIPE_ENRICHMENT_WATERMARK_KEY,)+_LIST_PIPELINE_BATCH = 5000 # Entries per RPUSH call during list population+_CHUNK_ACCOUNT_FIELDS = ("Id", "Website")-def _compress_redis_data(input_data: str) -> bytes:- return gzip.compress(input_data.encode("utf-8"))+async def _store_json_list(cache_key: str, items: list[dict[str, Any]], ttl: int) -> None:+ """Each item is its own list entry, so a reader fetches one page with LRANGE instead of parsing the whole dataset."""+ redis_client = get_async_client()+ pipe = await redis_client.pipeline()+ pipe.delete(cache_key)-def _decompress_and_parse_redis_data(raw_redis_data: bytes) -> list[dict[str, Any]]:- parsed = json.loads(gzip.decompress(raw_redis_data).decode("utf-8"))- if not isinstance(parsed, list):- raise TypeError(f"Expected list from Redis cache, got {type(parsed).__name__}")- return parsed+ for i in range(0, len(items), _LIST_PIPELINE_BATCH):+ batch = items[i : i + _LIST_PIPELINE_BATCH]+ pipe.rpush(cache_key, *[json.dumps(item, default=str) for item in batch])+ pipe.expire(cache_key, ttl)+ await pipe.execute()-async def _get_cached_list(cache_key: str, timeout: float = 30.0) -> list[dict[str, Any]] | None:- """Retrieve and decompress a cached list from Redis.- Returns:- Parsed list, or None if cache miss/error/timeout- """- try:- redis_client = get_async_client()- raw_redis_data = await asyncio.wait_for(redis_client.get(cache_key), timeout=timeout)+async def _get_list_page(cache_key: str, offset: int, limit: int) -> list[Any] | None:+ """Return the raw entries of one page, an empty list past the end, or None when the key is missing."""+ redis_client = get_async_client()+ raw_items = await redis_client.lrange(cache_key, offset, offset + limit - 1)+ if raw_items:+ return raw_items- if not raw_redis_data:- return None+ # LRANGE returns nothing both when the key is missing and when the offset is+ # past the end of the list; LLEN tells those cases apart.+ total = await redis_client.llen(cache_key)+ if total and offset >= total:+ return []+ return None- return _decompress_and_parse_redis_data(raw_redis_data)- except TimeoutError:- capture_exception(TimeoutError(f"Redis operation timed out after {timeout}s for key {cache_key}"))- return None+async def _get_list_length(cache_key: str) -> int | None:+ try:+ redis_client = get_async_client()+ count = await redis_client.llen(cache_key)+ return count if count > 0 else Noneexcept Exception as e:capture_exception(e)return None@@ -53,70 +59,27 @@ async def store_accounts_in_redis(accounts_data: list[dict[str, Any]],ttl: int = REDIS_TTL_SECONDS,) -> None:- """Store all Salesforce accounts in Redis with gzip compression."""- redis_client = get_async_client()-- accounts_json = json.dumps(accounts_data, default=str)- compressed_data = _compress_redis_data(accounts_json)-- await redis_client.setex(SALESFORCE_ACCOUNTS_CACHE_KEY, ttl, compressed_data)+ accounts = [{field: account[field] for field in _CHUNK_ACCOUNT_FIELDS} for account in accounts_data]+ await _store_json_list(SALESFORCE_ACCOUNTS_CACHE_KEY, accounts, ttl)-async def get_accounts_from_redis(- offset: int = 0,- limit: int = 1000,-) -> list[dict[str, Any]] | None:- """Retrieve accounts chunk from global Redis cache.-- Args:- offset: Starting index for pagination- limit: Number of accounts to retrieve-- Returns:- List of account dictionaries, or None if cache miss/error- """- all_accounts = await _get_cached_list(SALESFORCE_ACCOUNTS_CACHE_KEY)- if all_accounts is None:+async def get_accounts_from_redis(offset: int = 0, limit: int = 1000) -> list[dict[str, Any]] | None:+ """Return one page of cached accounts, an empty list past the end, or None when the cache is missing."""+ raw_items = await _get_list_page(SALESFORCE_ACCOUNTS_CACHE_KEY, offset, limit)+ if raw_items is None:return None-- if offset >= len(all_accounts):- return []-- return all_accounts[offset : offset + limit]+ return [json.loads(item) for item in raw_items]async def get_cached_accounts_count() -> int | None:- """Get the total count of cached accounts.-- Returns:- Total number of cached accounts, or None if cache miss/error- """- all_accounts = await _get_cached_list(SALESFORCE_ACCOUNTS_CACHE_KEY)- return len(all_accounts) if all_accounts is not None else None---_ORG_MAPPINGS_PIPELINE_BATCH = 5000 # Entries per RPUSH call during list population+ return await _get_list_length(SALESFORCE_ACCOUNTS_CACHE_KEY)async def store_org_mappings_in_redis(mappings_data: list[dict[str, Any]],ttl: int = REDIS_TTL_SECONDS,) -> None:- """Store Salesforce org mappings as a Redis List for efficient pagination.-- Each mapping is stored as a separate JSON entry so LRANGE can return- arbitrary pages without decompressing the entire dataset.- """- redis_client = get_async_client()- pipe = await redis_client.pipeline()- pipe.delete(SALESFORCE_ORG_MAPPINGS_CACHE_KEY)-- for i in range(0, len(mappings_data), _ORG_MAPPINGS_PIPELINE_BATCH):- batch = mappings_data[i : i + _ORG_MAPPINGS_PIPELINE_BATCH]- pipe.rpush(SALESFORCE_ORG_MAPPINGS_CACHE_KEY, *[json.dumps(m, default=str) for m in batch])-- pipe.expire(SALESFORCE_ORG_MAPPINGS_CACHE_KEY, ttl)- await pipe.execute()+ await _store_json_list(SALESFORCE_ORG_MAPPINGS_CACHE_KEY, mappings_data, ttl)async def _lrange_json_or_none(key: str, start: int, end: int) -> list[dict[str, Any]] | None:@@ -152,20 +115,13 @@ async def get_org_mappings_page(offset: int = 0, limit: int = 10000) -> list[dicwhen the key is missing or an entry cannot be parsed, so callers nevermistake an expired cache for a completed run."""- redis_client = get_async_client()- raw_items = await redis_client.lrange(SALESFORCE_ORG_MAPPINGS_CACHE_KEY, offset, offset + limit - 1)- if raw_items:- try:- return [json.loads(item) for item in raw_items]- except (TypeError, ValueError) as e:- raise OrgMappingsCacheMissingError("org mappings cache entries are unreadable") from e-- # LRANGE returns nothing both when the key is missing and when the offset is- # past the end of the list; LLEN tells those cases apart.- total = await redis_client.llen(SALESFORCE_ORG_MAPPINGS_CACHE_KEY)- if total and offset >= total:- return []- raise OrgMappingsCacheMissingError("org mappings cache key is missing")+ raw_items = await _get_list_page(SALESFORCE_ORG_MAPPINGS_CACHE_KEY, offset, limit)+ if raw_items is None:+ raise OrgMappingsCacheMissingError("org mappings cache key is missing")+ try:+ return [json.loads(item) for item in raw_items]+ except (TypeError, ValueError) as e:+ raise OrgMappingsCacheMissingError("org mappings cache entries are unreadable") from easync def get_org_mappings_from_redis() -> list[dict[str, Any]] | None:@@ -183,13 +139,7 @@ async def get_cached_org_mappings_count() -> int | None:Returns:Total number of cached mappings, or None if key missing"""- try:- redis_client = get_async_client()- count = await redis_client.llen(SALESFORCE_ORG_MAPPINGS_CACHE_KEY)- return count if count > 0 else None- except Exception as e:- capture_exception(e)- return None+ return await _get_list_length(SALESFORCE_ORG_MAPPINGS_CACHE_KEY)async def get_stripe_enrichment_watermark() -> tuple[dt.datetime, str] | None:diff --git a/ee/billing/test/test_redis_cache.py b/ee/billing/test/test_redis_cache.pyindex 764234311cb5..a65c95832774 100644--- a/ee/billing/test/test_redis_cache.py+++ b/ee/billing/test/test_redis_cache.py@@ -6,7 +6,6 @@ from unittest.mock import AsyncMock, patchfrom ee.billing.salesforce_enrichment.redis_cache import (OrgMappingsCacheMissingError,- _compress_redis_data,get_cached_org_mappings_count,get_org_mappings_from_redis,get_org_mappings_page,@@ -16,25 +15,6 @@ from ee.billing.salesforce_enrichment.redis_cache import ()-class TestCompressionHelpers:- def test_compression_produces_bytes(self):- original_data = '{"test": "data"}'-- compressed = _compress_redis_data(original_data)-- assert isinstance(compressed, bytes)-- def test_compression_round_trip(self):- import gzip-- original_data = '{"key": "value", "list": [1, 2, 3]}'-- compressed = _compress_redis_data(original_data)- decompressed = gzip.decompress(compressed).decode("utf-8")-- assert decompressed == original_data--class TestStoreOrgMappingsInRedis:@pytest.mark.asyncioasync def test_store_org_mappings_success(self):diff --git a/ee/billing/test/test_salesforce_enrichment.py b/ee/billing/test/test_salesforce_enrichment.pyindex 78cd151b641a..11b8a67af464 100644--- a/ee/billing/test/test_salesforce_enrichment.py+++ b/ee/billing/test/test_salesforce_enrichment.py@@ -10,7 +10,9 @@ from unittest.mock import AsyncMock, MagicMock, patchfrom django.test import SimpleTestCase+import fakeredisfrom parameterized import parameterized+from redis.exceptions import ConnectionError as RedisConnectionErrorfrom ee.billing.salesforce_enrichment.constants import (ACCOUNT_FILL_FIELDS,@@ -29,8 +31,10 @@ from ee.billing.salesforce_enrichment.enrichment import (prepare_account_update,prepare_mirror_clearing_data,prepare_salesforce_update_data,+ query_salesforce_accounts_chunk_async,transform_harmonic_data,)+from ee.billing.salesforce_enrichment.redis_cache import get_accounts_from_redis, store_accounts_in_redis@contextmanager@@ -679,6 +683,38 @@ class TestSalesforceAccountQuery(SimpleTestCase):# Verify it doesn't use the old %domain% pattern that would match unintended domainsassert "LIKE '%example.com%'" not in actual_query+ @pytest.mark.asyncio+ async def test_chunk_reads_rebuild_a_missing_cache_once_and_page_through_it(self) -> None:+ accounts = [{"Id": f"001SYN{i:03d}", "Website": f"https://company{i}.example.com"} for i in range(7)]+ sf = MagicMock()+ sf.query_all.return_value = {"records": accounts}++ with patch(+ "ee.billing.salesforce_enrichment.redis_cache.get_async_client",+ return_value=fakeredis.FakeAsyncRedis(server=fakeredis.FakeServer()),+ ):+ pages = [await query_salesforce_accounts_chunk_async(sf, offset, 3) for offset in (0, 3, 6, 9)]++ assert pages == [accounts[0:3], accounts[3:6], accounts[6:7], []]+ sf.query_all.assert_called_once()++ @pytest.mark.asyncio+ async def test_chunk_read_error_preserves_cached_accounts(self) -> None:+ accounts = [{"Id": f"001SYN{i:03d}", "Website": f"https://company{i}.example.com"} for i in range(3)]+ cached_accounts = list(reversed(accounts))+ sf = MagicMock()+ sf.query_all.return_value = {"records": accounts}+ redis_client = fakeredis.FakeAsyncRedis(server=fakeredis.FakeServer())++ with patch("ee.billing.salesforce_enrichment.redis_cache.get_async_client", return_value=redis_client):+ await store_accounts_in_redis(cached_accounts)+ with patch.object(redis_client, "lrange", side_effect=RedisConnectionError("Synthetic read failure")):+ page = await query_salesforce_accounts_chunk_async(sf, 0, 2)+ remaining_accounts = await get_accounts_from_redis(0, 3)++ assert page == accounts[:2]+ assert remaining_accounts == cached_accounts+class TestSpecificDomainEnrichment(SimpleTestCase):@pytest.mark.asyncio