Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(billing): read salesforce enrichment chunks from a redis list (#116397)

Matt Brooker276ecd48d7f3
diff --git a/ee/billing/salesforce_enrichment/constants.py b/ee/billing/salesforce_enrichment/constants.py
index 2c1928781353..b81b8a05ffe7 100644
--- a/ee/billing/salesforce_enrichment/constants.py
+++ b/ee/billing/salesforce_enrichment/constants.py
@@ -1,5 +1,5 @@
REDIS_TTL_SECONDS: int = 12 * 60 * 60 # 12h
-SALESFORCE_ACCOUNTS_CACHE_KEY: str = "salesforce-enrichment:global:all_accounts"
+SALESFORCE_ACCOUNTS_CACHE_KEY: str = "salesforce-enrichment:global:accounts"
SALESFORCE_ORG_MAPPINGS_CACHE_KEY: str = "salesforce-enrichment:global:org_mappings"
SALESFORCE_STRIPE_ENRICHMENT_WATERMARK_KEY: str = "salesforce-enrichment:stripe:last_watermark"
ORG_MAPPINGS_CACHE_MISSING_ERROR_TYPE: str = "OrgMappingsCacheMissing"
diff --git a/ee/billing/salesforce_enrichment/enrichment.py b/ee/billing/salesforce_enrichment/enrichment.py
index 63f256255888..c7792985e39f 100644
--- a/ee/billing/salesforce_enrichment/enrichment.py
+++ b/ee/billing/salesforce_enrichment/enrichment.py
@@ -23,12 +23,13 @@ from .constants import (
HARMONIC_BATCH_SIZE,
METRIC_PERIODS,
PERSONAL_EMAIL_DOMAINS,
+ SALESFORCE_ACCOUNTS_QUERY,
SALESFORCE_UPDATE_BATCH_SIZE,
YC_INVESTOR_NAME,
)
from .harmonic_client import AsyncHarmonicClient
from .harmonic_company import is_sub_entity
-from .redis_cache import get_accounts_from_redis
+from .redis_cache import get_accounts_from_redis, store_accounts_in_redis
from .salesforce_client import get_salesforce_client
if TYPE_CHECKING:
@@ -550,6 +551,7 @@ async def query_salesforce_accounts_chunk_async(sf, offset=0, limit=5000):
logger = LOGGER.bind(function="query_salesforce_accounts_chunk_async", offset=offset, limit=limit)
# Try Redis cache first
+ cache_missing = False
cache_start = time.time()
try:
cached_accounts = await get_accounts_from_redis(offset, limit)
@@ -558,36 +560,29 @@ async def query_salesforce_accounts_chunk_async(sf, offset=0, limit=5000):
if cached_accounts is not None:
logger.info("Redis cache hit", accounts_count=len(cached_accounts), cache_time=round(cache_time, 3))
return cached_accounts
+ cache_missing = True
except Exception as e:
cache_time = time.time() - cache_start
logger.exception("Redis cache error", error=str(e), cache_time=round(cache_time, 3))
capture_exception(e)
- # Fallback to Salesforce query
- query = """
- SELECT Id, Name, Website, CreatedDate
- FROM Account
- WHERE Website != null
- ORDER BY CreatedDate DESC
- """
-
try:
- accounts = sf.query_all(query)
-
- start_idx = offset
- end_idx = min(offset + limit, len(accounts["records"]))
-
- if start_idx >= len(accounts["records"]):
- return []
-
- chunk_records = accounts["records"][start_idx:end_idx]
-
- return chunk_records
+ accounts_result = await asyncio.to_thread(sf.query_all, SALESFORCE_ACCOUNTS_QUERY)
+ all_accounts = accounts_result["records"]
except Exception as e:
logger.exception("Salesforce query failed", error=str(e))
capture_exception(e)
return []
+ if cache_missing:
+ try:
+ await store_accounts_in_redis(all_accounts)
+ except Exception as e:
+ logger.exception("Failed to rebuild the accounts cache", error=str(e))
+ capture_exception(e)
+
+ return all_accounts[offset : offset + limit]
+
def _build_result(
chunk_number: int,
diff --git a/ee/billing/salesforce_enrichment/redis_cache.py b/ee/billing/salesforce_enrichment/redis_cache.py
index 25ebd507b3a0..1f030716ed67 100644
--- a/ee/billing/salesforce_enrichment/redis_cache.py
+++ b/ee/billing/salesforce_enrichment/redis_cache.py
@@ -1,6 +1,4 @@
-import gzip
import json
-import asyncio
import datetime as dt
from typing import Any
@@ -14,36 +12,44 @@ from .constants import (
SALESFORCE_STRIPE_ENRICHMENT_WATERMARK_KEY,
)
+_LIST_PIPELINE_BATCH = 5000 # Entries per RPUSH call during list population
+_CHUNK_ACCOUNT_FIELDS = ("Id", "Website")
-def _compress_redis_data(input_data: str) -> bytes:
- return gzip.compress(input_data.encode("utf-8"))
+async def _store_json_list(cache_key: str, items: list[dict[str, Any]], ttl: int) -> None:
+ """Each item is its own list entry, so a reader fetches one page with LRANGE instead of parsing the whole dataset."""
+ redis_client = get_async_client()
+ pipe = await redis_client.pipeline()
+ pipe.delete(cache_key)
-def _decompress_and_parse_redis_data(raw_redis_data: bytes) -> list[dict[str, Any]]:
- parsed = json.loads(gzip.decompress(raw_redis_data).decode("utf-8"))
- if not isinstance(parsed, list):
- raise TypeError(f"Expected list from Redis cache, got {type(parsed).__name__}")
- return parsed
+ for i in range(0, len(items), _LIST_PIPELINE_BATCH):
+ batch = items[i : i + _LIST_PIPELINE_BATCH]
+ pipe.rpush(cache_key, *[json.dumps(item, default=str) for item in batch])
+ pipe.expire(cache_key, ttl)
+ await pipe.execute()
-async def _get_cached_list(cache_key: str, timeout: float = 30.0) -> list[dict[str, Any]] | None:
- """Retrieve and decompress a cached list from Redis.
- Returns:
- Parsed list, or None if cache miss/error/timeout
- """
- try:
- redis_client = get_async_client()
- raw_redis_data = await asyncio.wait_for(redis_client.get(cache_key), timeout=timeout)
+async def _get_list_page(cache_key: str, offset: int, limit: int) -> list[Any] | None:
+ """Return the raw entries of one page, an empty list past the end, or None when the key is missing."""
+ redis_client = get_async_client()
+ raw_items = await redis_client.lrange(cache_key, offset, offset + limit - 1)
+ if raw_items:
+ return raw_items
- if not raw_redis_data:
- return None
+ # LRANGE returns nothing both when the key is missing and when the offset is
+ # past the end of the list; LLEN tells those cases apart.
+ total = await redis_client.llen(cache_key)
+ if total and offset >= total:
+ return []
+ return None
- return _decompress_and_parse_redis_data(raw_redis_data)
- except TimeoutError:
- capture_exception(TimeoutError(f"Redis operation timed out after {timeout}s for key {cache_key}"))
- return None
+async def _get_list_length(cache_key: str) -> int | None:
+ try:
+ redis_client = get_async_client()
+ count = await redis_client.llen(cache_key)
+ return count if count > 0 else None
except Exception as e:
capture_exception(e)
return None
@@ -53,70 +59,27 @@ async def store_accounts_in_redis(
accounts_data: list[dict[str, Any]],
ttl: int = REDIS_TTL_SECONDS,
) -> None:
- """Store all Salesforce accounts in Redis with gzip compression."""
- redis_client = get_async_client()
-
- accounts_json = json.dumps(accounts_data, default=str)
- compressed_data = _compress_redis_data(accounts_json)
-
- await redis_client.setex(SALESFORCE_ACCOUNTS_CACHE_KEY, ttl, compressed_data)
+ accounts = [{field: account[field] for field in _CHUNK_ACCOUNT_FIELDS} for account in accounts_data]
+ await _store_json_list(SALESFORCE_ACCOUNTS_CACHE_KEY, accounts, ttl)
-async def get_accounts_from_redis(
- offset: int = 0,
- limit: int = 1000,
-) -> list[dict[str, Any]] | None:
- """Retrieve accounts chunk from global Redis cache.
-
- Args:
- offset: Starting index for pagination
- limit: Number of accounts to retrieve
-
- Returns:
- List of account dictionaries, or None if cache miss/error
- """
- all_accounts = await _get_cached_list(SALESFORCE_ACCOUNTS_CACHE_KEY)
- if all_accounts is None:
+async def get_accounts_from_redis(offset: int = 0, limit: int = 1000) -> list[dict[str, Any]] | None:
+ """Return one page of cached accounts, an empty list past the end, or None when the cache is missing."""
+ raw_items = await _get_list_page(SALESFORCE_ACCOUNTS_CACHE_KEY, offset, limit)
+ if raw_items is None:
return None
-
- if offset >= len(all_accounts):
- return []
-
- return all_accounts[offset : offset + limit]
+ return [json.loads(item) for item in raw_items]
async def get_cached_accounts_count() -> int | None:
- """Get the total count of cached accounts.
-
- Returns:
- Total number of cached accounts, or None if cache miss/error
- """
- all_accounts = await _get_cached_list(SALESFORCE_ACCOUNTS_CACHE_KEY)
- return len(all_accounts) if all_accounts is not None else None
-
-
-_ORG_MAPPINGS_PIPELINE_BATCH = 5000 # Entries per RPUSH call during list population
+ return await _get_list_length(SALESFORCE_ACCOUNTS_CACHE_KEY)
async def store_org_mappings_in_redis(
mappings_data: list[dict[str, Any]],
ttl: int = REDIS_TTL_SECONDS,
) -> None:
- """Store Salesforce org mappings as a Redis List for efficient pagination.
-
- Each mapping is stored as a separate JSON entry so LRANGE can return
- arbitrary pages without decompressing the entire dataset.
- """
- redis_client = get_async_client()
- pipe = await redis_client.pipeline()
- pipe.delete(SALESFORCE_ORG_MAPPINGS_CACHE_KEY)
-
- for i in range(0, len(mappings_data), _ORG_MAPPINGS_PIPELINE_BATCH):
- batch = mappings_data[i : i + _ORG_MAPPINGS_PIPELINE_BATCH]
- pipe.rpush(SALESFORCE_ORG_MAPPINGS_CACHE_KEY, *[json.dumps(m, default=str) for m in batch])
-
- pipe.expire(SALESFORCE_ORG_MAPPINGS_CACHE_KEY, ttl)
- await pipe.execute()
+ await _store_json_list(SALESFORCE_ORG_MAPPINGS_CACHE_KEY, mappings_data, ttl)
async def _lrange_json_or_none(key: str, start: int, end: int) -> list[dict[str, Any]] | None:
@@ -152,20 +115,13 @@ async def get_org_mappings_page(offset: int = 0, limit: int = 10000) -> list[dic
when the key is missing or an entry cannot be parsed, so callers never
mistake an expired cache for a completed run.
"""
- redis_client = get_async_client()
- raw_items = await redis_client.lrange(SALESFORCE_ORG_MAPPINGS_CACHE_KEY, offset, offset + limit - 1)
- if raw_items:
- try:
- return [json.loads(item) for item in raw_items]
- except (TypeError, ValueError) as e:
- raise OrgMappingsCacheMissingError("org mappings cache entries are unreadable") from e
-
- # LRANGE returns nothing both when the key is missing and when the offset is
- # past the end of the list; LLEN tells those cases apart.
- total = await redis_client.llen(SALESFORCE_ORG_MAPPINGS_CACHE_KEY)
- if total and offset >= total:
- return []
- raise OrgMappingsCacheMissingError("org mappings cache key is missing")
+ raw_items = await _get_list_page(SALESFORCE_ORG_MAPPINGS_CACHE_KEY, offset, limit)
+ if raw_items is None:
+ raise OrgMappingsCacheMissingError("org mappings cache key is missing")
+ try:
+ return [json.loads(item) for item in raw_items]
+ except (TypeError, ValueError) as e:
+ raise OrgMappingsCacheMissingError("org mappings cache entries are unreadable") from e
async def get_org_mappings_from_redis() -> list[dict[str, Any]] | None:
@@ -183,13 +139,7 @@ async def get_cached_org_mappings_count() -> int | None:
Returns:
Total number of cached mappings, or None if key missing
"""
- try:
- redis_client = get_async_client()
- count = await redis_client.llen(SALESFORCE_ORG_MAPPINGS_CACHE_KEY)
- return count if count > 0 else None
- except Exception as e:
- capture_exception(e)
- return None
+ return await _get_list_length(SALESFORCE_ORG_MAPPINGS_CACHE_KEY)
async def get_stripe_enrichment_watermark() -> tuple[dt.datetime, str] | None:
diff --git a/ee/billing/test/test_redis_cache.py b/ee/billing/test/test_redis_cache.py
index 764234311cb5..a65c95832774 100644
--- a/ee/billing/test/test_redis_cache.py
+++ b/ee/billing/test/test_redis_cache.py
@@ -6,7 +6,6 @@ from unittest.mock import AsyncMock, patch
from ee.billing.salesforce_enrichment.redis_cache import (
OrgMappingsCacheMissingError,
- _compress_redis_data,
get_cached_org_mappings_count,
get_org_mappings_from_redis,
get_org_mappings_page,
@@ -16,25 +15,6 @@ from ee.billing.salesforce_enrichment.redis_cache import (
)
-class TestCompressionHelpers:
- def test_compression_produces_bytes(self):
- original_data = '{"test": "data"}'
-
- compressed = _compress_redis_data(original_data)
-
- assert isinstance(compressed, bytes)
-
- def test_compression_round_trip(self):
- import gzip
-
- original_data = '{"key": "value", "list": [1, 2, 3]}'
-
- compressed = _compress_redis_data(original_data)
- decompressed = gzip.decompress(compressed).decode("utf-8")
-
- assert decompressed == original_data
-
-
class TestStoreOrgMappingsInRedis:
@pytest.mark.asyncio
async def test_store_org_mappings_success(self):
diff --git a/ee/billing/test/test_salesforce_enrichment.py b/ee/billing/test/test_salesforce_enrichment.py
index 78cd151b641a..11b8a67af464 100644
--- a/ee/billing/test/test_salesforce_enrichment.py
+++ b/ee/billing/test/test_salesforce_enrichment.py
@@ -10,7 +10,9 @@ from unittest.mock import AsyncMock, MagicMock, patch
from django.test import SimpleTestCase
+import fakeredis
from parameterized import parameterized
+from redis.exceptions import ConnectionError as RedisConnectionError
from ee.billing.salesforce_enrichment.constants import (
ACCOUNT_FILL_FIELDS,
@@ -29,8 +31,10 @@ from ee.billing.salesforce_enrichment.enrichment import (
prepare_account_update,
prepare_mirror_clearing_data,
prepare_salesforce_update_data,
+ query_salesforce_accounts_chunk_async,
transform_harmonic_data,
)
+from ee.billing.salesforce_enrichment.redis_cache import get_accounts_from_redis, store_accounts_in_redis
@contextmanager
@@ -679,6 +683,38 @@ class TestSalesforceAccountQuery(SimpleTestCase):
# Verify it doesn't use the old %domain% pattern that would match unintended domains
assert "LIKE '%example.com%'" not in actual_query
+ @pytest.mark.asyncio
+ async def test_chunk_reads_rebuild_a_missing_cache_once_and_page_through_it(self) -> None:
+ accounts = [{"Id": f"001SYN{i:03d}", "Website": f"https://company{i}.example.com"} for i in range(7)]
+ sf = MagicMock()
+ sf.query_all.return_value = {"records": accounts}
+
+ with patch(
+ "ee.billing.salesforce_enrichment.redis_cache.get_async_client",
+ return_value=fakeredis.FakeAsyncRedis(server=fakeredis.FakeServer()),
+ ):
+ pages = [await query_salesforce_accounts_chunk_async(sf, offset, 3) for offset in (0, 3, 6, 9)]
+
+ assert pages == [accounts[0:3], accounts[3:6], accounts[6:7], []]
+ sf.query_all.assert_called_once()
+
+ @pytest.mark.asyncio
+ async def test_chunk_read_error_preserves_cached_accounts(self) -> None:
+ accounts = [{"Id": f"001SYN{i:03d}", "Website": f"https://company{i}.example.com"} for i in range(3)]
+ cached_accounts = list(reversed(accounts))
+ sf = MagicMock()
+ sf.query_all.return_value = {"records": accounts}
+ redis_client = fakeredis.FakeAsyncRedis(server=fakeredis.FakeServer())
+
+ with patch("ee.billing.salesforce_enrichment.redis_cache.get_async_client", return_value=redis_client):
+ await store_accounts_in_redis(cached_accounts)
+ with patch.object(redis_client, "lrange", side_effect=RedisConnectionError("Synthetic read failure")):
+ page = await query_salesforce_accounts_chunk_async(sf, 0, 2)
+ remaining_accounts = await get_accounts_from_redis(0, 3)
+
+ assert page == accounts[:2]
+ assert remaining_accounts == cached_accounts
+
class TestSpecificDomainEnrichment(SimpleTestCase):
@pytest.mark.asyncio