Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): record test sessions through an observer, not a module scan (#121865)

Shy Altere53f22e2ef03
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/transport.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/transport.py
index 9fc2d4e3f092..2b6161bf81cd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/transport.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/transport.py
@@ -16,10 +16,11 @@ from __future__ import annotations
import math
import time
-from collections.abc import Iterator, Mapping
+import functools
+from collections.abc import Callable, Iterator, Mapping
from contextlib import contextmanager
from contextvars import ContextVar
-from typing import Any, cast
+from typing import Any, ParamSpec, cast
from urllib.parse import urlparse
from django.conf import settings
@@ -427,6 +428,34 @@ class _NoRedirectSession(requests.Session):
return super().send(request, **kwargs)
+_P = ParamSpec("_P")
+_session_observers: list[Callable[[dict[str, Any]], None]] = []
+
+
+@contextmanager
+def observe_tracked_sessions(observer: Callable[[dict[str, Any]], None]) -> Iterator[None]:
+ """Pass the keyword arguments of each `make_tracked_session` call in the block to `observer`.
+
+ For the source test harness. Only the innermost observer sees a call, so a nested block records its own sessions.
+ """
+ _session_observers.append(observer)
+ try:
+ yield
+ finally:
+ _session_observers.remove(observer)
+
+
+def _observed(factory: Callable[_P, requests.Session]) -> Callable[_P, requests.Session]:
+ @functools.wraps(factory)
+ def make(*args: _P.args, **kwargs: _P.kwargs) -> requests.Session:
+ if _session_observers:
+ _session_observers[-1](dict(kwargs))
+ return factory(*args, **kwargs)
+
+ return make
+
+
+@_observed
def make_tracked_session(
*,
retry: Retry | None = None,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/fake_network.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/fake_network.py
index e62c6ee32c0c..fd701ae3c27a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/fake_network.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/fake_network.py
@@ -23,7 +23,9 @@ from urllib3.connection import HTTPConnection, HTTPSConnection
from posthog.dataclasses import frozen
from posthog.security import url_validation
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http.transport import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http.transport import (
+ observe_tracked_sessions,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
# A run stops at one of these limits, because the source would not stop by itself.
@@ -36,7 +38,6 @@ _SOURCES_PACKAGE = "products.warehouse_sources.backend.temporal.data_imports.sou
_UNSET = object()
_PUBLIC_ADDRESS = "93.184.216.34"
_REAL_SLEEP = time.sleep
-_REAL_MAKE_TRACKED_SESSION = make_tracked_session
_REAL_MONOTONIC = time.monotonic
_REAL_TIME = time.time
_REAL_ASYNC_SLEEP = asyncio.sleep
@@ -238,31 +239,6 @@ class _MemoryRedis:
self._data.pop(key, None)
-_session_factory_alias_cache: tuple[int, list[tuple[Any, str]]] = (-1, [])
-
-
-def _session_factory_aliases() -> list[tuple[Any, str]]:
- """Each binding of `make_tracked_session` in a loaded source module.
-
- A source imports the function by name, so one patch of its home module does not reach the copies.
- A test can import a source after an earlier run, so the scan repeats when more modules are loaded.
- """
- global _session_factory_alias_cache
- loaded = len(sys.modules)
- if _session_factory_alias_cache[0] == loaded:
- return _session_factory_alias_cache[1]
- aliases: list[tuple[Any, str]] = []
- for module_name, module in list(sys.modules.items()):
- # This module keeps the real function, which the recording wrapper calls.
- if not module_name.startswith(_SOURCES_PACKAGE) or module_name == __name__:
- continue
- aliases.extend(
- (module, name) for name, value in list(vars(module).items()) if value is _REAL_MAKE_TRACKED_SESSION
- )
- _session_factory_alias_cache = (loaded, aliases)
- return aliases
-
-
@functools.cache
def _sleep_aliases() -> list[tuple[Any, str]]:
"""Each `from time import sleep` binding in a loaded source module. A patch of `time.sleep` does not reach them."""
@@ -354,10 +330,6 @@ def fake_environment(responder: Responder, request_limit: int = _REQUEST_LIMIT)
def memory_redis(_manager: Any) -> Iterator[_MemoryRedis]:
yield redis
- def recording_session(**kwargs: Any) -> Any:
- network.session_options.append(kwargs)
- return _REAL_MAKE_TRACKED_SESSION(**kwargs)
-
patches: list[Any] = [
mock.patch.object(HTTPConnection, "connect", connect),
mock.patch.object(HTTPSConnection, "connect", connect),
@@ -374,7 +346,7 @@ def fake_environment(responder: Responder, request_limit: int = _REQUEST_LIMIT)
mock.patch.object(asyncio, "sleep", async_sleep),
mock.patch.object(ResumableSourceManager, "_get_redis", memory_redis),
*(mock.patch.object(module, name, sleep) for module, name in _sleep_aliases()),
- *(mock.patch.object(module, name, recording_session) for module, name in _session_factory_aliases()),
+ observe_tracked_sessions(network.session_options.append),
_waits_on_the_fake_clock(sleep),
]
with contextlib.ExitStack() as stack:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/tests/test_driver.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/tests/test_driver.py
index 456ccde30c11..7719d83d3129 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/tests/test_driver.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/tests/test_driver.py
@@ -1,3 +1,5 @@
+import sys
+import types
import dataclasses
from collections.abc import Iterator
from typing import Any
@@ -19,6 +21,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
SourceDriver,
UnexpectedRequest,
always,
+ scripted_network,
source_inputs,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
@@ -148,6 +151,19 @@ class TestSourceDriver:
assert result.session_options == [{"redact_values": ("t",)}]
+ def test_a_run_nested_in_a_scripted_network_records_its_sessions_and_leaves_later_runs_recording(self) -> None:
+ with patch.dict(sys.modules):
+ with scripted_network([]) as outer:
+ sys.modules["_imported_while_patched"] = types.ModuleType("_imported_while_patched")
+ nested = _driver().run("rows", [_page()])
+ later = _driver().run("rows", [_page()])
+
+ assert (outer.session_options, nested.session_options, later.session_options) == (
+ [],
+ [{"redact_values": ("t",)}],
+ [{"redact_values": ("t",)}],
+ )
+
def test_an_exception_in_the_script_fails_the_request_as_a_dropped_connection_does(self) -> None:
def dropped(_request: Any) -> ScriptedResponse:
raise ConnectionResetError("reset by peer")