:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(toolbox): reconnect to the pod you already claimed (#130855)
Keelan J.28bd1d3ca350
diff --git a/tools/infra-scripts/clitools/test_toolbox.py b/tools/infra-scripts/clitools/test_toolbox.pyindex 3824c62fb8b9..4ee2a67122ec 100644--- a/tools/infra-scripts/clitools/test_toolbox.py+++ b/tools/infra-scripts/clitools/test_toolbox.py@@ -8,6 +8,7 @@ import signalimport contextlibimport subprocessimport importlib.util+from collections.abc import Callableimport unittestfrom unittest.mock import MagicMock, patch@@ -26,7 +27,7 @@ from toolbox.kubernetes import (validate_context,wait_for_context_access,)-from toolbox.pod import ClaimRaceError, claim_pod, delete_pod, get_toolbox_pod+from toolbox.pod import KEPT_LABELS, ClaimRaceError, claim_pod, delete_pod, get_toolbox_podfrom toolbox.tailscale import SKIP_CHECK_ENV_VAR, TAILSCALE_RUNBOOK_URL, ensure_tailscale_connectedfrom toolbox.user import get_current_user, parse_arn, sanitize_label@@ -234,7 +235,7 @@ class TestToolbox(unittest.TestCase):self.assertEqual(pod_name, "toolbox-pod-1")self.assertFalse(is_claimed)self.assertEqual(resource_version, "12345")- mock_run.assert_called_once_with(+ mock_run.assert_called_with(["kubectl","get",@@ -279,7 +280,8 @@ class TestToolbox(unittest.TestCase):namespace="posthog",context="posthog-dev",)- mock_run.assert_called_once_with(+ self.assertEqual({call.args[0][1] for call in mock_run.call_args_list}, {"--context=posthog-dev"})+ mock_run.assert_called_with(["kubectl","--context=posthog-dev",@@ -328,7 +330,7 @@ class TestToolbox(unittest.TestCase):namespace="posthog-toolbox-django",extra_selector="app.kubernetes.io/component=app",)- mock_run.assert_called_once_with(+ mock_run.assert_called_with(["kubectl","get",@@ -345,39 +347,57 @@ class TestToolbox(unittest.TestCase):check=True,)+ @staticmethod+ def _kubectl_honoring_label_selector(pods: list[dict]) -> Callable[..., MagicMock]:+ def run(cmd: list[str], **kwargs: object) -> MagicMock:+ terms = [term.partition("=") for term in cmd[cmd.index("-l") + 1].split(",")]+ matching = [+ pod+ for pod in pods+ if all(+ pod["metadata"]["labels"].get(key) == value if sep else key in pod["metadata"]["labels"]+ for key, sep, value in terms+ )+ ]+ return MagicMock(stdout=json.dumps({"items": matching}))++ return run+@patch("subprocess.run")def test_get_toolbox_pod_returns_existing_claim_with_rv(self, mock_run):"""An already-claimed pod returns is_claimed=True and the observed resourceVersion."""- mock_response = MagicMock()- mock_response.stdout = json.dumps(- {- "items": [- {- "metadata": {- "name": "toolbox-pod-1",- "resourceVersion": "55555",- "labels": {- "app.kubernetes.io/name": "posthog-toolbox-django",- "toolbox-claimed": "michael.k_at_posthog.com",- },- "deletionTimestamp": None,- },- "status": {"phase": "Running"},- }- ]- }+ pool_labels = {+ "app.kubernetes.io/name": "posthog-toolbox-django",+ "app.kubernetes.io/component": "app",+ "pod-template-hash": "749c5d8db",+ "posthog.com/warpstream-client": "true",+ }+ claimed_labels = {key: value for key, value in pool_labels.items() if key in KEPT_LABELS}+ claimed_labels.update({"toolbox-claimed": "michael.k_at_posthog.com", "terminate-after": "1700000000"})+ mock_run.side_effect = self._kubectl_honoring_label_selector(+ [+ {+ "metadata": {"name": "toolbox-pod-unclaimed", "resourceVersion": "11111", "labels": pool_labels},+ "status": {"phase": "Running"},+ },+ {+ "metadata": {"name": "toolbox-pod-1", "resourceVersion": "55555", "labels": claimed_labels},+ "status": {"phase": "Running"},+ },+ ])- mock_run.return_value = mock_response- pod_name, is_claimed, rv = get_toolbox_pod(- "michael.k_at_posthog.com",- app_label="posthog-toolbox-django",- claimed_label_key="toolbox-claimed",- namespace="posthog",- )- self.assertEqual(pod_name, "toolbox-pod-1")- self.assertTrue(is_claimed)- self.assertEqual(rv, "55555")+ cases = [("posthog", None), ("posthog-toolbox-django", "app.kubernetes.io/component=app")]+ for namespace, extra_selector in cases:+ with self.subTest(extra_selector=extra_selector):+ result = get_toolbox_pod(+ "michael.k_at_posthog.com",+ app_label="posthog-toolbox-django",+ claimed_label_key="toolbox-claimed",+ namespace=namespace,+ extra_selector=extra_selector,+ )+ self.assertEqual(result, ("toolbox-pod-1", True, "55555"))@patch("subprocess.run")def test_claim_pod(self, mock_run):diff --git a/tools/infra-scripts/clitools/toolbox.py b/tools/infra-scripts/clitools/toolbox.pyindex 5785b42ea0ba..bf910b0f5d15 100755--- a/tools/infra-scripts/clitools/toolbox.py+++ b/tools/infra-scripts/clitools/toolbox.py@@ -23,10 +23,11 @@ from toolbox.user import get_current_user# `default_namespace` is where the pool lives when KUBE_NAMESPACE isn't set.## `extra_selectors_by_namespace` ANDs onto the base-# `app.kubernetes.io/name=<app_label>` selector. Keyed by namespace because the-# golden-chart deployment co-locates the toolbox release's own pgbouncer pods-# under the same `name` label, so we need `component=app` there to pick only-# the main pool pods. The legacy `posthog` namespace doesn't need a+# `app.kubernetes.io/name=<app_label>` selector when the wrapper looks for an+# unclaimed pod. Keyed by namespace because the golden-chart deployment+# co-locates the toolbox release's own pgbouncer pods under the same `name`+# label, so we need `component=app` there to pick only the main pool pods.+# The legacy `posthog` namespace doesn't need a# discriminator because its pgbouncers carry a different `name`.POOLS = {"toolbox-django": {diff --git a/tools/infra-scripts/clitools/toolbox/pod.py b/tools/infra-scripts/clitools/toolbox/pod.pyindex ad1f2b1a234e..ac5cc8d0ffbb 100644--- a/tools/infra-scripts/clitools/toolbox/pod.py+++ b/tools/infra-scripts/clitools/toolbox/pod.py@@ -30,6 +30,17 @@ def _is_conflict(stderr: str) -> bool:return "Conflict" in stderr or "the object has been modified" in stderr+def _list_pods(label_selector: str, *, namespace: str, context: str | None) -> list[dict]:+ """Return the pods in ``namespace`` that match ``label_selector``."""+ result = subprocess.run(+ kubectl_cmd("get", "pods", "-n", namespace, "-l", label_selector, "-o", "json", context=context),+ capture_output=True,+ text=True,+ check=True,+ )+ return json.loads(result.stdout)["items"]++def get_toolbox_pod(user: str,check_claimed: bool = True,@@ -50,20 +61,26 @@ def get_toolbox_pod(namespace: Kubernetes namespace the pool lives in.context: Optional kubernetes context to scope the kubectl call to.extra_selector: Optional additional label selector ANDed onto- ``app.kubernetes.io/name=<app_label>``. Used when other resources- in the namespace share the same name label (e.g. the golden chart- puts the toolbox's own pgbouncer pods under the same name) so we- need a second key like ``app.kubernetes.io/component=app`` to- pick out only the main pool.+ ``app.kubernetes.io/name=<app_label>`` when looking for unclaimed+ pods. Used when other resources in the namespace share the same+ name label (e.g. the golden chart puts the toolbox's own pgbouncer+ pods under the same name) so we need a second key like+ ``app.kubernetes.io/component=app`` to pick out only the main pool.+ The lookup for the user's own claimed pod does not use it.Returns:Tuple of (pod_name, is_already_claimed, resource_version). Theresource_version reflects the pod state at observation time and isused by ``claim_pod`` for optimistic concurrency control."""- label_selector = f"app.kubernetes.io/name={app_label}"+ # Get user ID from ARN and sanitize it+ user_id = sanitize_label(user.split("/")[-1] if "/" in user else user)+ pool_selector = f"app.kubernetes.io/name={app_label}"+ # claim_pod removes every label outside KEPT_LABELS, extra_selector's label too, so a+ # claimed pod no longer matches the pool selector. Find it by its claim label instead.+ claimed_selector = f"{pool_selector},{claimed_label_key}={user_id}"if extra_selector:- label_selector = f"{label_selector},{extra_selector}"+ pool_selector = f"{pool_selector},{extra_selector}"wait_start = datetime.now()max_wait = timedelta(minutes=5)update_interval = timedelta(seconds=30)@@ -71,31 +88,11 @@ def get_toolbox_pod(while True:try:- result = subprocess.run(- kubectl_cmd(- "get",- "pods",- "-n",- namespace,- "-l",- label_selector,- "-o",- "json",- context=context,- ),- capture_output=True,- text=True,- check=True,- )- pods = json.loads(result.stdout)["items"]-if check_claimed:- # Get user ID from ARN and sanitize it- user_id = sanitize_label(user.split("/")[-1] if "/" in user else user)# First look for pods already claimed by this userclaimed_pods = [(pod["metadata"]["name"], pod["metadata"].get("resourceVersion", ""))- for pod in pods+ for pod in _list_pods(claimed_selector, namespace=namespace, context=context)if pod["status"]["phase"] == "Running"and pod.get("metadata", {}).get("labels", {}).get(claimed_label_key) == user_idand not pod.get("metadata", {}).get("deletionTimestamp")@@ -109,7 +106,7 @@ def get_toolbox_pod(# Look for unclaimed podsavailable_pods = [(pod["metadata"]["name"], pod["metadata"].get("resourceVersion", ""))- for pod in pods+ for pod in _list_pods(pool_selector, namespace=namespace, context=context)if pod["status"]["phase"] in ("Running", "Pending")and claimed_label_key not in pod.get("metadata", {}).get("labels", {})and "terminate-after" not in pod.get("metadata", {}).get("labels", {})