Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(toolbox): reconnect to the pod you already claimed (#130855)

Keelan J.28bd1d3ca350
diff --git a/tools/infra-scripts/clitools/test_toolbox.py b/tools/infra-scripts/clitools/test_toolbox.py
index 3824c62fb8b9..4ee2a67122ec 100644
--- a/tools/infra-scripts/clitools/test_toolbox.py
+++ b/tools/infra-scripts/clitools/test_toolbox.py
@@ -8,6 +8,7 @@ import signal
import contextlib
import subprocess
import importlib.util
+from collections.abc import Callable
import unittest
from unittest.mock import MagicMock, patch
@@ -26,7 +27,7 @@ from toolbox.kubernetes import (
validate_context,
wait_for_context_access,
)
-from toolbox.pod import ClaimRaceError, claim_pod, delete_pod, get_toolbox_pod
+from toolbox.pod import KEPT_LABELS, ClaimRaceError, claim_pod, delete_pod, get_toolbox_pod
from toolbox.tailscale import SKIP_CHECK_ENV_VAR, TAILSCALE_RUNBOOK_URL, ensure_tailscale_connected
from toolbox.user import get_current_user, parse_arn, sanitize_label
@@ -234,7 +235,7 @@ class TestToolbox(unittest.TestCase):
self.assertEqual(pod_name, "toolbox-pod-1")
self.assertFalse(is_claimed)
self.assertEqual(resource_version, "12345")
- mock_run.assert_called_once_with(
+ mock_run.assert_called_with(
[
"kubectl",
"get",
@@ -279,7 +280,8 @@ class TestToolbox(unittest.TestCase):
namespace="posthog",
context="posthog-dev",
)
- mock_run.assert_called_once_with(
+ self.assertEqual({call.args[0][1] for call in mock_run.call_args_list}, {"--context=posthog-dev"})
+ mock_run.assert_called_with(
[
"kubectl",
"--context=posthog-dev",
@@ -328,7 +330,7 @@ class TestToolbox(unittest.TestCase):
namespace="posthog-toolbox-django",
extra_selector="app.kubernetes.io/component=app",
)
- mock_run.assert_called_once_with(
+ mock_run.assert_called_with(
[
"kubectl",
"get",
@@ -345,39 +347,57 @@ class TestToolbox(unittest.TestCase):
check=True,
)
+ @staticmethod
+ def _kubectl_honoring_label_selector(pods: list[dict]) -> Callable[..., MagicMock]:
+ def run(cmd: list[str], **kwargs: object) -> MagicMock:
+ terms = [term.partition("=") for term in cmd[cmd.index("-l") + 1].split(",")]
+ matching = [
+ pod
+ for pod in pods
+ if all(
+ pod["metadata"]["labels"].get(key) == value if sep else key in pod["metadata"]["labels"]
+ for key, sep, value in terms
+ )
+ ]
+ return MagicMock(stdout=json.dumps({"items": matching}))
+
+ return run
+
@patch("subprocess.run")
def test_get_toolbox_pod_returns_existing_claim_with_rv(self, mock_run):
"""An already-claimed pod returns is_claimed=True and the observed resourceVersion."""
- mock_response = MagicMock()
- mock_response.stdout = json.dumps(
- {
- "items": [
- {
- "metadata": {
- "name": "toolbox-pod-1",
- "resourceVersion": "55555",
- "labels": {
- "app.kubernetes.io/name": "posthog-toolbox-django",
- "toolbox-claimed": "michael.k_at_posthog.com",
- },
- "deletionTimestamp": None,
- },
- "status": {"phase": "Running"},
- }
- ]
- }
+ pool_labels = {
+ "app.kubernetes.io/name": "posthog-toolbox-django",
+ "app.kubernetes.io/component": "app",
+ "pod-template-hash": "749c5d8db",
+ "posthog.com/warpstream-client": "true",
+ }
+ claimed_labels = {key: value for key, value in pool_labels.items() if key in KEPT_LABELS}
+ claimed_labels.update({"toolbox-claimed": "michael.k_at_posthog.com", "terminate-after": "1700000000"})
+ mock_run.side_effect = self._kubectl_honoring_label_selector(
+ [
+ {
+ "metadata": {"name": "toolbox-pod-unclaimed", "resourceVersion": "11111", "labels": pool_labels},
+ "status": {"phase": "Running"},
+ },
+ {
+ "metadata": {"name": "toolbox-pod-1", "resourceVersion": "55555", "labels": claimed_labels},
+ "status": {"phase": "Running"},
+ },
+ ]
)
- mock_run.return_value = mock_response
- pod_name, is_claimed, rv = get_toolbox_pod(
- "michael.k_at_posthog.com",
- app_label="posthog-toolbox-django",
- claimed_label_key="toolbox-claimed",
- namespace="posthog",
- )
- self.assertEqual(pod_name, "toolbox-pod-1")
- self.assertTrue(is_claimed)
- self.assertEqual(rv, "55555")
+ cases = [("posthog", None), ("posthog-toolbox-django", "app.kubernetes.io/component=app")]
+ for namespace, extra_selector in cases:
+ with self.subTest(extra_selector=extra_selector):
+ result = get_toolbox_pod(
+ "michael.k_at_posthog.com",
+ app_label="posthog-toolbox-django",
+ claimed_label_key="toolbox-claimed",
+ namespace=namespace,
+ extra_selector=extra_selector,
+ )
+ self.assertEqual(result, ("toolbox-pod-1", True, "55555"))
@patch("subprocess.run")
def test_claim_pod(self, mock_run):
diff --git a/tools/infra-scripts/clitools/toolbox.py b/tools/infra-scripts/clitools/toolbox.py
index 5785b42ea0ba..bf910b0f5d15 100755
--- a/tools/infra-scripts/clitools/toolbox.py
+++ b/tools/infra-scripts/clitools/toolbox.py
@@ -23,10 +23,11 @@ from toolbox.user import get_current_user
# `default_namespace` is where the pool lives when KUBE_NAMESPACE isn't set.
#
# `extra_selectors_by_namespace` ANDs onto the base
-# `app.kubernetes.io/name=<app_label>` selector. Keyed by namespace because the
-# golden-chart deployment co-locates the toolbox release's own pgbouncer pods
-# under the same `name` label, so we need `component=app` there to pick only
-# the main pool pods. The legacy `posthog` namespace doesn't need a
+# `app.kubernetes.io/name=<app_label>` selector when the wrapper looks for an
+# unclaimed pod. Keyed by namespace because the golden-chart deployment
+# co-locates the toolbox release's own pgbouncer pods under the same `name`
+# label, so we need `component=app` there to pick only the main pool pods.
+# The legacy `posthog` namespace doesn't need a
# discriminator because its pgbouncers carry a different `name`.
POOLS = {
"toolbox-django": {
diff --git a/tools/infra-scripts/clitools/toolbox/pod.py b/tools/infra-scripts/clitools/toolbox/pod.py
index ad1f2b1a234e..ac5cc8d0ffbb 100644
--- a/tools/infra-scripts/clitools/toolbox/pod.py
+++ b/tools/infra-scripts/clitools/toolbox/pod.py
@@ -30,6 +30,17 @@ def _is_conflict(stderr: str) -> bool:
return "Conflict" in stderr or "the object has been modified" in stderr
+def _list_pods(label_selector: str, *, namespace: str, context: str | None) -> list[dict]:
+ """Return the pods in ``namespace`` that match ``label_selector``."""
+ result = subprocess.run(
+ kubectl_cmd("get", "pods", "-n", namespace, "-l", label_selector, "-o", "json", context=context),
+ capture_output=True,
+ text=True,
+ check=True,
+ )
+ return json.loads(result.stdout)["items"]
+
+
def get_toolbox_pod(
user: str,
check_claimed: bool = True,
@@ -50,20 +61,26 @@ def get_toolbox_pod(
namespace: Kubernetes namespace the pool lives in.
context: Optional kubernetes context to scope the kubectl call to.
extra_selector: Optional additional label selector ANDed onto
- ``app.kubernetes.io/name=<app_label>``. Used when other resources
- in the namespace share the same name label (e.g. the golden chart
- puts the toolbox's own pgbouncer pods under the same name) so we
- need a second key like ``app.kubernetes.io/component=app`` to
- pick out only the main pool.
+ ``app.kubernetes.io/name=<app_label>`` when looking for unclaimed
+ pods. Used when other resources in the namespace share the same
+ name label (e.g. the golden chart puts the toolbox's own pgbouncer
+ pods under the same name) so we need a second key like
+ ``app.kubernetes.io/component=app`` to pick out only the main pool.
+ The lookup for the user's own claimed pod does not use it.
Returns:
Tuple of (pod_name, is_already_claimed, resource_version). The
resource_version reflects the pod state at observation time and is
used by ``claim_pod`` for optimistic concurrency control.
"""
- label_selector = f"app.kubernetes.io/name={app_label}"
+ # Get user ID from ARN and sanitize it
+ user_id = sanitize_label(user.split("/")[-1] if "/" in user else user)
+ pool_selector = f"app.kubernetes.io/name={app_label}"
+ # claim_pod removes every label outside KEPT_LABELS, extra_selector's label too, so a
+ # claimed pod no longer matches the pool selector. Find it by its claim label instead.
+ claimed_selector = f"{pool_selector},{claimed_label_key}={user_id}"
if extra_selector:
- label_selector = f"{label_selector},{extra_selector}"
+ pool_selector = f"{pool_selector},{extra_selector}"
wait_start = datetime.now()
max_wait = timedelta(minutes=5)
update_interval = timedelta(seconds=30)
@@ -71,31 +88,11 @@ def get_toolbox_pod(
while True:
try:
- result = subprocess.run(
- kubectl_cmd(
- "get",
- "pods",
- "-n",
- namespace,
- "-l",
- label_selector,
- "-o",
- "json",
- context=context,
- ),
- capture_output=True,
- text=True,
- check=True,
- )
- pods = json.loads(result.stdout)["items"]
-
if check_claimed:
- # Get user ID from ARN and sanitize it
- user_id = sanitize_label(user.split("/")[-1] if "/" in user else user)
# First look for pods already claimed by this user
claimed_pods = [
(pod["metadata"]["name"], pod["metadata"].get("resourceVersion", ""))
- for pod in pods
+ for pod in _list_pods(claimed_selector, namespace=namespace, context=context)
if pod["status"]["phase"] == "Running"
and pod.get("metadata", {}).get("labels", {}).get(claimed_label_key) == user_id
and not pod.get("metadata", {}).get("deletionTimestamp")
@@ -109,7 +106,7 @@ def get_toolbox_pod(
# Look for unclaimed pods
available_pods = [
(pod["metadata"]["name"], pod["metadata"].get("resourceVersion", ""))
- for pod in pods
+ for pod in _list_pods(pool_selector, namespace=namespace, context=context)
if pod["status"]["phase"] in ("Running", "Pending")
and claimed_label_key not in pod.get("metadata", {}).get("labels", {})
and "terminate-after" not in pod.get("metadata", {}).get("labels", {})