:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(toolbox): make kubectl stream debug logs opt-in (#123067)
Pawel Cebula45ee4ad0bb76
diff --git a/tools/infra-scripts/clitools/README.md b/tools/infra-scripts/clitools/README.mdindex 131676483d52..b6cfb193fdec 100644--- a/tools/infra-scripts/clitools/README.md+++ b/tools/infra-scripts/clitools/README.md@@ -57,14 +57,18 @@ The toolbox CLI:### Available Flags-| Flag | Description | Default |-| ------------------------ | ------------------------------------------------ | ------- |-| `--claim-duration HOURS` | Number of hours to claim the pod for | 12 |-| `--update-claim` | Update the termination time of your existing pod | False |+| Flag | Description | Default |+| ------------------------ | --------------------------------------------------------- | ------- |+| `--claim-duration HOURS` | Number of hours to claim the pod for | 12 |+| `--update-claim` | Update the termination time of your existing pod | False |+| `--kubectl-debug` | Enable kubectl stream debug logs for the shell connection | False |Set `KUBE_CONTEXT` to bypass environment selection for expert workflows. The context is scoped to this process andmust have the pod permissions required by the toolbox; the CLI never changes your global current context.+Shell connections ignore the local `DEBUG` variable by default because Flox sets it for Django development.+Use `--kubectl-debug` to enable stream debug logs explicitly. This only changes the environment of the local kubectl connection.+### Examples1. Connect to a pod with default 12-hour claim:diff --git a/tools/infra-scripts/clitools/test_toolbox.py b/tools/infra-scripts/clitools/test_toolbox.pyindex af752bf864aa..74bf11f77028 100644--- a/tools/infra-scripts/clitools/test_toolbox.py+++ b/tools/infra-scripts/clitools/test_toolbox.py@@ -1283,9 +1283,9 @@ class TestToolbox(unittest.TestCase):mock_run.assert_called_once()+ @patch.dict(os.environ, {"DEBUG": "1", "PATH": "/test/bin", "AWS_PROFILE": "test-profile"}, clear=True)@patch("subprocess.run")def test_connect_to_pod_returns_zero_on_success(self, mock_run):- """connect_to_pod returns 0 when kubectl exec exits cleanly."""from toolbox.pod import connect_to_podmock_result = MagicMock()@@ -1294,6 +1294,8 @@ class TestToolbox(unittest.TestCase):rc = connect_to_pod("toolbox-pod-1", namespace="posthog")self.assertEqual(rc, 0)+ self.assertEqual(mock_run.call_args.kwargs["env"], {"PATH": "/test/bin", "AWS_PROFILE": "test-profile"})+ self.assertEqual(os.environ["DEBUG"], "1")@patch("subprocess.run")def test_connect_to_pod_returns_nonzero_on_failure(self, mock_run):@@ -1312,16 +1314,16 @@ class TestToolbox(unittest.TestCase):rc = connect_to_pod("toolbox-pod-1", namespace="posthog")self.assertEqual(rc, 1)+ @patch.dict(os.environ, {"PATH": "/test/bin", "AWS_PROFILE": "test-profile"}, clear=True)@patch("subprocess.run")def test_connect_to_pod_with_context(self, mock_run):- """connect_to_pod with context= passes --context to kubectl exec."""from toolbox.pod import connect_to_podmock_result = MagicMock()mock_result.returncode = 0mock_run.return_value = mock_result- connect_to_pod("toolbox-pod-1", namespace="posthog", context="posthog-dev")+ connect_to_pod("toolbox-pod-1", namespace="posthog", context="posthog-dev", kubectl_debug=True)mock_run.assert_called_once_with(["kubectl",@@ -1333,8 +1335,10 @@ class TestToolbox(unittest.TestCase):"toolbox-pod-1","--","bash",- ]+ ],+ env={"PATH": "/test/bin", "AWS_PROFILE": "test-profile", "DEBUG": "1"},)+ self.assertNotIn("DEBUG", os.environ)def _patch_main_collaborators(self, *, get_pod_returns=("toolbox-pod-1", False, "12345")):"""Common patch stack for main() integration tests.@@ -1717,7 +1721,6 @@ class TestToolbox(unittest.TestCase):self.assertEqual(m_claim.call_args.kwargs["resource_version"], None)def test_main_propagates_connect_to_pod_exit_code(self):- """When kubectl exec returns non-zero, main() exits with the same code."""patches = self._patch_main_collaborators()patches["connect_to_pod"] = patch.object(toolbox_script, "connect_to_pod", return_value=42)@@ -1725,11 +1728,11 @@ class TestToolbox(unittest.TestCase):patches["get_current_user"],patches["get_toolbox_pod"],patches["claim_pod"],- patches["connect_to_pod"],+ patches["connect_to_pod"] as mock_connect,patches["delete_pod"],patches["select_context"],patches["validate_context"],- patch.object(toolbox_script.sys, "argv", ["toolbox.py", "--auto-delete"]),+ patch.object(toolbox_script.sys, "argv", ["toolbox.py", "--auto-delete", "--kubectl-debug"]),patch.dict(os.environ, {}, clear=False),):self._clean_env()@@ -1737,6 +1740,7 @@ class TestToolbox(unittest.TestCase):toolbox_script.main()self.assertEqual(ctx.exception.code, 42)+ self.assertTrue(mock_connect.call_args.kwargs["kubectl_debug"])def test_main_retries_on_claim_race_and_picks_new_pod(self):"""A ClaimRaceError on the first attempt triggers a re-fetch and retry against the next available pod."""@@ -1777,7 +1781,9 @@ class TestToolbox(unittest.TestCase):self.assertEqual(m_get_pod.call_count, 2)self.assertEqual(m_claim.call_count, 2)# We connected to the pod we won, not the one we lost.- m_connect.assert_called_once_with("toolbox-pod-B", namespace="posthog-toolbox-django", context="posthog-dev")+ m_connect.assert_called_once_with(+ "toolbox-pod-B", namespace="posthog-toolbox-django", context="posthog-dev", kubectl_debug=False+ )# atexit was registered twice (once per candidate pod) and unregistered once# to clear the stale registration after the race.self.assertEqual(m_atexit.call_count, 2)@@ -1867,7 +1873,9 @@ class TestToolbox(unittest.TestCase):with self.assertRaises(SystemExit):toolbox_script.main()- m_connect.assert_called_once_with("toolbox-pod-mine", namespace="posthog-toolbox-django", context="posthog-dev")+ m_connect.assert_called_once_with(+ "toolbox-pod-mine", namespace="posthog-toolbox-django", context="posthog-dev", kubectl_debug=False+ )# claim_pod was called once (the racing one) and not retried since we found our own pod.self.assertEqual(m_claim.call_count, 1)# First atexit register armed cleanup for pod-A; on the race we unregistered;diff --git a/tools/infra-scripts/clitools/toolbox.py b/tools/infra-scripts/clitools/toolbox.pyindex f33c3c498ccc..5785b42ea0ba 100755--- a/tools/infra-scripts/clitools/toolbox.py+++ b/tools/infra-scripts/clitools/toolbox.py@@ -90,6 +90,11 @@ def main():action="store_true",help="Skip the [y/N] prompt on exit and unconditionally delete the claimed pod on normal exit, Ctrl-C, or terminal close.",)+ parser.add_argument(+ "--kubectl-debug",+ action="store_true",+ help="Enable kubectl stream debug logs for the shell connection.",+ )args = parser.parse_args()# The cluster endpoints are only reachable over the tailnet; without this@@ -274,10 +279,14 @@ def main():print("✅ Connecting to your existing pod (use --update-claim to extend the duration)") # noqa: T201if args.auto_delete:- rc = connect_to_pod(pod_name, namespace=namespace, context=selected_context)+ rc = connect_to_pod(+ pod_name, namespace=namespace, context=selected_context, kubectl_debug=args.kubectl_debug+ )else:try:- rc = connect_to_pod(pod_name, namespace=namespace, context=selected_context)+ rc = connect_to_pod(+ pod_name, namespace=namespace, context=selected_context, kubectl_debug=args.kubectl_debug+ )finally:if will_claim:delete_pod(pod_name, namespace=namespace, context=selected_context)diff --git a/tools/infra-scripts/clitools/toolbox/pod.py b/tools/infra-scripts/clitools/toolbox/pod.pyindex eb5bb5798eb4..15be583ab4f1 100644--- a/tools/infra-scripts/clitools/toolbox/pod.py+++ b/tools/infra-scripts/clitools/toolbox/pod.py@@ -1,6 +1,7 @@#!/usr/bin/env python3"""Pod management functions."""+import osimport sysimport jsonimport time@@ -246,7 +247,7 @@ def claim_pod(sys.exit(1)-def connect_to_pod(pod_name: str, *, namespace: str, context: str | None = None) -> int:+def connect_to_pod(pod_name: str, *, namespace: str, context: str | None = None, kubectl_debug: bool = False) -> int:"""Connect to the specified pod using kubectl exec, returning the exec exit code.The caller is responsible for surfacing a non-zero return as a non-zero@@ -254,7 +255,14 @@ def connect_to_pod(pod_name: str, *, namespace: str, context: str | None = None)as success and trigger downstream cleanup of a session that never opened."""_safe_print(f"🚀 Connecting to pod {pod_name}...")- result = subprocess.run(kubectl_cmd("exec", "-it", "-n", namespace, pod_name, "--", "bash", context=context))+ env = os.environ.copy()+ # The streaming library enables debug logs for any nonempty DEBUG value, including Flox's Django setting.+ env.pop("DEBUG", None)+ if kubectl_debug:+ env["DEBUG"] = "1"+ result = subprocess.run(+ kubectl_cmd("exec", "-it", "-n", namespace, pod_name, "--", "bash", context=context), env=env+ )return result.returncode