Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(toolbox): make kubectl stream debug logs opt-in (#123067)

Pawel Cebula45ee4ad0bb76
diff --git a/tools/infra-scripts/clitools/README.md b/tools/infra-scripts/clitools/README.md
index 131676483d52..b6cfb193fdec 100644
--- a/tools/infra-scripts/clitools/README.md
+++ b/tools/infra-scripts/clitools/README.md
@@ -57,14 +57,18 @@ The toolbox CLI:
### Available Flags
-| Flag | Description | Default |
-| ------------------------ | ------------------------------------------------ | ------- |
-| `--claim-duration HOURS` | Number of hours to claim the pod for | 12 |
-| `--update-claim` | Update the termination time of your existing pod | False |
+| Flag | Description | Default |
+| ------------------------ | --------------------------------------------------------- | ------- |
+| `--claim-duration HOURS` | Number of hours to claim the pod for | 12 |
+| `--update-claim` | Update the termination time of your existing pod | False |
+| `--kubectl-debug` | Enable kubectl stream debug logs for the shell connection | False |
Set `KUBE_CONTEXT` to bypass environment selection for expert workflows. The context is scoped to this process and
must have the pod permissions required by the toolbox; the CLI never changes your global current context.
+Shell connections ignore the local `DEBUG` variable by default because Flox sets it for Django development.
+Use `--kubectl-debug` to enable stream debug logs explicitly. This only changes the environment of the local kubectl connection.
+
### Examples
1. Connect to a pod with default 12-hour claim:
diff --git a/tools/infra-scripts/clitools/test_toolbox.py b/tools/infra-scripts/clitools/test_toolbox.py
index af752bf864aa..74bf11f77028 100644
--- a/tools/infra-scripts/clitools/test_toolbox.py
+++ b/tools/infra-scripts/clitools/test_toolbox.py
@@ -1283,9 +1283,9 @@ class TestToolbox(unittest.TestCase):
mock_run.assert_called_once()
+ @patch.dict(os.environ, {"DEBUG": "1", "PATH": "/test/bin", "AWS_PROFILE": "test-profile"}, clear=True)
@patch("subprocess.run")
def test_connect_to_pod_returns_zero_on_success(self, mock_run):
- """connect_to_pod returns 0 when kubectl exec exits cleanly."""
from toolbox.pod import connect_to_pod
mock_result = MagicMock()
@@ -1294,6 +1294,8 @@ class TestToolbox(unittest.TestCase):
rc = connect_to_pod("toolbox-pod-1", namespace="posthog")
self.assertEqual(rc, 0)
+ self.assertEqual(mock_run.call_args.kwargs["env"], {"PATH": "/test/bin", "AWS_PROFILE": "test-profile"})
+ self.assertEqual(os.environ["DEBUG"], "1")
@patch("subprocess.run")
def test_connect_to_pod_returns_nonzero_on_failure(self, mock_run):
@@ -1312,16 +1314,16 @@ class TestToolbox(unittest.TestCase):
rc = connect_to_pod("toolbox-pod-1", namespace="posthog")
self.assertEqual(rc, 1)
+ @patch.dict(os.environ, {"PATH": "/test/bin", "AWS_PROFILE": "test-profile"}, clear=True)
@patch("subprocess.run")
def test_connect_to_pod_with_context(self, mock_run):
- """connect_to_pod with context= passes --context to kubectl exec."""
from toolbox.pod import connect_to_pod
mock_result = MagicMock()
mock_result.returncode = 0
mock_run.return_value = mock_result
- connect_to_pod("toolbox-pod-1", namespace="posthog", context="posthog-dev")
+ connect_to_pod("toolbox-pod-1", namespace="posthog", context="posthog-dev", kubectl_debug=True)
mock_run.assert_called_once_with(
[
"kubectl",
@@ -1333,8 +1335,10 @@ class TestToolbox(unittest.TestCase):
"toolbox-pod-1",
"--",
"bash",
- ]
+ ],
+ env={"PATH": "/test/bin", "AWS_PROFILE": "test-profile", "DEBUG": "1"},
)
+ self.assertNotIn("DEBUG", os.environ)
def _patch_main_collaborators(self, *, get_pod_returns=("toolbox-pod-1", False, "12345")):
"""Common patch stack for main() integration tests.
@@ -1717,7 +1721,6 @@ class TestToolbox(unittest.TestCase):
self.assertEqual(m_claim.call_args.kwargs["resource_version"], None)
def test_main_propagates_connect_to_pod_exit_code(self):
- """When kubectl exec returns non-zero, main() exits with the same code."""
patches = self._patch_main_collaborators()
patches["connect_to_pod"] = patch.object(toolbox_script, "connect_to_pod", return_value=42)
@@ -1725,11 +1728,11 @@ class TestToolbox(unittest.TestCase):
patches["get_current_user"],
patches["get_toolbox_pod"],
patches["claim_pod"],
- patches["connect_to_pod"],
+ patches["connect_to_pod"] as mock_connect,
patches["delete_pod"],
patches["select_context"],
patches["validate_context"],
- patch.object(toolbox_script.sys, "argv", ["toolbox.py", "--auto-delete"]),
+ patch.object(toolbox_script.sys, "argv", ["toolbox.py", "--auto-delete", "--kubectl-debug"]),
patch.dict(os.environ, {}, clear=False),
):
self._clean_env()
@@ -1737,6 +1740,7 @@ class TestToolbox(unittest.TestCase):
toolbox_script.main()
self.assertEqual(ctx.exception.code, 42)
+ self.assertTrue(mock_connect.call_args.kwargs["kubectl_debug"])
def test_main_retries_on_claim_race_and_picks_new_pod(self):
"""A ClaimRaceError on the first attempt triggers a re-fetch and retry against the next available pod."""
@@ -1777,7 +1781,9 @@ class TestToolbox(unittest.TestCase):
self.assertEqual(m_get_pod.call_count, 2)
self.assertEqual(m_claim.call_count, 2)
# We connected to the pod we won, not the one we lost.
- m_connect.assert_called_once_with("toolbox-pod-B", namespace="posthog-toolbox-django", context="posthog-dev")
+ m_connect.assert_called_once_with(
+ "toolbox-pod-B", namespace="posthog-toolbox-django", context="posthog-dev", kubectl_debug=False
+ )
# atexit was registered twice (once per candidate pod) and unregistered once
# to clear the stale registration after the race.
self.assertEqual(m_atexit.call_count, 2)
@@ -1867,7 +1873,9 @@ class TestToolbox(unittest.TestCase):
with self.assertRaises(SystemExit):
toolbox_script.main()
- m_connect.assert_called_once_with("toolbox-pod-mine", namespace="posthog-toolbox-django", context="posthog-dev")
+ m_connect.assert_called_once_with(
+ "toolbox-pod-mine", namespace="posthog-toolbox-django", context="posthog-dev", kubectl_debug=False
+ )
# claim_pod was called once (the racing one) and not retried since we found our own pod.
self.assertEqual(m_claim.call_count, 1)
# First atexit register armed cleanup for pod-A; on the race we unregistered;
diff --git a/tools/infra-scripts/clitools/toolbox.py b/tools/infra-scripts/clitools/toolbox.py
index f33c3c498ccc..5785b42ea0ba 100755
--- a/tools/infra-scripts/clitools/toolbox.py
+++ b/tools/infra-scripts/clitools/toolbox.py
@@ -90,6 +90,11 @@ def main():
action="store_true",
help="Skip the [y/N] prompt on exit and unconditionally delete the claimed pod on normal exit, Ctrl-C, or terminal close.",
)
+ parser.add_argument(
+ "--kubectl-debug",
+ action="store_true",
+ help="Enable kubectl stream debug logs for the shell connection.",
+ )
args = parser.parse_args()
# The cluster endpoints are only reachable over the tailnet; without this
@@ -274,10 +279,14 @@ def main():
print("✅ Connecting to your existing pod (use --update-claim to extend the duration)") # noqa: T201
if args.auto_delete:
- rc = connect_to_pod(pod_name, namespace=namespace, context=selected_context)
+ rc = connect_to_pod(
+ pod_name, namespace=namespace, context=selected_context, kubectl_debug=args.kubectl_debug
+ )
else:
try:
- rc = connect_to_pod(pod_name, namespace=namespace, context=selected_context)
+ rc = connect_to_pod(
+ pod_name, namespace=namespace, context=selected_context, kubectl_debug=args.kubectl_debug
+ )
finally:
if will_claim:
delete_pod(pod_name, namespace=namespace, context=selected_context)
diff --git a/tools/infra-scripts/clitools/toolbox/pod.py b/tools/infra-scripts/clitools/toolbox/pod.py
index eb5bb5798eb4..15be583ab4f1 100644
--- a/tools/infra-scripts/clitools/toolbox/pod.py
+++ b/tools/infra-scripts/clitools/toolbox/pod.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python3
"""Pod management functions."""
+import os
import sys
import json
import time
@@ -246,7 +247,7 @@ def claim_pod(
sys.exit(1)
-def connect_to_pod(pod_name: str, *, namespace: str, context: str | None = None) -> int:
+def connect_to_pod(pod_name: str, *, namespace: str, context: str | None = None, kubectl_debug: bool = False) -> int:
"""Connect to the specified pod using kubectl exec, returning the exec exit code.
The caller is responsible for surfacing a non-zero return as a non-zero
@@ -254,7 +255,14 @@ def connect_to_pod(pod_name: str, *, namespace: str, context: str | None = None)
as success and trigger downstream cleanup of a session that never opened.
"""
_safe_print(f"🚀 Connecting to pod {pod_name}...")
- result = subprocess.run(kubectl_cmd("exec", "-it", "-n", namespace, pod_name, "--", "bash", context=context))
+ env = os.environ.copy()
+ # The streaming library enables debug logs for any nonempty DEBUG value, including Flox's Django setting.
+ env.pop("DEBUG", None)
+ if kubectl_debug:
+ env["DEBUG"] = "1"
+ result = subprocess.run(
+ kubectl_cmd("exec", "-it", "-n", namespace, pod_name, "--", "bash", context=context), env=env
+ )
return result.returncode