:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(notebooks): rate limit notebook create and kernel sandbox starts (#127056)
posthog[bot]40eb37a49fc8
diff --git a/products/notebooks/backend/kernel_runtime.py b/products/notebooks/backend/kernel_runtime.pyindex cb627acd1470..12080fa35fba 100644--- a/products/notebooks/backend/kernel_runtime.py+++ b/products/notebooks/backend/kernel_runtime.py@@ -43,6 +43,14 @@ logger = structlog.get_logger(__name__)# (`SANDBOX_TTL_SECONDS`) is longer because it has to outlast an unattended agent run.NOTEBOOK_KERNEL_TTL_SECONDS = 1 * 60 * 60+# Each sandbox is paid compute until its TTL. A normal user starts a few a day.+NOTEBOOK_KERNEL_STARTS_PER_USER_PER_HOUR = 30+NOTEBOOK_KERNEL_MAX_RUNNING_PER_TEAM = 20+++class KernelSandboxLimitExceeded(RuntimeError):+ pass+@dataclassclass KernelExecutionResult:@@ -721,7 +729,26 @@ class KernelRuntimeService:return KernelRuntime.Backend.DOCKERreturn None+ def _check_sandbox_limits(self, notebook: Notebook, user: User | None) -> None:+ now = timezone.now()+ if isinstance(user, User):+ recent_starts = KernelRuntime.objects.filter(user=user, created_at__gte=now - timedelta(hours=1)).count()+ if recent_starts >= NOTEBOOK_KERNEL_STARTS_PER_USER_PER_HOUR:+ logger.warning("notebook_kernel_user_start_limit", team_id=notebook.team_id, user_id=user.id)+ raise KernelSandboxLimitExceeded(+ "You started too many notebook kernels in the last hour. Try again later."+ )+ running = KernelRuntime.objects.filter(+ team_id=notebook.team_id, ended_at__isnull=True, ttl_expires_at__gt=now+ ).count()+ if running >= NOTEBOOK_KERNEL_MAX_RUNNING_PER_TEAM:+ logger.warning("notebook_kernel_team_running_limit", team_id=notebook.team_id)+ raise KernelSandboxLimitExceeded(+ "This project has too many notebook kernels running. Stop a kernel or try again later."+ )+def _create_kernel_handle(self, notebook: Notebook, user: User | None, backend: str) -> _KernelHandle:+ self._check_sandbox_limits(notebook, user)runtime = self._create_runtime(notebook, user, backend)connection_file = f"/tmp/jupyter/kernel-{runtime.id}.json"kernel_id = f"kernel-{runtime.id}"diff --git a/products/notebooks/backend/presentation/notebook_throttles.py b/products/notebooks/backend/presentation/notebook_throttles.pynew file mode 100644index 000000000000..6aec0d184818--- /dev/null+++ b/products/notebooks/backend/presentation/notebook_throttles.py@@ -0,0 +1,22 @@+from posthog.rate_limit import _UserBucketRateThrottle+++# Each new notebook can start a paid kernel sandbox, so these rates stop a scripted create loop.+class NotebookCreateBurstThrottle(_UserBucketRateThrottle):+ scope = "notebook_create_burst"+ rate = "20/minute"+++class NotebookCreateSustainedThrottle(_UserBucketRateThrottle):+ scope = "notebook_create_sustained"+ rate = "120/hour"+++class NotebookKernelStartBurstThrottle(_UserBucketRateThrottle):+ scope = "notebook_kernel_start_burst"+ rate = "10/minute"+++class NotebookKernelStartSustainedThrottle(_UserBucketRateThrottle):+ scope = "notebook_kernel_start_sustained"+ rate = "60/hour"diff --git a/products/notebooks/backend/presentation/views/notebook.py b/products/notebooks/backend/presentation/views/notebook.pyindex 3d58eb125b41..fc025ef843ca 100644--- a/products/notebooks/backend/presentation/views/notebook.py+++ b/products/notebooks/backend/presentation/views/notebook.py@@ -28,6 +28,7 @@ from rest_framework.exceptions import PermissionDeniedfrom rest_framework.request import Requestfrom rest_framework.response import Responsefrom rest_framework.serializers import BaseSerializer+from rest_framework.throttling import BaseThrottlefrom posthog.hogql.direct_connection import get_direct_connection_source@@ -115,8 +116,18 @@ from products.notebooks.backend.facade.widgets import (set_widget_instance_version,start_widget_generation,)-from products.notebooks.backend.kernel_runtime import build_notebook_sandbox_config, get_kernel_runtime+from products.notebooks.backend.kernel_runtime import (+ KernelSandboxLimitExceeded,+ build_notebook_sandbox_config,+ get_kernel_runtime,+)from products.notebooks.backend.models import KernelRuntime, Notebook, NotebookNodeRun+from products.notebooks.backend.presentation.notebook_throttles import (+ NotebookCreateBurstThrottle,+ NotebookCreateSustainedThrottle,+ NotebookKernelStartBurstThrottle,+ NotebookKernelStartSustainedThrottle,+)from products.notebooks.backend.presentation.reusable_widget_serializers import (ReusableWidgetAttachRequestSerializer,ReusableWidgetDetailSerializer,@@ -729,6 +740,14 @@ class NotebookViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, ForbidDdef get_serializer_class(self) -> type[BaseSerializer]:return NotebookMinimalSerializer if self.action == "list" else NotebookSerializer+ def get_throttles(self) -> list[BaseThrottle]:+ throttles = super().get_throttles()+ if self.action == "create":+ return [*throttles, NotebookCreateBurstThrottle(), NotebookCreateSustainedThrottle()]+ if self.action in ("kernel_start", "kernel_restart"):+ return [*throttles, NotebookKernelStartBurstThrottle(), NotebookKernelStartSustainedThrottle()]+ return throttles+def _get_notebook_for_kernel(self) -> Notebook:if self.kwargs.get(self.lookup_field) == "scratchpad":notebook = Notebook(@@ -1542,6 +1561,8 @@ class NotebookViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, ForbidDnotebook = self._get_notebook_for_kernel()try:kernel_runtime = get_kernel_runtime(notebook, self._current_user()).ensure()+ except KernelSandboxLimitExceeded as err:+ return Response({"detail": str(err)}, status=429)except SandboxProvisionError:logger.exception("notebook_kernel_start_failed", notebook_short_id=notebook.short_id)return Response({"detail": "Failed to start notebook kernel."}, status=503)@@ -1565,6 +1586,8 @@ class NotebookViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, ForbidDnotebook = self._get_notebook_for_kernel()try:kernel_runtime = get_kernel_runtime(notebook, self._current_user()).restart()+ except KernelSandboxLimitExceeded as err:+ return Response({"detail": str(err)}, status=429)except SandboxProvisionError:logger.exception("notebook_kernel_restart_failed", notebook_short_id=notebook.short_id)return Response({"detail": "Failed to restart notebook kernel."}, status=503)@@ -1899,6 +1922,8 @@ class NotebookViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, ForbidDvariable_names=variable_names,timeout=serializer.validated_data.get("timeout"),)+ except KernelSandboxLimitExceeded as err:+ return Response({"detail": str(err)}, status=429)except SandboxProvisionError:logger.exception("notebook_kernel_execute_failed", notebook_short_id=notebook.short_id)return Response({"detail": "Failed to execute notebook code."}, status=503)diff --git a/products/notebooks/backend/test/test_kernel_runtime.py b/products/notebooks/backend/test/test_kernel_runtime.pyindex 71e2d61e6da2..83cbac984fd9 100644--- a/products/notebooks/backend/test/test_kernel_runtime.py+++ b/products/notebooks/backend/test/test_kernel_runtime.py@@ -14,8 +14,11 @@ from django.utils import timezonefrom parameterized import parameterizedfrom products.notebooks.backend.kernel_runtime import (+ NOTEBOOK_KERNEL_MAX_RUNNING_PER_TEAM,+ NOTEBOOK_KERNEL_STARTS_PER_USER_PER_HOUR,NOTEBOOK_KERNEL_TTL_SECONDS,KernelRuntimeService,+ KernelSandboxLimitExceeded,_KernelHandle,_NotebookBridgeParser,build_notebook_sandbox_config,@@ -233,6 +236,47 @@ class TestKernelRuntimeService(BaseTest):assert properties["ended_reason"] == KernelRuntime.Status.STOPPEDassert properties["sandbox_still_running"] is destroy_fails+ @parameterized.expand(+ [+ ("user_hourly_starts", "same_user_ended", NOTEBOOK_KERNEL_STARTS_PER_USER_PER_HOUR, True),+ ("team_running_sandboxes", "other_user_running", NOTEBOOK_KERNEL_MAX_RUNNING_PER_TEAM, True),+ ("ended_sandboxes_of_other_users", "other_user_ended", NOTEBOOK_KERNEL_MAX_RUNNING_PER_TEAM, False),+ ("under_both_limits", "same_user_ended", NOTEBOOK_KERNEL_STARTS_PER_USER_PER_HOUR - 1, False),+ ]+ )+ def test_ensure_kernel_refuses_new_sandbox_over_limit(+ self, _name: str, existing: str, count: int, refused: bool+ ) -> None:+ notebook = Notebook.objects.create(team=self.team)+ other_user = self._create_user("[email protected]")+ now = timezone.now()+ for _ in range(count):+ runtime = KernelRuntime.objects.create(+ team=self.team,+ notebook_short_id=notebook.short_id,+ user=self.user if existing == "same_user_ended" else other_user,+ status=KernelRuntime.Status.RUNNING,+ backend=KernelRuntime.Backend.MODAL,+ )+ KernelRuntime.objects.filter(pk=runtime.pk).update(+ ttl_expires_at=now + timedelta(minutes=30),+ ended_at=None if existing == "other_user_running" else now,+ )+ sandbox_class = MagicMock()+ sandbox_class.create.side_effect = RuntimeError("provider called")+ service = KernelRuntimeService()++ with (+ patch.object(service, "_get_backend", return_value=KernelRuntime.Backend.MODAL),+ patch.object(service, "_acquire_lock", return_value=_DummyLock()),+ patch.object(service, "_get_sandbox_class", return_value=sandbox_class),+ ):+ with self.assertRaises(KernelSandboxLimitExceeded if refused else RuntimeError) as ctx:+ service.ensure_kernel(notebook, self.user)++ assert isinstance(ctx.exception, KernelSandboxLimitExceeded) is refused+ assert sandbox_class.create.called is not refused+def test_execute_filters_invalid_variable_names(self) -> None:service = KernelRuntimeService()notebook = Notebook.objects.create(team=self.team)