Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(notebooks): rate limit notebook create and kernel sandbox starts (#127056)

posthog[bot]40eb37a49fc8
diff --git a/products/notebooks/backend/kernel_runtime.py b/products/notebooks/backend/kernel_runtime.py
index cb627acd1470..12080fa35fba 100644
--- a/products/notebooks/backend/kernel_runtime.py
+++ b/products/notebooks/backend/kernel_runtime.py
@@ -43,6 +43,14 @@ logger = structlog.get_logger(__name__)
# (`SANDBOX_TTL_SECONDS`) is longer because it has to outlast an unattended agent run.
NOTEBOOK_KERNEL_TTL_SECONDS = 1 * 60 * 60
+# Each sandbox is paid compute until its TTL. A normal user starts a few a day.
+NOTEBOOK_KERNEL_STARTS_PER_USER_PER_HOUR = 30
+NOTEBOOK_KERNEL_MAX_RUNNING_PER_TEAM = 20
+
+
+class KernelSandboxLimitExceeded(RuntimeError):
+ pass
+
@dataclass
class KernelExecutionResult:
@@ -721,7 +729,26 @@ class KernelRuntimeService:
return KernelRuntime.Backend.DOCKER
return None
+ def _check_sandbox_limits(self, notebook: Notebook, user: User | None) -> None:
+ now = timezone.now()
+ if isinstance(user, User):
+ recent_starts = KernelRuntime.objects.filter(user=user, created_at__gte=now - timedelta(hours=1)).count()
+ if recent_starts >= NOTEBOOK_KERNEL_STARTS_PER_USER_PER_HOUR:
+ logger.warning("notebook_kernel_user_start_limit", team_id=notebook.team_id, user_id=user.id)
+ raise KernelSandboxLimitExceeded(
+ "You started too many notebook kernels in the last hour. Try again later."
+ )
+ running = KernelRuntime.objects.filter(
+ team_id=notebook.team_id, ended_at__isnull=True, ttl_expires_at__gt=now
+ ).count()
+ if running >= NOTEBOOK_KERNEL_MAX_RUNNING_PER_TEAM:
+ logger.warning("notebook_kernel_team_running_limit", team_id=notebook.team_id)
+ raise KernelSandboxLimitExceeded(
+ "This project has too many notebook kernels running. Stop a kernel or try again later."
+ )
+
def _create_kernel_handle(self, notebook: Notebook, user: User | None, backend: str) -> _KernelHandle:
+ self._check_sandbox_limits(notebook, user)
runtime = self._create_runtime(notebook, user, backend)
connection_file = f"/tmp/jupyter/kernel-{runtime.id}.json"
kernel_id = f"kernel-{runtime.id}"
diff --git a/products/notebooks/backend/presentation/notebook_throttles.py b/products/notebooks/backend/presentation/notebook_throttles.py
new file mode 100644
index 000000000000..6aec0d184818
--- /dev/null
+++ b/products/notebooks/backend/presentation/notebook_throttles.py
@@ -0,0 +1,22 @@
+from posthog.rate_limit import _UserBucketRateThrottle
+
+
+# Each new notebook can start a paid kernel sandbox, so these rates stop a scripted create loop.
+class NotebookCreateBurstThrottle(_UserBucketRateThrottle):
+ scope = "notebook_create_burst"
+ rate = "20/minute"
+
+
+class NotebookCreateSustainedThrottle(_UserBucketRateThrottle):
+ scope = "notebook_create_sustained"
+ rate = "120/hour"
+
+
+class NotebookKernelStartBurstThrottle(_UserBucketRateThrottle):
+ scope = "notebook_kernel_start_burst"
+ rate = "10/minute"
+
+
+class NotebookKernelStartSustainedThrottle(_UserBucketRateThrottle):
+ scope = "notebook_kernel_start_sustained"
+ rate = "60/hour"
diff --git a/products/notebooks/backend/presentation/views/notebook.py b/products/notebooks/backend/presentation/views/notebook.py
index 3d58eb125b41..fc025ef843ca 100644
--- a/products/notebooks/backend/presentation/views/notebook.py
+++ b/products/notebooks/backend/presentation/views/notebook.py
@@ -28,6 +28,7 @@ from rest_framework.exceptions import PermissionDenied
from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework.serializers import BaseSerializer
+from rest_framework.throttling import BaseThrottle
from posthog.hogql.direct_connection import get_direct_connection_source
@@ -115,8 +116,18 @@ from products.notebooks.backend.facade.widgets import (
set_widget_instance_version,
start_widget_generation,
)
-from products.notebooks.backend.kernel_runtime import build_notebook_sandbox_config, get_kernel_runtime
+from products.notebooks.backend.kernel_runtime import (
+ KernelSandboxLimitExceeded,
+ build_notebook_sandbox_config,
+ get_kernel_runtime,
+)
from products.notebooks.backend.models import KernelRuntime, Notebook, NotebookNodeRun
+from products.notebooks.backend.presentation.notebook_throttles import (
+ NotebookCreateBurstThrottle,
+ NotebookCreateSustainedThrottle,
+ NotebookKernelStartBurstThrottle,
+ NotebookKernelStartSustainedThrottle,
+)
from products.notebooks.backend.presentation.reusable_widget_serializers import (
ReusableWidgetAttachRequestSerializer,
ReusableWidgetDetailSerializer,
@@ -729,6 +740,14 @@ class NotebookViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, ForbidD
def get_serializer_class(self) -> type[BaseSerializer]:
return NotebookMinimalSerializer if self.action == "list" else NotebookSerializer
+ def get_throttles(self) -> list[BaseThrottle]:
+ throttles = super().get_throttles()
+ if self.action == "create":
+ return [*throttles, NotebookCreateBurstThrottle(), NotebookCreateSustainedThrottle()]
+ if self.action in ("kernel_start", "kernel_restart"):
+ return [*throttles, NotebookKernelStartBurstThrottle(), NotebookKernelStartSustainedThrottle()]
+ return throttles
+
def _get_notebook_for_kernel(self) -> Notebook:
if self.kwargs.get(self.lookup_field) == "scratchpad":
notebook = Notebook(
@@ -1542,6 +1561,8 @@ class NotebookViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, ForbidD
notebook = self._get_notebook_for_kernel()
try:
kernel_runtime = get_kernel_runtime(notebook, self._current_user()).ensure()
+ except KernelSandboxLimitExceeded as err:
+ return Response({"detail": str(err)}, status=429)
except SandboxProvisionError:
logger.exception("notebook_kernel_start_failed", notebook_short_id=notebook.short_id)
return Response({"detail": "Failed to start notebook kernel."}, status=503)
@@ -1565,6 +1586,8 @@ class NotebookViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, ForbidD
notebook = self._get_notebook_for_kernel()
try:
kernel_runtime = get_kernel_runtime(notebook, self._current_user()).restart()
+ except KernelSandboxLimitExceeded as err:
+ return Response({"detail": str(err)}, status=429)
except SandboxProvisionError:
logger.exception("notebook_kernel_restart_failed", notebook_short_id=notebook.short_id)
return Response({"detail": "Failed to restart notebook kernel."}, status=503)
@@ -1899,6 +1922,8 @@ class NotebookViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, ForbidD
variable_names=variable_names,
timeout=serializer.validated_data.get("timeout"),
)
+ except KernelSandboxLimitExceeded as err:
+ return Response({"detail": str(err)}, status=429)
except SandboxProvisionError:
logger.exception("notebook_kernel_execute_failed", notebook_short_id=notebook.short_id)
return Response({"detail": "Failed to execute notebook code."}, status=503)
diff --git a/products/notebooks/backend/test/test_kernel_runtime.py b/products/notebooks/backend/test/test_kernel_runtime.py
index 71e2d61e6da2..83cbac984fd9 100644
--- a/products/notebooks/backend/test/test_kernel_runtime.py
+++ b/products/notebooks/backend/test/test_kernel_runtime.py
@@ -14,8 +14,11 @@ from django.utils import timezone
from parameterized import parameterized
from products.notebooks.backend.kernel_runtime import (
+ NOTEBOOK_KERNEL_MAX_RUNNING_PER_TEAM,
+ NOTEBOOK_KERNEL_STARTS_PER_USER_PER_HOUR,
NOTEBOOK_KERNEL_TTL_SECONDS,
KernelRuntimeService,
+ KernelSandboxLimitExceeded,
_KernelHandle,
_NotebookBridgeParser,
build_notebook_sandbox_config,
@@ -233,6 +236,47 @@ class TestKernelRuntimeService(BaseTest):
assert properties["ended_reason"] == KernelRuntime.Status.STOPPED
assert properties["sandbox_still_running"] is destroy_fails
+ @parameterized.expand(
+ [
+ ("user_hourly_starts", "same_user_ended", NOTEBOOK_KERNEL_STARTS_PER_USER_PER_HOUR, True),
+ ("team_running_sandboxes", "other_user_running", NOTEBOOK_KERNEL_MAX_RUNNING_PER_TEAM, True),
+ ("ended_sandboxes_of_other_users", "other_user_ended", NOTEBOOK_KERNEL_MAX_RUNNING_PER_TEAM, False),
+ ("under_both_limits", "same_user_ended", NOTEBOOK_KERNEL_STARTS_PER_USER_PER_HOUR - 1, False),
+ ]
+ )
+ def test_ensure_kernel_refuses_new_sandbox_over_limit(
+ self, _name: str, existing: str, count: int, refused: bool
+ ) -> None:
+ notebook = Notebook.objects.create(team=self.team)
+ other_user = self._create_user("[email protected]")
+ now = timezone.now()
+ for _ in range(count):
+ runtime = KernelRuntime.objects.create(
+ team=self.team,
+ notebook_short_id=notebook.short_id,
+ user=self.user if existing == "same_user_ended" else other_user,
+ status=KernelRuntime.Status.RUNNING,
+ backend=KernelRuntime.Backend.MODAL,
+ )
+ KernelRuntime.objects.filter(pk=runtime.pk).update(
+ ttl_expires_at=now + timedelta(minutes=30),
+ ended_at=None if existing == "other_user_running" else now,
+ )
+ sandbox_class = MagicMock()
+ sandbox_class.create.side_effect = RuntimeError("provider called")
+ service = KernelRuntimeService()
+
+ with (
+ patch.object(service, "_get_backend", return_value=KernelRuntime.Backend.MODAL),
+ patch.object(service, "_acquire_lock", return_value=_DummyLock()),
+ patch.object(service, "_get_sandbox_class", return_value=sandbox_class),
+ ):
+ with self.assertRaises(KernelSandboxLimitExceeded if refused else RuntimeError) as ctx:
+ service.ensure_kernel(notebook, self.user)
+
+ assert isinstance(ctx.exception, KernelSandboxLimitExceeded) is refused
+ assert sandbox_class.create.called is not refused
+
def test_execute_filters_invalid_variable_names(self) -> None:
service = KernelRuntimeService()
notebook = Notebook.objects.create(team=self.team)