Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(notebooks): lower kernel limits and show refusal reason (#128185)

Sinan Kucukkoseler10eceeebee1c
diff --git a/frontend/src/scenes/notebooks/Notebook/notebookJupyterLogic.test.ts b/frontend/src/scenes/notebooks/Notebook/notebookJupyterLogic.test.ts
index 668357edc44f..8d8547af0ff4 100644
--- a/frontend/src/scenes/notebooks/Notebook/notebookJupyterLogic.test.ts
+++ b/frontend/src/scenes/notebooks/Notebook/notebookJupyterLogic.test.ts
@@ -1,7 +1,9 @@
import { expectLogic } from 'kea-test-utils'
import type { NotebookComponentBlockNode } from 'lib/components/MarkdownNotebook/types'
+import { lemonToast } from 'lib/lemon-ui/LemonToast'
+import { useMocks } from '~/mocks/jest'
import { initKeaTests } from '~/test/init'
import { notebookJupyterLogic } from './notebookJupyterLogic'
@@ -41,6 +43,22 @@ describe('notebookJupyterLogic', () => {
}).toMatchValues({ executionCounter: 1, executionCounts: { a: 3, b: 1 } })
})
+ it('shows the reason when the API refuses a restart', async () => {
+ const detail = 'You started too many notebook kernels in the last hour. Try again later.'
+ const toastSpy = jest.spyOn(lemonToast, 'error').mockImplementation()
+ useMocks({
+ post: { '/api/projects/:team_id/notebooks/:short_id/kernel/restart/': () => [429, { detail }] },
+ })
+ const logic = notebookJupyterLogic({ shortId: SHORT_ID })
+ logic.mount()
+
+ await expectLogic(logic, () => {
+ logic.actions.restartKernel(false)
+ }).toFinishAllListeners()
+
+ expect(toastSpy).toHaveBeenCalledWith(detail)
+ })
+
it('counts nothing for a notebook outside Jupyter mode', async () => {
const logic = notebookJupyterLogic({ shortId: SHORT_ID })
logic.mount()
diff --git a/frontend/src/scenes/notebooks/Notebook/notebookJupyterLogic.ts b/frontend/src/scenes/notebooks/Notebook/notebookJupyterLogic.ts
index 70eb62fe70b1..e5b62bcf5af8 100644
--- a/frontend/src/scenes/notebooks/Notebook/notebookJupyterLogic.ts
+++ b/frontend/src/scenes/notebooks/Notebook/notebookJupyterLogic.ts
@@ -8,7 +8,7 @@ import { lemonToast } from 'lib/lemon-ui/LemonToast'
import { notebooksKernelRestartCreate } from 'products/notebooks/frontend/generated/api'
-import { notebookKernelInfoLogic } from './notebookKernelInfoLogic'
+import { kernelActionErrorMessage, notebookKernelInfoLogic } from './notebookKernelInfoLogic'
import { notebookNodeStalenessLogic } from './notebookNodeStalenessLogic'
import { notebookRunLogic } from './notebookRunLogic'
@@ -173,8 +173,8 @@ export const notebookJupyterLogic = kea<notebookJupyterLogicType>([
if (runAll) {
notebookRunLogic.findMounted({ shortId: props.shortId })?.actions.startRun()
}
- } catch {
- lemonToast.error('The kernel could not be restarted. Try again.')
+ } catch (error) {
+ lemonToast.error(kernelActionErrorMessage(error, 'The kernel could not be restarted. Try again.'))
} finally {
actions.setIsRestartingKernel(false)
notebookKernelInfoLogic.findMounted({ shortId: props.shortId })?.actions.loadKernelInfo()
diff --git a/frontend/src/scenes/notebooks/Notebook/notebookKernelInfoLogic.test.ts b/frontend/src/scenes/notebooks/Notebook/notebookKernelInfoLogic.test.ts
index a397feeb6e10..d34bb8195845 100644
--- a/frontend/src/scenes/notebooks/Notebook/notebookKernelInfoLogic.test.ts
+++ b/frontend/src/scenes/notebooks/Notebook/notebookKernelInfoLogic.test.ts
@@ -1,10 +1,12 @@
import api from 'lib/api'
+import { ApiError } from 'lib/api-error'
+import { lemonToast } from 'lib/lemon-ui/LemonToast'
import { initKeaTests } from '~/test/init'
import { notebooksKernelComputeOptionsRetrieve } from 'products/notebooks/frontend/generated/api'
-import { notebookKernelInfoLogic } from './notebookKernelInfoLogic'
+import { kernelActionErrorMessage, notebookKernelInfoLogic } from './notebookKernelInfoLogic'
import type { NotebookLogicMode } from './notebookLogic'
jest.mock('products/notebooks/frontend/generated/api', () => ({
@@ -25,6 +27,9 @@ const COMPUTE_OPTIONS = {
allowed_idle_timeout_seconds: [600, 1800],
}
+const SANDBOX_LIMIT_DETAIL = 'You started too many notebook kernels in the last hour. Try again later.'
+const THROTTLE_DETAIL = 'Request was throttled. Expected available in 120 seconds.'
+
const setHidden = (hidden: boolean): void => {
Object.defineProperty(document, 'hidden', { configurable: true, get: () => hidden })
document.dispatchEvent(new Event('visibilitychange'))
@@ -204,6 +209,56 @@ describe('notebookKernelInfoLogic', () => {
expect(restartSpy).not.toHaveBeenCalled()
})
+ test.each([
+ [
+ 'a sandbox limit shows the sentence from the API',
+ new ApiError('refused', 429, undefined, { detail: SANDBOX_LIMIT_DETAIL }),
+ SANDBOX_LIMIT_DETAIL,
+ ],
+ [
+ 'a throttle shows the wait from Retry-After',
+ new ApiError('refused', 429, new Headers({ 'Retry-After': '120' }), { detail: THROTTLE_DETAIL }),
+ 'Too many kernel starts and restarts. Try again in 2m.',
+ ],
+ [
+ 'a throttle with no Retry-After still shows a sentence',
+ new ApiError('refused', 429, undefined, { detail: THROTTLE_DETAIL }),
+ 'Too many kernel starts and restarts. Try again later.',
+ ],
+ [
+ 'a provider failure keeps the fallback',
+ new ApiError('failed', 503, undefined, { detail: 'Failed to start notebook kernel.' }),
+ 'fallback',
+ ],
+ ['a network failure keeps the fallback', new Error('offline'), 'fallback'],
+ ])('kernelActionErrorMessage: %s', (_, error, expected) => {
+ expect(kernelActionErrorMessage(error, 'fallback')).toEqual(expected)
+ })
+
+ test.each(['kernelStart', 'kernelRestart'] as const)(
+ 'shows the refusal and refreshes the panel when %s is refused',
+ async (endpoint) => {
+ const toastSpy = jest.spyOn(lemonToast, 'error').mockImplementation()
+ jest.spyOn(api.notebooks, endpoint).mockRejectedValue(
+ new ApiError('refused', 429, undefined, { detail: SANDBOX_LIMIT_DETAIL })
+ )
+ logic = notebookKernelInfoLogic({ shortId: `refused-${endpoint}-01890abc`, mode: 'notebook' })
+ logic.mount()
+ await jest.advanceTimersByTimeAsync(0)
+ kernelStatusSpy.mockClear()
+
+ if (endpoint === 'kernelStart') {
+ logic.actions.startKernel()
+ } else {
+ logic.actions.restartKernel()
+ }
+ await jest.advanceTimersByTimeAsync(0)
+
+ expect(toastSpy).toHaveBeenCalledWith(SANDBOX_LIMIT_DETAIL)
+ expect(kernelStatusSpy).toHaveBeenCalled()
+ }
+ )
+
test('a shared notebook issues no team-scoped kernel requests', async () => {
// A shared view renders from cachedNotebook so a logged-out viewer makes no team-scoped
// call. Both kernel endpoints are team-scoped, so mounting here used to 401 twice.
diff --git a/frontend/src/scenes/notebooks/Notebook/notebookKernelInfoLogic.ts b/frontend/src/scenes/notebooks/Notebook/notebookKernelInfoLogic.ts
index 552e595e7cdf..77882885b31d 100644
--- a/frontend/src/scenes/notebooks/Notebook/notebookKernelInfoLogic.ts
+++ b/frontend/src/scenes/notebooks/Notebook/notebookKernelInfoLogic.ts
@@ -16,7 +16,9 @@ import { loaders } from 'kea-loaders'
import api from 'lib/api'
import { ApiConfig } from 'lib/api'
+import { ApiError } from 'lib/api-error'
import { FEATURE_FLAGS } from 'lib/constants'
+import { lemonToast } from 'lib/lemon-ui/LemonToast'
import { FeatureFlagsSet, featureFlagLogic } from 'lib/logic/featureFlagLogic'
import { notebooksKernelComputeOptionsRetrieve } from 'products/notebooks/frontend/generated/api'
@@ -28,6 +30,18 @@ import type {
import { PythonKernelExecuteResponse } from '../Nodes/pythonExecution'
import type { NotebookLogicMode } from './notebookLogic'
+/** A refused kernel start can stay refused for up to an hour, so "try again" copy would mislead the user. */
+export function kernelActionErrorMessage(error: unknown, fallback: string): string {
+ if (!(error instanceof ApiError) || error.status !== 429) {
+ return fallback
+ }
+ // The default throttle text counts raw seconds. A sandbox limit sends a sentence written for the user.
+ if (error.detail && !error.detail.startsWith('Request was throttled')) {
+ return error.detail
+ }
+ return `Too many kernel starts and restarts. Try again ${error.formattedRetryAfter ?? 'later'}.`
+}
+
/** A DuckDB object a SQL node can currently SELECT from, as reported by the kernel's last run. */
export type NotebookKernelFrame = {
name: string
@@ -667,6 +681,8 @@ export const notebookKernelInfoLogic = kea<notebookKernelInfoLogicType>([
startKernel: async () => {
try {
await api.notebooks.kernelStart(props.shortId)
+ } catch (error) {
+ lemonToast.error(kernelActionErrorMessage(error, 'The kernel could not be started. Try again.'))
} finally {
actions.loadKernelInfo()
}
@@ -681,6 +697,8 @@ export const notebookKernelInfoLogic = kea<notebookKernelInfoLogicType>([
restartKernel: async () => {
try {
await api.notebooks.kernelRestart(props.shortId)
+ } catch (error) {
+ lemonToast.error(kernelActionErrorMessage(error, 'The kernel could not be restarted. Try again.'))
} finally {
actions.loadKernelInfo()
}
diff --git a/products/notebooks/backend/kernel_runtime.py b/products/notebooks/backend/kernel_runtime.py
index 12080fa35fba..f537e40dd22b 100644
--- a/products/notebooks/backend/kernel_runtime.py
+++ b/products/notebooks/backend/kernel_runtime.py
@@ -44,7 +44,7 @@ logger = structlog.get_logger(__name__)
NOTEBOOK_KERNEL_TTL_SECONDS = 1 * 60 * 60
# Each sandbox is paid compute until its TTL. A normal user starts a few a day.
-NOTEBOOK_KERNEL_STARTS_PER_USER_PER_HOUR = 30
+NOTEBOOK_KERNEL_STARTS_PER_USER_PER_HOUR = 10
NOTEBOOK_KERNEL_MAX_RUNNING_PER_TEAM = 20
diff --git a/products/notebooks/backend/presentation/notebook_throttles.py b/products/notebooks/backend/presentation/notebook_throttles.py
index 6aec0d184818..d779fedadf10 100644
--- a/products/notebooks/backend/presentation/notebook_throttles.py
+++ b/products/notebooks/backend/presentation/notebook_throttles.py
@@ -4,19 +4,19 @@ from posthog.rate_limit import _UserBucketRateThrottle
# Each new notebook can start a paid kernel sandbox, so these rates stop a scripted create loop.
class NotebookCreateBurstThrottle(_UserBucketRateThrottle):
scope = "notebook_create_burst"
- rate = "20/minute"
+ rate = "10/minute"
class NotebookCreateSustainedThrottle(_UserBucketRateThrottle):
scope = "notebook_create_sustained"
- rate = "120/hour"
+ rate = "30/hour"
class NotebookKernelStartBurstThrottle(_UserBucketRateThrottle):
scope = "notebook_kernel_start_burst"
- rate = "10/minute"
+ rate = "5/minute"
class NotebookKernelStartSustainedThrottle(_UserBucketRateThrottle):
scope = "notebook_kernel_start_sustained"
- rate = "60/hour"
+ rate = "20/hour"