:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(notebooks): lower kernel limits and show refusal reason (#128185)
Sinan Kucukkoseler10eceeebee1c
diff --git a/frontend/src/scenes/notebooks/Notebook/notebookJupyterLogic.test.ts b/frontend/src/scenes/notebooks/Notebook/notebookJupyterLogic.test.tsindex 668357edc44f..8d8547af0ff4 100644--- a/frontend/src/scenes/notebooks/Notebook/notebookJupyterLogic.test.ts+++ b/frontend/src/scenes/notebooks/Notebook/notebookJupyterLogic.test.ts@@ -1,7 +1,9 @@import { expectLogic } from 'kea-test-utils'import type { NotebookComponentBlockNode } from 'lib/components/MarkdownNotebook/types'+import { lemonToast } from 'lib/lemon-ui/LemonToast'+import { useMocks } from '~/mocks/jest'import { initKeaTests } from '~/test/init'import { notebookJupyterLogic } from './notebookJupyterLogic'@@ -41,6 +43,22 @@ describe('notebookJupyterLogic', () => {}).toMatchValues({ executionCounter: 1, executionCounts: { a: 3, b: 1 } })})+ it('shows the reason when the API refuses a restart', async () => {+ const detail = 'You started too many notebook kernels in the last hour. Try again later.'+ const toastSpy = jest.spyOn(lemonToast, 'error').mockImplementation()+ useMocks({+ post: { '/api/projects/:team_id/notebooks/:short_id/kernel/restart/': () => [429, { detail }] },+ })+ const logic = notebookJupyterLogic({ shortId: SHORT_ID })+ logic.mount()++ await expectLogic(logic, () => {+ logic.actions.restartKernel(false)+ }).toFinishAllListeners()++ expect(toastSpy).toHaveBeenCalledWith(detail)+ })+it('counts nothing for a notebook outside Jupyter mode', async () => {const logic = notebookJupyterLogic({ shortId: SHORT_ID })logic.mount()diff --git a/frontend/src/scenes/notebooks/Notebook/notebookJupyterLogic.ts b/frontend/src/scenes/notebooks/Notebook/notebookJupyterLogic.tsindex 70eb62fe70b1..e5b62bcf5af8 100644--- a/frontend/src/scenes/notebooks/Notebook/notebookJupyterLogic.ts+++ b/frontend/src/scenes/notebooks/Notebook/notebookJupyterLogic.ts@@ -8,7 +8,7 @@ import { lemonToast } from 'lib/lemon-ui/LemonToast'import { notebooksKernelRestartCreate } from 'products/notebooks/frontend/generated/api'-import { notebookKernelInfoLogic } from './notebookKernelInfoLogic'+import { kernelActionErrorMessage, notebookKernelInfoLogic } from './notebookKernelInfoLogic'import { notebookNodeStalenessLogic } from './notebookNodeStalenessLogic'import { notebookRunLogic } from './notebookRunLogic'@@ -173,8 +173,8 @@ export const notebookJupyterLogic = kea<notebookJupyterLogicType>([if (runAll) {notebookRunLogic.findMounted({ shortId: props.shortId })?.actions.startRun()}- } catch {- lemonToast.error('The kernel could not be restarted. Try again.')+ } catch (error) {+ lemonToast.error(kernelActionErrorMessage(error, 'The kernel could not be restarted. Try again.'))} finally {actions.setIsRestartingKernel(false)notebookKernelInfoLogic.findMounted({ shortId: props.shortId })?.actions.loadKernelInfo()diff --git a/frontend/src/scenes/notebooks/Notebook/notebookKernelInfoLogic.test.ts b/frontend/src/scenes/notebooks/Notebook/notebookKernelInfoLogic.test.tsindex a397feeb6e10..d34bb8195845 100644--- a/frontend/src/scenes/notebooks/Notebook/notebookKernelInfoLogic.test.ts+++ b/frontend/src/scenes/notebooks/Notebook/notebookKernelInfoLogic.test.ts@@ -1,10 +1,12 @@import api from 'lib/api'+import { ApiError } from 'lib/api-error'+import { lemonToast } from 'lib/lemon-ui/LemonToast'import { initKeaTests } from '~/test/init'import { notebooksKernelComputeOptionsRetrieve } from 'products/notebooks/frontend/generated/api'-import { notebookKernelInfoLogic } from './notebookKernelInfoLogic'+import { kernelActionErrorMessage, notebookKernelInfoLogic } from './notebookKernelInfoLogic'import type { NotebookLogicMode } from './notebookLogic'jest.mock('products/notebooks/frontend/generated/api', () => ({@@ -25,6 +27,9 @@ const COMPUTE_OPTIONS = {allowed_idle_timeout_seconds: [600, 1800],}+const SANDBOX_LIMIT_DETAIL = 'You started too many notebook kernels in the last hour. Try again later.'+const THROTTLE_DETAIL = 'Request was throttled. Expected available in 120 seconds.'+const setHidden = (hidden: boolean): void => {Object.defineProperty(document, 'hidden', { configurable: true, get: () => hidden })document.dispatchEvent(new Event('visibilitychange'))@@ -204,6 +209,56 @@ describe('notebookKernelInfoLogic', () => {expect(restartSpy).not.toHaveBeenCalled()})+ test.each([+ [+ 'a sandbox limit shows the sentence from the API',+ new ApiError('refused', 429, undefined, { detail: SANDBOX_LIMIT_DETAIL }),+ SANDBOX_LIMIT_DETAIL,+ ],+ [+ 'a throttle shows the wait from Retry-After',+ new ApiError('refused', 429, new Headers({ 'Retry-After': '120' }), { detail: THROTTLE_DETAIL }),+ 'Too many kernel starts and restarts. Try again in 2m.',+ ],+ [+ 'a throttle with no Retry-After still shows a sentence',+ new ApiError('refused', 429, undefined, { detail: THROTTLE_DETAIL }),+ 'Too many kernel starts and restarts. Try again later.',+ ],+ [+ 'a provider failure keeps the fallback',+ new ApiError('failed', 503, undefined, { detail: 'Failed to start notebook kernel.' }),+ 'fallback',+ ],+ ['a network failure keeps the fallback', new Error('offline'), 'fallback'],+ ])('kernelActionErrorMessage: %s', (_, error, expected) => {+ expect(kernelActionErrorMessage(error, 'fallback')).toEqual(expected)+ })++ test.each(['kernelStart', 'kernelRestart'] as const)(+ 'shows the refusal and refreshes the panel when %s is refused',+ async (endpoint) => {+ const toastSpy = jest.spyOn(lemonToast, 'error').mockImplementation()+ jest.spyOn(api.notebooks, endpoint).mockRejectedValue(+ new ApiError('refused', 429, undefined, { detail: SANDBOX_LIMIT_DETAIL })+ )+ logic = notebookKernelInfoLogic({ shortId: `refused-${endpoint}-01890abc`, mode: 'notebook' })+ logic.mount()+ await jest.advanceTimersByTimeAsync(0)+ kernelStatusSpy.mockClear()++ if (endpoint === 'kernelStart') {+ logic.actions.startKernel()+ } else {+ logic.actions.restartKernel()+ }+ await jest.advanceTimersByTimeAsync(0)++ expect(toastSpy).toHaveBeenCalledWith(SANDBOX_LIMIT_DETAIL)+ expect(kernelStatusSpy).toHaveBeenCalled()+ }+ )+test('a shared notebook issues no team-scoped kernel requests', async () => {// A shared view renders from cachedNotebook so a logged-out viewer makes no team-scoped// call. Both kernel endpoints are team-scoped, so mounting here used to 401 twice.diff --git a/frontend/src/scenes/notebooks/Notebook/notebookKernelInfoLogic.ts b/frontend/src/scenes/notebooks/Notebook/notebookKernelInfoLogic.tsindex 552e595e7cdf..77882885b31d 100644--- a/frontend/src/scenes/notebooks/Notebook/notebookKernelInfoLogic.ts+++ b/frontend/src/scenes/notebooks/Notebook/notebookKernelInfoLogic.ts@@ -16,7 +16,9 @@ import { loaders } from 'kea-loaders'import api from 'lib/api'import { ApiConfig } from 'lib/api'+import { ApiError } from 'lib/api-error'import { FEATURE_FLAGS } from 'lib/constants'+import { lemonToast } from 'lib/lemon-ui/LemonToast'import { FeatureFlagsSet, featureFlagLogic } from 'lib/logic/featureFlagLogic'import { notebooksKernelComputeOptionsRetrieve } from 'products/notebooks/frontend/generated/api'@@ -28,6 +30,18 @@ import type {import { PythonKernelExecuteResponse } from '../Nodes/pythonExecution'import type { NotebookLogicMode } from './notebookLogic'+/** A refused kernel start can stay refused for up to an hour, so "try again" copy would mislead the user. */+export function kernelActionErrorMessage(error: unknown, fallback: string): string {+ if (!(error instanceof ApiError) || error.status !== 429) {+ return fallback+ }+ // The default throttle text counts raw seconds. A sandbox limit sends a sentence written for the user.+ if (error.detail && !error.detail.startsWith('Request was throttled')) {+ return error.detail+ }+ return `Too many kernel starts and restarts. Try again ${error.formattedRetryAfter ?? 'later'}.`+}+/** A DuckDB object a SQL node can currently SELECT from, as reported by the kernel's last run. */export type NotebookKernelFrame = {name: string@@ -667,6 +681,8 @@ export const notebookKernelInfoLogic = kea<notebookKernelInfoLogicType>([startKernel: async () => {try {await api.notebooks.kernelStart(props.shortId)+ } catch (error) {+ lemonToast.error(kernelActionErrorMessage(error, 'The kernel could not be started. Try again.'))} finally {actions.loadKernelInfo()}@@ -681,6 +697,8 @@ export const notebookKernelInfoLogic = kea<notebookKernelInfoLogicType>([restartKernel: async () => {try {await api.notebooks.kernelRestart(props.shortId)+ } catch (error) {+ lemonToast.error(kernelActionErrorMessage(error, 'The kernel could not be restarted. Try again.'))} finally {actions.loadKernelInfo()}diff --git a/products/notebooks/backend/kernel_runtime.py b/products/notebooks/backend/kernel_runtime.pyindex 12080fa35fba..f537e40dd22b 100644--- a/products/notebooks/backend/kernel_runtime.py+++ b/products/notebooks/backend/kernel_runtime.py@@ -44,7 +44,7 @@ logger = structlog.get_logger(__name__)NOTEBOOK_KERNEL_TTL_SECONDS = 1 * 60 * 60# Each sandbox is paid compute until its TTL. A normal user starts a few a day.-NOTEBOOK_KERNEL_STARTS_PER_USER_PER_HOUR = 30+NOTEBOOK_KERNEL_STARTS_PER_USER_PER_HOUR = 10NOTEBOOK_KERNEL_MAX_RUNNING_PER_TEAM = 20diff --git a/products/notebooks/backend/presentation/notebook_throttles.py b/products/notebooks/backend/presentation/notebook_throttles.pyindex 6aec0d184818..d779fedadf10 100644--- a/products/notebooks/backend/presentation/notebook_throttles.py+++ b/products/notebooks/backend/presentation/notebook_throttles.py@@ -4,19 +4,19 @@ from posthog.rate_limit import _UserBucketRateThrottle# Each new notebook can start a paid kernel sandbox, so these rates stop a scripted create loop.class NotebookCreateBurstThrottle(_UserBucketRateThrottle):scope = "notebook_create_burst"- rate = "20/minute"+ rate = "10/minute"class NotebookCreateSustainedThrottle(_UserBucketRateThrottle):scope = "notebook_create_sustained"- rate = "120/hour"+ rate = "30/hour"class NotebookKernelStartBurstThrottle(_UserBucketRateThrottle):scope = "notebook_kernel_start_burst"- rate = "10/minute"+ rate = "5/minute"class NotebookKernelStartSustainedThrottle(_UserBucketRateThrottle):scope = "notebook_kernel_start_sustained"- rate = "60/hour"+ rate = "20/hour"