Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(signals): let the daily cap be the only scout chat limit (#127217)

posthog[bot]250fea3174b4
diff --git a/products/signals/backend/scout_chat.py b/products/signals/backend/scout_chat.py
index f7bcfcccebd8..c23da0af70b7 100644
--- a/products/signals/backend/scout_chat.py
+++ b/products/signals/backend/scout_chat.py
@@ -18,7 +18,6 @@ from rest_framework import exceptions, serializers, status, viewsets
from rest_framework.permissions import IsAuthenticated
from rest_framework.request import Request
from rest_framework.response import Response
-from rest_framework.throttling import UserRateThrottle
from posthog.api.mixins import validated_request
from posthog.api.routing import TeamAndOrgViewSetMixin
@@ -154,6 +153,11 @@ def consume_daily_attempt(key_prefix: str, scope_id: int | str, cap: int) -> boo
return attempts <= cap
+def seconds_until_next_utc_day() -> int:
+ """Seconds until the `consume_daily_attempt` window resets."""
+ return 86400 - int(time.time()) % 86400
+
+
def refund_daily_attempt(key_prefix: str, scope_id: int | str) -> None:
"""Give back one attempt consumed this UTC day, for a request that did no work."""
window = int(time.time()) // 86400
@@ -163,16 +167,6 @@ def refund_daily_attempt(key_prefix: str, scope_id: int | str) -> None:
pass
-class ScoutChatBurstRateThrottle(UserRateThrottle):
- scope = "signals_scout_chat_burst"
- rate = "2/hour"
-
-
-class ScoutChatSustainedRateThrottle(UserRateThrottle):
- scope = "signals_scout_chat_day"
- rate = "5/day"
-
-
class ScoutChatTaskCreateSerializer(serializers.Serializer):
chat_type = serializers.ChoiceField(
choices=sorted(SCOUT_CHAT_TEMPLATES),
@@ -245,7 +239,6 @@ class SignalScoutChatTaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet
scope_object = "task"
serializer_class = ScoutChatTaskSerializer
pagination_class = None
- throttle_classes = [ScoutChatBurstRateThrottle, ScoutChatSustainedRateThrottle]
# No model backs this endpoint; a queryset is still required by the team/org viewset mixin
# and `create` never reads it.
queryset = SignalScoutConfig.objects.unscoped()
@@ -303,7 +296,9 @@ class SignalScoutChatTaskViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet
prompt = _author_prompt_from_request(user_prompt)
if not consume_daily_attempt("signals_scout_chat_attempts", request.user.id, SCOUT_CHAT_DAILY_ATTEMPT_CAP):
- raise exceptions.Throttled(detail="You've reached today's limit for scout chats. Try again tomorrow.")
+ raise exceptions.Throttled(
+ wait=seconds_until_next_utc_day(), detail="You've reached today's limit for scout chats."
+ )
# Repo-less on purpose: these chats read PostHog data over MCP and never touch code.
# create_pr=False marks the session non-PR-opening, and the pending user message is
# self-delivered by the agent server on boot so the interactive run has a first turn.
diff --git a/products/signals/backend/test/test_scout_chat_api.py b/products/signals/backend/test/test_scout_chat_api.py
index ace3daf8f50f..92fe48f4b3d4 100644
--- a/products/signals/backend/test/test_scout_chat_api.py
+++ b/products/signals/backend/test/test_scout_chat_api.py
@@ -63,18 +63,26 @@ class TestScoutChatTaskAPI(APIBaseTest):
self.assertFalse(Task.objects.filter(origin_product=Task.OriginProduct.SIGNALS_CHAT).exists())
mock_workflow.assert_not_called()
- @patch("products.signals.backend.scout_chat.SCOUT_CHAT_DAILY_ATTEMPT_CAP", 0)
+ @patch("products.signals.backend.scout_chat.SCOUT_CHAT_DAILY_ATTEMPT_CAP", 3)
@patch("products.tasks.backend.temporal.client.execute_task_processing_workflow")
- def test_daily_attempt_cap_creates_no_task(self, mock_workflow):
- response = self.client.post(
- f"/api/projects/{self.team.id}/signals/scout/chat_tasks/",
- {"chat_type": "recent_signals"},
- format="json",
- )
+ def test_daily_attempt_cap_is_the_only_per_user_limit(self, _mock_workflow):
+ cache.clear()
+ statuses = [
+ self.client.post(
+ f"/api/projects/{self.team.id}/signals/scout/chat_tasks/",
+ {"chat_type": "recent_signals"},
+ format="json",
+ )
+ for _ in range(4)
+ ]
- self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS)
- self.assertFalse(Task.objects.filter(origin_product=Task.OriginProduct.SIGNALS_CHAT).exists())
- mock_workflow.assert_not_called()
+ self.assertEqual(
+ [r.status_code for r in statuses],
+ [status.HTTP_201_CREATED] * 3 + [status.HTTP_429_TOO_MANY_REQUESTS],
+ )
+ self.assertEqual(statuses[-1].json()["code"], "throttled")
+ self.assertTrue(0 < int(statuses[-1]["Retry-After"]) <= 86400)
+ self.assertEqual(Task.objects.filter(origin_product=Task.OriginProduct.SIGNALS_CHAT).count(), 3)
@parameterized.expand([("declined", False), ("undecided", None)])
@patch("products.tasks.backend.temporal.client.execute_task_processing_workflow")
diff --git a/products/signals/frontend/inbox/components/config/scouts/ScoutChatModal.tsx b/products/signals/frontend/inbox/components/config/scouts/ScoutChatModal.tsx
index 191152675fe0..4ca606ef2c42 100644
--- a/products/signals/frontend/inbox/components/config/scouts/ScoutChatModal.tsx
+++ b/products/signals/frontend/inbox/components/config/scouts/ScoutChatModal.tsx
@@ -53,7 +53,7 @@ export interface ScoutChatModalProps {
export function ScoutChatModal({ initialPrompt = '', onClose, onSwitchToForm }: ScoutChatModalProps): JSX.Element {
const [prompt, setPrompt] = useState(initialPrompt)
const [templateId, setTemplateId] = useState<string | null>(null)
- const { runningChatType, aiConsentDisabledReason } = useValues(scoutFleetLogic)
+ const { runningChatType, scoutChatDisabledReason } = useValues(scoutFleetLogic)
const { startScoutChatTask } = useActions(scoutFleetLogic)
const creationDisabledReason = useScoutCreateDisabledReason()
@@ -61,7 +61,7 @@ export function ScoutChatModal({ initialPrompt = '', onClose, onSwitchToForm }:
const trimmedPrompt = prompt.trim()
const startDisabledReason =
creationDisabledReason ??
- aiConsentDisabledReason ??
+ scoutChatDisabledReason ??
(runningChatType !== null && !isStarting ? 'Starting another task…' : null) ??
(trimmedPrompt ? null : 'Describe what the scout should watch')
diff --git a/products/signals/frontend/inbox/components/config/scouts/ScoutNewButton.tsx b/products/signals/frontend/inbox/components/config/scouts/ScoutNewButton.tsx
index b3544b0b194a..25c2ed3d42ae 100644
--- a/products/signals/frontend/inbox/components/config/scouts/ScoutNewButton.tsx
+++ b/products/signals/frontend/inbox/components/config/scouts/ScoutNewButton.tsx
@@ -47,11 +47,11 @@ export function ScoutNewButton({
size = 'small',
}: ScoutNewButtonProps): JSX.Element {
const [openModal, setOpenModal] = useState<OpenScoutModal | null>(null)
- const { runningChatType, aiConsentDisabledReason } = useValues(scoutFleetLogic)
+ const { runningChatType, scoutChatDisabledReason } = useValues(scoutFleetLogic)
const creationDisabledReason = useScoutCreateDisabledReason()
const todayRailNav = useFeatureFlag('TODAY_RAIL_NAV')
const chatDisabledReason =
- aiConsentDisabledReason ?? (runningChatType !== null ? 'Starting another task…' : undefined)
+ scoutChatDisabledReason ?? (runningChatType !== null ? 'Starting another task…' : undefined)
const openChat = (): void => {
captureScoutCreatePathChosen({ path: 'chat', surface })
diff --git a/products/signals/frontend/inbox/components/config/scouts/ScoutSuggestionCard.tsx b/products/signals/frontend/inbox/components/config/scouts/ScoutSuggestionCard.tsx
index 608cc4004000..2201dc2ae540 100644
--- a/products/signals/frontend/inbox/components/config/scouts/ScoutSuggestionCard.tsx
+++ b/products/signals/frontend/inbox/components/config/scouts/ScoutSuggestionCard.tsx
@@ -133,7 +133,7 @@ function SuggestionTags({ item }: { item: ScoutSuggestionItemApi }): JSX.Element
/** The row's primary action, which depends on its kind, the chat that refines it, and dismiss. */
function SuggestionActions({ item, surface, isBusy }: ScoutSuggestionCardProps & { isBusy: boolean }): JSX.Element {
- const { aiConsentDisabledReason, runningChatType } = useValues(scoutSuggestionsLogic)
+ const { scoutChatDisabledReason, runningChatType } = useValues(scoutSuggestionsLogic)
const { refineSuggestionWithAi, openCreateFromSuggestion, dismissSuggestion } = useActions(scoutSuggestionsLogic)
// A canonical pick opens the same form on the scout that already exists, so the person reads it
// before it runs, and submitting only writes the config. Creating a draft and refining with AI
@@ -143,7 +143,7 @@ function SuggestionActions({ item, surface, isBusy }: ScoutSuggestionCardProps &
? 'Starting a task…'
: runningChatType !== null
? 'Starting another task…'
- : (creationDisabledReason ?? aiConsentDisabledReason ?? undefined)
+ : (creationDisabledReason ?? scoutChatDisabledReason ?? undefined)
const isCanonical = item.kind === 'canonical'
return (
diff --git a/products/signals/frontend/inbox/components/config/scouts/ScoutSuggestionsStrip.tsx b/products/signals/frontend/inbox/components/config/scouts/ScoutSuggestionsStrip.tsx
index 87eba5d9da90..72dfae102d78 100644
--- a/products/signals/frontend/inbox/components/config/scouts/ScoutSuggestionsStrip.tsx
+++ b/products/signals/frontend/inbox/components/config/scouts/ScoutSuggestionsStrip.tsx
@@ -209,7 +209,7 @@ function SuggestionsSkeleton(): JSX.Element {
}
function SuggestWithAiLink(): JSX.Element {
- const { aiConsentDisabledReason, runningChatType } = useValues(scoutSuggestionsLogic)
+ const { aiConsentDisabledReason, scoutChatDisabledReason, runningChatType } = useValues(scoutSuggestionsLogic)
const { startScoutChatTask } = useActions(scoutSuggestionsLogic)
// The chat ends in a skill write, so it carries the same editor gate as the other authoring paths.
const creationDisabledReason = useScoutCreateDisabledReason()
@@ -223,7 +223,7 @@ function SuggestWithAiLink(): JSX.Element {
<Link
// The request only shows on screen once it navigates, so without this a second click
// lands in the gap and mints a second cloud task.
- disabledReason={runningChatType !== null ? 'Starting a task…' : undefined}
+ disabledReason={runningChatType !== null ? 'Starting a task…' : (scoutChatDisabledReason ?? undefined)}
onClick={() => startScoutChatTask('author_scout', 'scout authoring task')}
>
suggest a scout with AI
diff --git a/products/signals/frontend/inbox/components/config/scouts/ScoutsRosterActions.tsx b/products/signals/frontend/inbox/components/config/scouts/ScoutsRosterActions.tsx
index 97675957a217..40443dee46d9 100644
--- a/products/signals/frontend/inbox/components/config/scouts/ScoutsRosterActions.tsx
+++ b/products/signals/frontend/inbox/components/config/scouts/ScoutsRosterActions.tsx
@@ -58,7 +58,7 @@ export function ScoutsRosterActions(): JSX.Element {
* reopens the closed strip; with none it opens the authoring chat.
*/
function ShowSuggestionsButton(): JSX.Element | null {
- const { suggestButtonVisible, hasPicks, isRefreshing, suggestionSetLoading, aiConsentDisabledReason } =
+ const { suggestButtonVisible, hasPicks, isRefreshing, suggestionSetLoading, scoutChatDisabledReason } =
useValues(scoutSuggestionsLogic)
const { runningChatType } = useValues(scoutFleetLogic)
const { askForSuggestions } = useActions(scoutSuggestionsLogic)
@@ -69,7 +69,7 @@ function ShowSuggestionsButton(): JSX.Element | null {
return null
}
const busyReason = showSuggestionsBusyReason({ hasPicks, isRefreshing, suggestionSetLoading, runningChatType })
- const gateReason = hasPicks ? null : (creationDisabledReason ?? aiConsentDisabledReason)
+ const gateReason = hasPicks ? null : (creationDisabledReason ?? scoutChatDisabledReason)
if (todayRailNav) {
return (
<Button
@@ -138,7 +138,7 @@ function showSuggestionsBusyReason({
*/
function AskAboutScoutsMenu(): JSX.Element {
const { startScoutChatTask } = useActions(scoutFleetLogic)
- const { runningChatType, aiConsentDisabledReason } = useValues(scoutFleetLogic)
+ const { runningChatType, scoutChatDisabledReason } = useValues(scoutFleetLogic)
const prompts: { label: string; chatType: ScoutChatType }[] = [
{ label: 'How is my scout troop performing?', chatType: 'fleet_overview' },
{ label: 'What signals were emitted recently?', chatType: 'recent_signals' },
@@ -168,7 +168,7 @@ function AskAboutScoutsMenu(): JSX.Element {
{prompts.map(({ label, chatType }) => (
<DropdownMenuItem
key={chatType}
- disabled={anotherTaskIsStarting || isStarting || !!aiConsentDisabledReason}
+ disabled={anotherTaskIsStarting || isStarting || !!scoutChatDisabledReason}
onClick={() => startScoutChatTask(chatType, label)}
>
{label}
@@ -188,7 +188,7 @@ function AskAboutScoutsMenu(): JSX.Element {
? 'Starting another task…'
: isStarting
? 'Starting a task…'
- : (aiConsentDisabledReason ?? undefined),
+ : (scoutChatDisabledReason ?? undefined),
}))}
>
<LemonButton
diff --git a/products/signals/frontend/inbox/logics/scoutFleetLogic.test.ts b/products/signals/frontend/inbox/logics/scoutFleetLogic.test.ts
index 67e968b2c976..01093152f9e5 100644
--- a/products/signals/frontend/inbox/logics/scoutFleetLogic.test.ts
+++ b/products/signals/frontend/inbox/logics/scoutFleetLogic.test.ts
@@ -726,6 +726,30 @@ describe('scoutFleetLogic', () => {
expect(mockSignalsScoutChatTasksCreate).not.toHaveBeenCalled()
})
+ it('disables every scout chat after the daily cap refusal until its retry time', async () => {
+ jest.useFakeTimers()
+ try {
+ mockSignalsScoutChatTasksCreate.mockRejectedValue(
+ new ApiError('Request was throttled.', 429, new Headers({ 'Retry-After': '3600' }), {
+ code: 'throttled',
+ })
+ )
+
+ logic.actions.startScoutChatTask('author_scout', 'scout authoring task')
+ await jest.advanceTimersByTimeAsync(0)
+ expect(logic.values.scoutChatDisabledReason).toMatch(/^You've reached today's limit for scout chats/)
+
+ logic.actions.startScoutChatTask('fleet_overview', 'fleet overview task')
+ await jest.advanceTimersByTimeAsync(0)
+ expect(mockSignalsScoutChatTasksCreate).toHaveBeenCalledTimes(1)
+
+ await jest.advanceTimersByTimeAsync(3600 * 1000)
+ expect(logic.values.scoutChatDisabledReason).toBeNull()
+ } finally {
+ jest.useRealTimers()
+ }
+ })
+
it('reports roster filtering with the search length, not the term', async () => {
jest.useFakeTimers()
try {
diff --git a/products/signals/frontend/inbox/logics/scoutFleetLogic.ts b/products/signals/frontend/inbox/logics/scoutFleetLogic.ts
index 2eaa2785a6cc..3dd6c3480978 100644
--- a/products/signals/frontend/inbox/logics/scoutFleetLogic.ts
+++ b/products/signals/frontend/inbox/logics/scoutFleetLogic.ts
@@ -105,6 +105,19 @@ const SCOUT_RECENCY_TIMESTAMPS: Record<
last_run: (config) => config.last_run_at,
}
+/** When a refused scout chat may start again, from the `Retry-After` of the server's daily-cap 429. */
+function scoutChatRetryAtFromError(error: unknown): number | null {
+ if (!(error instanceof ApiError) || error.status !== 429 || error.code !== 'throttled') {
+ return null
+ }
+ const seconds = Number(error.headers?.get('Retry-After'))
+ return Number.isSafeInteger(seconds) && seconds > 0 ? Date.now() + seconds * 1000 : null
+}
+
+function scoutChatLimitReason(retryAt: number): string {
+ return `You've reached today's limit for scout chats. Try again after ${dayjs(retryAt).format('h:mm A')}.`
+}
+
function compareByRecency(firstTimestamp: string | null, secondTimestamp: string | null): number {
if (!firstTimestamp || !secondTimestamp) {
return firstTimestamp === secondTimestamp ? 0 : firstTimestamp ? -1 : 1
@@ -432,6 +445,8 @@ export interface scoutFleetLogicValues {
runsWindowLoadedOnce: boolean
runsWindowLoading: boolean
scoutBannerMessage: string | null
+ scoutChatDisabledReason: string | null
+ scoutChatRetryAt: number | null
scoutConfigs: SignalScoutConfig[] | null
scoutConfigsLoading: boolean
scoutCostRollups: Map<string, ScoutCostRollup>
@@ -666,6 +681,9 @@ export interface scoutFleetLogicActions {
setRunNotePopoverConfigId: (configId: string | null) => {
configId: string | null
}
+ setScoutChatRetryAt: (retryAt: number | null) => {
+ retryAt: number | null
+ }
setScoutEnabledFilter: (filter: ScoutEnabledFilter) => {
filter: ScoutEnabledFilter
}
@@ -756,6 +774,10 @@ export interface scoutFleetLogicMeta {
dataProcessingAccepted: boolean,
dataProcessingApprovalDisabledReason: string | null
) => string | null
+ scoutChatDisabledReason: (
+ aiConsentDisabledReason: string | null,
+ scoutChatRetryAt: number | null
+ ) => string | null
rollups: (scoutRuns: SignalScoutRunSummary[]) => Map<string, ScoutRollup>
scoutRunsCoverFleet: (scoutConfigs: SignalScoutConfigApi[] | null) => boolean
expensiveRunCostThreshold: (
@@ -883,6 +905,7 @@ export const scoutFleetLogic = kea<scoutFleetLogicType>([
}),
startScoutChatTaskSuccess: true,
startScoutChatTaskFailure: true,
+ setScoutChatRetryAt: (retryAt: number | null) => ({ retryAt }),
setScoutFleetSyncOutcome: (outcome: ScoutFleetSyncOutcome) => ({ outcome }),
mergeScoutRunCosts: (costs: Map<string, number>) => ({ costs }),
}),
@@ -1308,6 +1331,12 @@ export const scoutFleetLogic = kea<scoutFleetLogicType>([
startScoutChatTaskFailure: () => null,
},
],
+ scoutChatRetryAt: [
+ null as number | null,
+ {
+ setScoutChatRetryAt: (_, { retryAt }) => retryAt,
+ },
+ ],
// The enabled, tag, owner, and sort choices persist like the report list's filters, so a new
// visit opens the roster as the user left it. Search does not persist, the same as the report list.
selectedScoutTags: [
@@ -1509,6 +1538,12 @@ export const scoutFleetLogic = kea<scoutFleetLogicType>([
(dataProcessingAccepted: boolean, dataProcessingApprovalDisabledReason: string | null): string | null =>
aiConsentDisabledReason(dataProcessingAccepted, dataProcessingApprovalDisabledReason),
],
+ // Every scout chat entry point shares one per-user daily cap, so one refusal disables them all.
+ scoutChatDisabledReason: [
+ (s) => [s.aiConsentDisabledReason, s.scoutChatRetryAt],
+ (aiConsentDisabledReason: string | null, scoutChatRetryAt: number | null): string | null =>
+ aiConsentDisabledReason ?? (scoutChatRetryAt !== null ? scoutChatLimitReason(scoutChatRetryAt) : null),
+ ],
rollups: [
(s) => [s.scoutRuns],
(scoutRuns: SignalScoutRunSummary[]): Map<string, ScoutRollup> => computeScoutRollups(scoutRuns),
@@ -2044,8 +2079,8 @@ export const scoutFleetLogic = kea<scoutFleetLogicType>([
// template, then navigate to it. Not a live chat.
// The CTAs carry this as a `disabledReason`; this backstops the paths that don't go
// through a button press, since the endpoint enforces no consent of its own.
- if (values.aiConsentDisabledReason) {
- lemonToast.error(values.aiConsentDisabledReason)
+ if (values.scoutChatDisabledReason) {
+ lemonToast.error(values.scoutChatDisabledReason)
actions.startScoutChatTaskFailure()
return
}
@@ -2080,7 +2115,17 @@ export const scoutFleetLogic = kea<scoutFleetLogicType>([
actions.startScoutChatTaskSuccess()
router.actions.push(urls.taskDetail(task.task_id))
} catch (error: any) {
- lemonToast.error(error?.detail || error?.message || `Failed to start ${taskLabel}`)
+ const retryAt = scoutChatRetryAtFromError(error)
+ if (retryAt !== null) {
+ lemonToast.error(scoutChatLimitReason(retryAt))
+ actions.setScoutChatRetryAt(retryAt)
+ cache.disposables.add(() => {
+ const timeout = setTimeout(() => actions.setScoutChatRetryAt(null), retryAt - Date.now())
+ return () => clearTimeout(timeout)
+ }, 'scoutChatRetry')
+ } else {
+ lemonToast.error(error?.detail || error?.message || `Failed to start ${taskLabel}`)
+ }
actions.startScoutChatTaskFailure()
} finally {
cache.chatTaskStarting = false
diff --git a/products/signals/frontend/inbox/logics/scoutSuggestionsLogic.ts b/products/signals/frontend/inbox/logics/scoutSuggestionsLogic.ts
index 882fcab5fd68..a469f6957f73 100644
--- a/products/signals/frontend/inbox/logics/scoutSuggestionsLogic.ts
+++ b/products/signals/frontend/inbox/logics/scoutSuggestionsLogic.ts
@@ -68,6 +68,7 @@ const scanFromBatch = (suggestionSet: ScoutSuggestionSetApi | null): ScoutSugges
export interface scoutSuggestionsLogicValues {
aiConsentDisabledReason: string | null // scoutFleetLogic
runningChatType: ScoutChatType | null // scoutFleetLogic
+ scoutChatDisabledReason: string | null // scoutFleetLogic
scoutConfigs: SignalScoutConfig[] | null // scoutFleetLogic
currentProjectId: number | string // teamLogic
currentTeamId: number | null // teamLogic
@@ -259,7 +260,7 @@ export const scoutSuggestionsLogic = kea<scoutSuggestionsLogicType>([
connect(() => ({
values: [
scoutFleetLogic,
- ['scoutConfigs', 'aiConsentDisabledReason', 'runningChatType'],
+ ['scoutConfigs', 'aiConsentDisabledReason', 'scoutChatDisabledReason', 'runningChatType'],
teamLogic,
['currentTeamId', 'currentProjectId'],
],