:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import {
AccessControlLevel,
FeatureFlagBucketingIdentifier,
FeatureFlagEvaluationRuntime,
FeatureFlagFilters,
FeatureFlagGroupType,
FeatureFlagType,
FeatureFlagWithV1Config,
PropertyFilterType,
PropertyOperator,
} from '~/types'
import { contextItemPath } from 'products/posthog_ai/frontend/utils/contextItemLink'
import { wrapWithPosthogContext } from 'products/posthog_ai/frontend/utils/posthogContextBlock'
import {
FEATURE_FLAG_CLEANUP_SKILL,
featureFlagCleanupAssessmentContextItems,
featureFlagContextItems,
mutationTargetsFeatureFlag,
} from './featureFlagAiContext'
// The backend's own cap on a text attachment (MAX_TEXT_LENGTH in the posthog_ai backend). Asserting
// the literal rather than importing the frontend constant is deliberate: raising the frontend cap
// past what the backend accepts is exactly the regression these cases guard against.
const MAX_BACKEND_TEXT_LENGTH = 4096
describe('featureFlagAiContext', () => {
const baseFeatureFlag: FeatureFlagWithV1Config = {
id: 1,
key: 'test-flag',
name: '',
created_at: '2021-01-01',
updated_at: '2021-01-01',
created_by: null,
is_remote_configuration: false,
filters: { groups: [], payloads: {}, multivariate: null },
deleted: false,
archived: false,
active: true,
experiment_set: null,
experiment_set_metadata: null,
features: null,
surveys: null,
can_edit: true,
tags: [],
ensure_experience_continuity: null,
user_access_level: AccessControlLevel.Admin,
status: 'ACTIVE',
has_encrypted_payloads: false,
version: 0,
last_modified_by: null,
evaluation_runtime: FeatureFlagEvaluationRuntime.ALL,
evaluation_contexts: [],
bucketing_identifier: FeatureFlagBucketingIdentifier.DISTINCT_ID,
}
const personCondition: FeatureFlagGroupType = {
properties: [
{ key: 'email', value: 'is_set', operator: PropertyOperator.IsSet, type: PropertyFilterType.Person },
],
rollout_percentage: 50,
variant: null,
}
function withFilters(filters: Partial<FeatureFlagFilters>): FeatureFlagType {
return { ...baseFeatureFlag, filters: { ...baseFeatureFlag.filters, ...filters } }
}
function targetingValue(featureFlag: FeatureFlagType): string {
const item = featureFlagContextItems(featureFlag).find((item) => item.type === 'feature_flag_targeting')
if (!item?.value) {
throw new Error('no targeting item')
}
return item.value
}
test.each(['test-flag', '12'])('links flag %s by ID and keeps its context key', (key) => {
const items = featureFlagContextItems({ ...baseFeatureFlag, key })
expect(contextItemPath(items[0])).toBe('/feature-flags/1')
expect(wrapWithPosthogContext('Review this flag', items)).toContain(`feature_flag ${key}`)
})
it('renders the release conditions into the context block the agent reads', () => {
const block = wrapWithPosthogContext(
'who matches this flag?',
featureFlagContextItems(withFilters({ groups: [personCondition] }))
)
expect(block).toContain('feature_flag test-flag')
expect(block).toContain('release_conditions')
expect(block).toContain('email')
})
it('carries the targeting a group-targeted multivariate flag needs beyond its release conditions', () => {
const targeting = JSON.parse(
targetingValue(
withFilters({
groups: [personCondition],
aggregation_group_type_index: 0,
multivariate: { variants: [{ key: 'control', rollout_percentage: 50 }] },
holdout_groups: [personCondition],
})
)
)
expect(targeting.aggregation_group_type_index).toBe(0)
expect(targeting.multivariate.variants[0].key).toBe('control')
expect(targeting.holdout_groups).toHaveLength(1)
})
const flagWithPastedValues = withFilters({
groups: [
{
properties: [
{
key: 'email',
value: Array.from({ length: 500 }, (_, index) => `person${index}@example.com`),
operator: PropertyOperator.Exact,
type: PropertyFilterType.Person,
},
],
rollout_percentage: 100,
variant: null,
},
],
})
function rulesV2Flag(ruleCount: number): FeatureFlagType {
return {
...baseFeatureFlag,
filters: {
version: 2,
return_type: 'boolean',
default_value: false,
rules: Array.from({ length: ruleCount }, (_, index) => ({
id: `rule-${index}`,
rule_type: 'targeted_release' as const,
targeting: { properties: personCondition.properties ?? [] },
value: true,
})),
},
}
}
const oversized: [string, FeatureFlagType][] = [
['a condition holding hundreds of pasted values', flagWithPastedValues],
['a very long description', { ...withFilters({ groups: [personCondition] }), name: 'x'.repeat(20000) }],
['a rules v2 document with hundreds of rules', rulesV2Flag(300)],
]
test.each(oversized)('keeps the attachment under the backend limit with %s', (_name, featureFlag) => {
const value = targetingValue(featureFlag)
expect(value.length).toBeLessThanOrEqual(MAX_BACKEND_TEXT_LENGTH)
expect(() => JSON.parse(value)).not.toThrow()
})
it('summarizes the release conditions it drops to fit', () => {
const value = targetingValue(flagWithPastedValues)
expect(value).not.toContain('[email protected]')
// Without the count the agent can't tell a flag with no targeting from one whose targeting
// was too large to send.
expect(JSON.parse(value).release_condition_count).toBe(1)
})
it('attaches a rules v2 document whole, and only its config version once it is too large', () => {
const small = rulesV2Flag(1)
expect(JSON.parse(targetingValue(small)).config).toEqual(small.filters)
expect(JSON.parse(targetingValue(rulesV2Flag(300)))).toEqual({
key: 'test-flag',
active: true,
config_version: 2,
})
})
describe('featureFlagCleanupAssessmentContextItems', () => {
it('includes the cleanup skill for the agent', () => {
const items = featureFlagCleanupAssessmentContextItems(baseFeatureFlag, 123)
expect(items).toContainEqual(expect.objectContaining({ type: 'skill', key: FEATURE_FLAG_CLEANUP_SKILL }))
})
it('tells the agent this request is assessment only and not a green light to change the flag', () => {
const instructions = featureFlagCleanupAssessmentContextItems(baseFeatureFlag, 123).find(
(item) => item.type === 'instructions'
)
expect(instructions?.value).toContain(FEATURE_FLAG_CLEANUP_SKILL)
expect(instructions?.value).toMatch(/assessment only/i)
expect(instructions?.value).toMatch(/re-fetch its current definition/i)
expect(instructions?.value).toMatch(/do not.*change this flag/i)
})
it('never puts the flag key or description inside the trusted instruction text', () => {
// The flag's own key/name are project-authored strings a customer chose, so they must stay
// in the untrusted `feature_flag_cleanup_target` item, never interpolated into the trusted
// block the agent is told to follow unconditionally.
const flag = { ...baseFeatureFlag, key: 'a-distinctive-flag-key', name: 'a distinctive description' }
const instructions = featureFlagCleanupAssessmentContextItems(flag, 123).find(
(item) => item.type === 'instructions'
)
expect(instructions?.value).not.toContain('a-distinctive-flag-key')
expect(instructions?.value).not.toContain('a distinctive description')
})
it('carries the saved project, id, and key as untrusted data the agent can act on', () => {
const block = wrapWithPosthogContext(
'assess this flag',
featureFlagCleanupAssessmentContextItems(baseFeatureFlag, 123)
)
const target = featureFlagCleanupAssessmentContextItems(baseFeatureFlag, 123).find(
(item) => item.type === 'feature_flag_cleanup_target'
)
expect(JSON.parse(target?.value ?? '')).toEqual({
project_id: 123,
id: baseFeatureFlag.id,
key: baseFeatureFlag.key,
})
// The trusted block may name the item ("the feature_flag_cleanup_target item") without
// leaking its data - the flag's own key is the thing that must stay out of it.
const trustedBlock = block.match(/<posthog_trusted_context>[\s\S]*?<\/posthog_trusted_context>/)?.[0] ?? ''
expect(trustedBlock).not.toContain(baseFeatureFlag.key)
expect(block).toContain(baseFeatureFlag.key)
})
})
test.each([
['the same id as a number', { id: 1 }, true],
['the same id as a string', { id: '1' }, true],
// Every mutation tool declares these spellings as aliases for `id`, and the server resolves
// them after this matcher has already seen the raw arguments the model wrote.
['the flagId alias', { flagId: 1 }, true],
['the feature_flag_id alias', { feature_flag_id: '1' }, true],
['an alias naming another flag', { flagId: 2 }, false],
['another flag', { id: 2 }, false],
['no id', { key: 'test-flag' }, false],
['unparseable args', null, false],
])('matches a mutation against the open flag with %s', (_name, innerInput, expected) => {
expect(mutationTargetsFeatureFlag(innerInput, 1)).toBe(expected)
})
})