Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

18.8 KB
Raw Blame
import { deepEqual as equal } from 'fast-equals'
import { useActions, useValues } from 'kea'
import { useState } from 'react'

import { IconPlus } from '@posthog/icons'

import { RestrictionScope, useRestrictedArea } from 'lib/components/RestrictedArea'
import { VerticalNestedDND } from 'lib/components/VerticalNestedDND/VerticalNestedDND'
import { TeamMembershipLevel } from 'lib/constants'
import { LemonBanner } from 'lib/lemon-ui/LemonBanner'
import { LemonButton } from 'lib/lemon-ui/LemonButton'
import { LemonInput } from 'lib/lemon-ui/LemonInput'
import { LemonLabel } from 'lib/lemon-ui/LemonLabel'
import { LemonSelect } from 'lib/lemon-ui/LemonSelect'
import { LemonTag } from 'lib/lemon-ui/LemonTag'
import { Link } from 'lib/lemon-ui/Link'
import { uuid } from 'lib/utils/dom'
import { teamLogic } from 'scenes/teamLogic'

import { CustomBotCondition, CustomBotField, CustomBotMatcher, CustomBotRule } from '~/queries/schema/schema-general'
import { FilterLogicalOperator } from '~/types'

import {
    CUSTOM_BOT_CATEGORY,
    CUSTOM_BOT_CATEGORY_OPTIONS,
    CUSTOM_BOT_FIELD_OPTIONS,
    MAX_CONDITIONS_PER_RULE,
    MAX_CUSTOM_BOT_RULES,
    MAX_TOTAL_CONDITIONS,
    defaultMatcherFor,
    fieldLabel,
    matcherLabel,
    matcherOptionsFor,
    patternPlaceholderFor,
    ruleMatchesValues,
    sanitizeCustomBotRules,
    parseCustomBotRules,
    validateCustomBotCondition,
    validateCustomBotRule,
    validateCustomBotRuleSet,
} from './customBotRulesUtils'

// Lowercase: the labels render mid-sentence ("when all conditions are met").
const combinerOptions = [
    { label: 'all', value: FilterLogicalOperator.And },
    { label: 'any', value: FilterLogicalOperator.Or },
]

function categoryLabel(category: string | undefined): string {
    const value = category || CUSTOM_BOT_CATEGORY
    return CUSTOM_BOT_CATEGORY_OPTIONS.find((option) => option.value === value)?.label ?? value
}

/** The list without edit affordances: the shared editor always renders drag handles and delete
buttons, which would look actionable to someone whose changes can never be saved. */
function ReadOnlyBotRules({ rules }: { rules: CustomBotRule[] }): JSX.Element {
    return (
        <div className="flex flex-col gap-2">
            {rules.map((rule) => (
                <div key={rule.id} className="border rounded p-3 flex flex-col gap-1">
                    <div className="flex items-center gap-2 flex-wrap">
                        <span className="font-semibold">{rule.name}</span>
                        <LemonTag>{categoryLabel(rule.category)}</LemonTag>
                    </div>
                    {rule.items.length > 1 ? (
                        <span className="text-muted text-xs">
                            When {rule.combiner === FilterLogicalOperator.Or ? 'any' : 'all'} of these conditions are
                            met:
                        </span>
                    ) : null}
                    {rule.items.map((condition) => (
                        <span key={condition.id} className="text-sm">
                            {fieldLabel(condition.key)} {matcherLabel(condition.matcher)}{' '}
                            <code>{condition.pattern}</code>
                        </span>
                    ))}
                </div>
            ))}
        </div>
    )
}

function newCondition(): CustomBotCondition {
    return {
        id: uuid(),
        key: CustomBotField.RawUserAgent,
        matcher: CustomBotMatcher.Contains,
        pattern: '',
    }
}

function newRule(): CustomBotRule {
    return {
        id: uuid(),
        name: '',
        category: CUSTOM_BOT_CATEGORY,
        combiner: FilterLogicalOperator.And,
        items: [newCondition()],
    }
}

export function CustomBotRules(): JSX.Element {
    const { currentTeam, currentTeamLoading } = useValues(teamLogic)
    const { updateCurrentTeam } = useActions(teamLogic)

    // The saved state is whatever the server currently holds, so a save that the backend rejects
    // leaves the editor dirty and retryable instead of falsely reading as saved.
    const rawSavedRules = currentTeam?.modifiers?.customBotDefinitions
    const savedRules = parseCustomBotRules(rawSavedRules)
    // A save writes the parsed list back in full, so entries that did not parse would be removed
    // silently. Tell the user instead of losing them without a trace.
    const droppedCount = (Array.isArray(rawSavedRules) ? rawSavedRules.length : 0) - savedRules.length
    const [rules, setRules] = useState<CustomBotRule[]>(savedRules)
    const [testValues, setTestValues] = useState<Partial<Record<CustomBotField, string>>>({})

    const restrictedReason = useRestrictedArea({
        scope: RestrictionScope.Project,
        minimumAccessLevel: TeamMembershipLevel.Admin,
    })
    const canEdit = !restrictedReason

    const anyRuleError = rules.map(validateCustomBotRule).find(Boolean)
    const setError = validateCustomBotRuleSet(rules)
    const totalConditions = rules.reduce((sum, rule) => sum + rule.items.length, 0)
    // Sanitize both sides: a rule written through the API can lack the optional category or carry
    // unpadded whitespace, and a pristine editor must not read as dirty for normalization alone.
    const isUnchanged = equal(sanitizeCustomBotRules(rules), sanitizeCustomBotRules(savedRules))
    const usedFields = CUSTOM_BOT_FIELD_OPTIONS.filter((option) =>
        rules.some((rule) => rule.items.some((condition) => condition.key === option.value))
    )
    const matched = rules.filter((rule) => ruleMatchesValues(rule, testValues))
    // Only values for a property still in use count as test input, so removing a rule does not leave
    // a stale value showing a phantom "no match".
    const hasTestInput = usedFields.some((field) => testValues[field.value]?.trim())
    // $ip is dropped on ingest when a project anonymizes IPs, so a range would never match.
    const ipRulesAreDead =
        currentTeam?.anonymize_ips &&
        rules.some((rule) => rule.items.some((condition) => condition.key === CustomBotField.IP))

    const save = (): void => {
        // On success the team reloads with these rules and isUnchanged flips to true; on a
        // rejected save the team is unchanged, so the editor stays dirty and the error is actionable.
        updateCurrentTeam({
            modifiers: { ...currentTeam?.modifiers, customBotDefinitions: sanitizeCustomBotRules(rules) },
        })
    }

    return (
        <div className="flex flex-col gap-4">
            <p className="mb-0">
                A bot you add here counts as a bot everywhere <code>Is bot</code> is available, including insights, web
                analytics, and SQL. PostHog's built-in list already covers crawlers that identify themselves, like
                GPTBot and Googlebot. Your rules are checked first, so you can give one of those a different name or
                category.
            </p>
            <p className="mb-0">
                Match the user agent to catch a crawler that names itself, the IP address to catch one that sends a
                browser user agent from a range you know, or combine conditions to catch one that only stands out
                through a combination, like an 800x600 screen.{' '}
                <Link to="https://posthog.com/docs/web-analytics/bot-detection">Read more about bot detection</Link>
            </p>

            {rules.length === 0 ? (
                <p className="text-muted mb-0">
                    No bots added yet. PostHog's built-in list still applies. Add one to extend it.
                </p>
            ) : null}

            {droppedCount > 0 ? (
                <LemonBanner type="warning">
                    {droppedCount === 1 ? 'One saved rule' : `${droppedCount} saved rules`} could not be read and will
                    be removed if you save. Contact support if you did not expect this.
                </LemonBanner>
            ) : null}

            {!canEdit ? (
                <ReadOnlyBotRules rules={rules} />
            ) : (
                <VerticalNestedDND<CustomBotCondition, CustomBotRule>
                    initialItems={rules}
                    onChange={setRules}
                    renderContainerItem={(rule, { updateContainerItem }) => {
                        const ruleError = validateCustomBotRule(rule)
                        return (
                            <div className="flex flex-col gap-2">
                                <div className="flex flex-row items-center gap-2 flex-wrap">
                                    <span>Flag as bot</span>
                                    <LemonInput
                                        className="flex-1 min-w-40"
                                        value={rule.name}
                                        onChange={(name) => updateContainerItem({ ...rule, name })}
                                        placeholder="Acme scraper"
                                        disabledReason={restrictedReason}
                                    />
                                    <span>in category</span>
                                    <LemonSelect
                                        value={rule.category || CUSTOM_BOT_CATEGORY}
                                        options={CUSTOM_BOT_CATEGORY_OPTIONS}
                                        onChange={(category) => updateContainerItem({ ...rule, category })}
                                        disabledReason={restrictedReason}
                                    />
                                </div>
                                {rule.items.length === 1 ? (
                                    <span>when this condition is met</span>
                                ) : (
                                    <div className="flex flex-row items-center gap-2">
                                        <span>when</span>
                                        <LemonSelect
                                            value={rule.combiner}
                                            options={combinerOptions}
                                            onChange={(combiner) => updateContainerItem({ ...rule, combiner })}
                                            disabledReason={restrictedReason}
                                        />
                                        <span>conditions are met</span>
                                    </div>
                                )}
                                {ruleError && !rule.items.some(validateCustomBotCondition) ? (
                                    <span className="text-danger text-xs">{ruleError}</span>
                                ) : null}
                            </div>
                        )
                    }}
                    renderChildItem={(condition, { updateChildItem }) => {
                        const error = validateCustomBotCondition(condition)
                        const changeKey = (key: CustomBotField): void => {
                            // Regex works on every property, so treat it as a deliberate choice and keep
                            // it. Anything else follows the new property, which moves an IP condition
                            // onto ranges and a screen dimension onto equality.
                            const matcher =
                                condition.matcher === CustomBotMatcher.Regex
                                    ? CustomBotMatcher.Regex
                                    : defaultMatcherFor(key)
                            updateChildItem({ ...condition, key, matcher })
                        }
                        return (
                            <div className="w-full flex flex-col gap-1">
                                <div className="flex flex-row items-center gap-2">
                                    <LemonSelect
                                        value={condition.key}
                                        options={CUSTOM_BOT_FIELD_OPTIONS}
                                        onChange={changeKey}
                                        disabledReason={restrictedReason}
                                    />
                                    <LemonSelect
                                        value={condition.matcher}
                                        options={matcherOptionsFor(condition.key)}
                                        onChange={(matcher) => updateChildItem({ ...condition, matcher })}
                                        disabledReason={restrictedReason}
                                    />
                                    <LemonInput
                                        className="flex-1 font-mono"
                                        value={condition.pattern}
                                        onChange={(pattern) => updateChildItem({ ...condition, pattern })}
                                        placeholder={patternPlaceholderFor(condition.key, condition.matcher)}
                                        status={error ? 'danger' : undefined}
                                        disabledReason={restrictedReason}
                                    />
                                </div>
                                {error ? <span className="text-danger text-xs">{error}</span> : null}
                            </div>
                        )
                    }}
                    renderAddChildItem={(rule, { onAddChild }) =>
                        canEdit ? (
                            <LemonButton
                                type="secondary"
                                icon={<IconPlus />}
                                onClick={() => onAddChild(rule.id)}
                                data-attr="custom-bot-rules-add-condition"
                                disabledReason={
                                    rule.items.length >= MAX_CONDITIONS_PER_RULE
                                        ? `A rule can have at most ${MAX_CONDITIONS_PER_RULE} conditions`
                                        : totalConditions >= MAX_TOTAL_CONDITIONS
                                          ? `You can have at most ${MAX_TOTAL_CONDITIONS} conditions across all rules`
                                          : undefined
                                }
                            >
                                Add condition
                            </LemonButton>
                        ) : null
                    }
                    renderAddContainerItem={({ onAddContainer }) =>
                        canEdit ? (
                            <LemonButton
                                type="secondary"
                                icon={<IconPlus />}
                                onClick={onAddContainer}
                                data-attr="custom-bot-rules-add-rule"
                                disabledReason={
                                    rules.length >= MAX_CUSTOM_BOT_RULES
                                        ? `You can define at most ${MAX_CUSTOM_BOT_RULES} bots`
                                        : totalConditions >= MAX_TOTAL_CONDITIONS
                                          ? `You can have at most ${MAX_TOTAL_CONDITIONS} conditions across all rules`
                                          : undefined
                                }
                            >
                                Add bot
                            </LemonButton>
                        ) : null
                    }
                    renderAdditionalControls={() =>
                        canEdit ? (
                            <LemonButton
                                type="primary"
                                onClick={save}
                                loading={currentTeamLoading}
                                data-attr="custom-bot-rules-save"
                                disabledReason={
                                    currentTeamLoading
                                        ? 'Saving'
                                        : anyRuleError
                                          ? 'Fix the errors above first'
                                          : (setError ?? (isUnchanged ? 'No changes to save' : undefined))
                                }
                            >
                                Save
                            </LemonButton>
                        ) : null
                    }
                    createNewContainerItem={newRule}
                    createNewChildItem={newCondition}
                />
            )}

            {ipRulesAreDead ? (
                <LemonBanner type="warning">
                    This project anonymizes IP addresses, so events arrive without one and a condition on the IP address
                    never matches. Turn off "Discard client IP data" in your project settings, or match on another
                    property.
                </LemonBanner>
            ) : null}

            {usedFields.length > 0 ? (
                <div className="flex flex-col gap-2">
                    <LemonLabel info="Only your own bots are checked here. A real query also matches PostHog's built-in list, after your rules.">
                        Test a value
                    </LemonLabel>
                    {usedFields.map((field) => (
                        <div key={field.value} className="flex items-center gap-2">
                            <span className="w-32 shrink-0 text-muted text-xs">{field.label}</span>
                            <LemonInput
                                className="flex-1 font-mono"
                                value={testValues[field.value] ?? ''}
                                onChange={(value) => setTestValues({ ...testValues, [field.value]: value })}
                                placeholder={
                                    field.value === CustomBotField.IP ? '192.0.2.55' : `Paste a ${field.label}`
                                }
                            />
                        </div>
                    ))}
                    {matched.length > 0 ? (
                        <span className="flex items-center gap-1 flex-wrap">
                            Matches
                            {matched.map((rule) => (
                                <LemonTag key={rule.id} type="success">
                                    {rule.name || fieldLabel(rule.items[0]?.key ?? CustomBotField.RawUserAgent)}
                                </LemonTag>
                            ))}
                        </span>
                    ) : hasTestInput ? (
                        <span className="text-muted">
                            No custom rule matched. PostHog's built-in list still applies.
                        </span>
                    ) : null}
                </div>
            ) : null}

            {savedRules.length > 0 ? (
                <LemonBanner type="info">
                    Changes apply to new and existing data. Refresh an insight or dashboard to see them applied.
                </LemonBanner>
            ) : null}
        </div>
    )
}