Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

11.7 KB
Raw Blame
package events

import (
	"fmt"
	"log"
	"regexp"
	"slices"
	"strconv"
	"strings"
	"sync/atomic"

	"github.com/gofrs/uuid/v5"
	"github.com/posthog/posthog/livestream/auth"
	"github.com/posthog/posthog/livestream/metrics"
	"github.com/prometheus/client_golang/prometheus"
)

const (
	OpExact        = "exact"
	OpIsNot        = "is_not"
	OpIContains    = "icontains"
	OpNotIContains = "not_icontains"
	OpRegex        = "regex"
	OpNotRegex     = "not_regex"
	OpGreaterThan  = "gt"
	OpGreaterEqual = "gte"
	OpLessThan     = "lt"
	OpLessEqual    = "lte"
	OpIsSet        = "is_set"
	OpIsNotSet     = "is_not_set"
)

type CompiledPropertyFilter struct {
	Key      string
	Operator string
	Values   []string

	lowerValues []string
	regexes     []*regexp.Regexp
	numbers     []float64
	numericOK   []bool
}

func NewCompiledPropertyFilter(key, operator string, values []string) CompiledPropertyFilter {
	f := CompiledPropertyFilter{Key: key, Operator: operator, Values: values}
	switch operator {
	case OpIContains, OpNotIContains:
		f.lowerValues = make([]string, len(values))
		for i, v := range values {
			f.lowerValues[i] = strings.ToLower(v)
		}
	case OpRegex, OpNotRegex:
		f.regexes = make([]*regexp.Regexp, len(values))
		for i, v := range values {
			if re, err := regexp.Compile(v); err == nil {
				f.regexes[i] = re
			} else {
				log.Printf("WARNING: ignoring invalid regex in %s filter for key=%s value=%q: %v", operator, key, v, err)
			}
		}
	case OpGreaterThan, OpGreaterEqual, OpLessThan, OpLessEqual:
		f.numbers = make([]float64, len(values))
		f.numericOK = make([]bool, len(values))
		for i, v := range values {
			if n, err := strconv.ParseFloat(strings.TrimSpace(v), 64); err == nil {
				f.numbers[i] = n
				f.numericOK[i] = true
			}
		}
	}
	return f
}

type Subscription struct {
	SubID uint64

	// Filters
	TeamId          int
	Token           string
	DistinctId      string
	EventTypes      []string
	PropertyFilters []CompiledPropertyFilter

	Geo     bool
	Columns []string

	// Transformations
	PathCleaner *PathCleaner

	// Channels
	EventChan   chan interface{}
	ShouldClose *atomic.Bool

	// Stats
	DroppedEvents *atomic.Uint64
}

//easyjson:json
type ResponsePostHogEvent struct {
	Uuid       string                 `json:"uuid"`
	Timestamp  interface{}            `json:"timestamp"`
	DistinctId string                 `json:"distinct_id"`
	PersonId   string                 `json:"person_id"`
	Event      string                 `json:"event"`
	Properties map[string]interface{} `json:"properties"`
}

//easyjson:json
type ResponseGeoEvent struct {
	Lat         float64 `json:"lat"`
	Lng         float64 `json:"lng"`
	CountryCode string  `json:"country_code"`
	DistinctId  string  `json:"distinct_id"`
	Count       uint    `json:"count"`
}

type Filter struct {
	inboundChan chan PostHogEvent
	SubChan     chan Subscription
	UnSubChan   chan Subscription
	subs        []Subscription
}

func NewFilter(subChan chan Subscription, unSubChan chan Subscription, inboundChan chan PostHogEvent) *Filter {
	return &Filter{SubChan: subChan, UnSubChan: unSubChan, inboundChan: inboundChan, subs: make([]Subscription, 0)}
}

func convertToResponseGeoEvent(event PostHogEvent) *ResponseGeoEvent {
	return &ResponseGeoEvent{
		Lat:         event.Lat,
		Lng:         event.Lng,
		CountryCode: event.CountryCode,
		DistinctId:  event.DistinctId,
		Count:       1,
	}
}

// Person and group properties ride along on events under these keys, so a
// restricted person or group property has to be removed from inside them as well.
var (
	personPropertyContainers = []string{"$set", "$set_once"}
	groupPropertyContainer   = "$group_set"
	groupTypeProperty        = "$group_type"
)

// A filter on a hidden key would let the subscriber infer its values from which events match.
// Filters match the whole stringified $set / $set_once map, so those are hidden too once any person key is.
func RestrictedFilterKey(filters []CompiledPropertyFilter, restrictions *auth.PropertyRestrictions) string {
	for i := range filters {
		key := filters[i].Key
		if restrictions.RestrictsEventProperty(key) {
			return key
		}
		if restrictions.HasPersonRestrictions() && slices.Contains(personPropertyContainers, key) {
			return key
		}
		if restrictions.HasGroupRestrictions() && key == groupPropertyContainer {
			return key
		}
	}
	return ""
}

func groupTypeOf(properties map[string]interface{}) string {
	groupType, _ := properties[groupTypeProperty].(string)
	return groupType
}

// StripRestricted removes hidden properties from a built response. The handler calls it at
// write time, off the fan-out goroutine, with whatever rules the latest re-check returned.
func (e *ResponsePostHogEvent) StripRestricted(restrictions *auth.PropertyRestrictions) {
	if restrictions == nil {
		return
	}
	properties := make(map[string]interface{}, len(e.Properties))
	groupType := groupTypeOf(e.Properties)
	for k, v := range e.Properties {
		if visible, ok := visibleProperty(k, v, groupType, restrictions); ok {
			properties[k] = visible
		}
	}
	if restrictions.RestrictsEventProperty("$pathname") {
		delete(properties, "$virt_cleaned_pathname")
	}
	e.Properties = properties
}

func visibleProperty(key string, value interface{}, groupType string, restrictions *auth.PropertyRestrictions) (interface{}, bool) {
	if restrictions.RestrictsEventProperty(key) {
		return nil, false
	}
	var hidden func(string) bool
	switch {
	case restrictions.HasPersonRestrictions() && slices.Contains(personPropertyContainers, key):
		hidden = restrictions.RestrictsPersonProperty
	case restrictions.HasGroupRestrictions() && key == groupPropertyContainer:
		hidden = func(k string) bool { return restrictions.RestrictsGroupProperty(groupType, k) }
	default:
		return value, true
	}
	nested, ok := value.(map[string]interface{})
	if !ok {
		return value, true
	}
	stripped := make(map[string]interface{}, len(nested))
	for k, v := range nested {
		if !hidden(k) {
			stripped[k] = v
		}
	}
	return stripped, true
}

func convertToResponsePostHogEvent(
	event PostHogEvent,
	teamId int,
	columns []string,
	pathCleaner *PathCleaner,
) *ResponsePostHogEvent {
	var properties map[string]interface{}
	if columns == nil {
		properties = event.Properties
		if pathCleaner != nil {
			// About to inject a per-subscriber property; copy so the shared
			// event map never carries one subscription's cleaning into another.
			properties = make(map[string]interface{}, len(event.Properties)+1)
			for k, v := range event.Properties {
				properties[k] = v
			}
		}
	} else {
		properties = make(map[string]interface{})
		for _, key := range columns {
			if val, ok := event.Properties[key]; ok {
				properties[key] = val
			}
		}
	}

	// Always pass through $virt_* bot classification properties
	// regardless of requested columns
	for _, key := range []string{"$virt_is_bot", "$virt_traffic_type", "$virt_traffic_category", "$virt_bot_name"} {
		if val, ok := event.Properties[key]; ok {
			properties[key] = val
		}
	}

	if pathCleaner != nil {
		if pathname, ok := event.Properties["$pathname"].(string); ok {
			properties["$virt_cleaned_pathname"] = pathCleaner.Clean(pathname)
		}
	}

	return &ResponsePostHogEvent{
		Uuid:       event.Uuid,
		Timestamp:  event.Timestamp,
		DistinctId: event.DistinctId,
		PersonId:   uuidFromDistinctId(teamId, event.DistinctId),
		Event:      event.Event,
		Properties: properties,
	}
}

var personUUIDV5Namespace = uuid.Must(uuid.FromString("932979b4-65c3-4424-8467-0b66ec27bc22"))

func uuidFromDistinctId(teamId int, distinctId string) string {
	if teamId == 0 || distinctId == "" {
		return ""
	}

	input := fmt.Sprintf("%d:%s", teamId, distinctId)
	return uuid.NewV5(personUUIDV5Namespace, input).String()
}

func logUnsubscribe(sub Subscription) {
	if dropped := sub.DroppedEvents.Load(); dropped > 0 {
		log.Printf("Team %d dropped %d events", sub.TeamId, dropped)
	}
	metrics.SubTotal.Dec()
}

func removeSubscription(subID uint64, subs []Subscription) []Subscription {
	for i, sub := range subs {
		if subID == sub.SubID {
			logUnsubscribe(sub)
			return slices.Delete(subs, i, i+1)
		}
	}
	return subs
}

func (c *Filter) Run() {
	for {
		select {
		case newSub := <-c.SubChan:
			c.subs = append(c.subs, newSub)
			metrics.SubTotal.Inc()
		case unSub := <-c.UnSubChan:
			c.subs = removeSubscription(unSub.SubID, c.subs)
		case event := <-c.inboundChan:
			matching := make([]Subscription, 0, len(c.subs))
			for _, sub := range c.subs {
				if sub.Token != "" && event.Token != sub.Token {
					continue
				}
				matching = append(matching, sub)
			}
			deliverEvent(event, matching)
		}
	}
}

func matchesPropertyFilters(props map[string]interface{}, filters []CompiledPropertyFilter) bool {
	for i := range filters {
		if !filters[i].matches(props) {
			return false
		}
	}
	return true
}

func (f *CompiledPropertyFilter) hasValidRegex() bool {
	for _, re := range f.regexes {
		if re != nil {
			return true
		}
	}
	return false
}

func (f *CompiledPropertyFilter) matches(props map[string]interface{}) bool {
	raw, present := props[f.Key]

	switch f.Operator {
	case OpIsSet:
		return present
	case OpIsNotSet:
		return !present
	}

	if (f.Operator == OpRegex || f.Operator == OpNotRegex) && !f.hasValidRegex() {
		return false
	}

	if !present {
		switch f.Operator {
		case OpIsNot, OpNotIContains, OpNotRegex:
			return true
		default:
			return false
		}
	}

	actual := fmt.Sprint(raw)

	switch f.Operator {
	case OpExact:
		return slices.Contains(f.Values, actual)
	case OpIsNot:
		return !slices.Contains(f.Values, actual)
	case OpIContains:
		lower := strings.ToLower(actual)
		for _, v := range f.lowerValues {
			if strings.Contains(lower, v) {
				return true
			}
		}
		return false
	case OpNotIContains:
		lower := strings.ToLower(actual)
		for _, v := range f.lowerValues {
			if strings.Contains(lower, v) {
				return false
			}
		}
		return true
	case OpRegex:
		for _, re := range f.regexes {
			if re != nil && re.MatchString(actual) {
				return true
			}
		}
		return false
	case OpNotRegex:
		for _, re := range f.regexes {
			if re != nil && re.MatchString(actual) {
				return false
			}
		}
		return true
	case OpGreaterThan, OpGreaterEqual, OpLessThan, OpLessEqual:
		actualNum, err := strconv.ParseFloat(strings.TrimSpace(actual), 64)
		if err != nil {
			return false
		}
		for i, ok := range f.numericOK {
			if ok && compareNumeric(f.Operator, actualNum, f.numbers[i]) {
				return true
			}
		}
		return false
	default:
		return false
	}
}

func compareNumeric(operator string, actual, want float64) bool {
	switch operator {
	case OpGreaterThan:
		return actual > want
	case OpGreaterEqual:
		return actual >= want
	case OpLessThan:
		return actual < want
	case OpLessEqual:
		return actual <= want
	default:
		return false
	}
}

// Routes a single event to all matching subscriptions.
// Used by both Filter (in-memory path) and TokenRouter (Redis pub/sub path).
func deliverEvent(event PostHogEvent, subs []Subscription) {
	var responseGeoEvent *ResponseGeoEvent

	for _, sub := range subs {
		if sub.ShouldClose.Load() {
			continue
		}

		if sub.DistinctId != "" && event.DistinctId != sub.DistinctId {
			continue
		}

		if len(sub.EventTypes) > 0 && !slices.Contains(sub.EventTypes, event.Event) {
			continue
		}

		if len(sub.PropertyFilters) > 0 && !matchesPropertyFilters(event.Properties, sub.PropertyFilters) {
			continue
		}

		if sub.Geo {
			if event.Lat != 0.0 {
				if responseGeoEvent == nil {
					responseGeoEvent = convertToResponseGeoEvent(event)
				}

				select {
				case sub.EventChan <- *responseGeoEvent:
				default:
					sub.DroppedEvents.Add(1)
					metrics.DroppedEvents.With(prometheus.Labels{"channel": "geo"}).Inc()
				}
			}
		} else {
			responseEvent := convertToResponsePostHogEvent(event, sub.TeamId, sub.Columns, sub.PathCleaner)

			select {
			case sub.EventChan <- *responseEvent:
			default:
				sub.DroppedEvents.Add(1)
				metrics.DroppedEvents.With(prometheus.Labels{"channel": "events"}).Inc()
			}
		}
	}
}