:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
#!/bin/bash
set -e
help () {
echo "$0 - start PostHog's Celery worker"
echo
echo "$0 [options]"
echo
echo "Options:"
echo " --help, -h show this brief help"
echo " --with-scheduler start RedBeat, the Celery scheduler (deprecates --with-beat)"
echo " --concurrency=<N> start N workers (overrides env var WEB_CONCURRENCY)"
echo
echo "Advanced Celery options (disabled by default):"
echo " --with-heartbeat start Celery internal heartbeat (normally not useful)"
echo " --with-mingle start Celery mingle (normally not useful)"
exit 0
}
with_scheduler=false
with_gossip=true
with_heartbeat=false
with_mingle=false
while test $# -gt 0; do
case "$1" in
-h|--help)
help
;;
--with-scheduler)
with_scheduler=true
shift
;;
--with-beat) # Deprecated since the name is too similar to "heartbeat"
echo "⚠️ Using docker-worker-celery with --with-beat. This argument is deprecated. Use --with-scheduler instead!"
with_scheduler=true
shift
;;
--with-heartbeat)
with_heartbeat=true
shift
;;
--with-mingle)
with_mingle=true
shift
;;
--concurrency*)
export WEB_CONCURRENCY=`echo $1 | sed -e 's/^[^=]*=//g'`
shift
;;
*)
break
;;
esac
done
# Forward SIGTERM to all background jobs and wait for them to finish.
# Without the explicit wait, bash (PID 1) exits immediately after the
# trap fires and the kernel tears down the container — killing in-flight
# Celery tasks before they can drain, even though K8s gives us 1230s of
# grace time via terminationGracePeriodSeconds.
cleanup() {
kill $(jobs -p) 2>/dev/null
wait
}
trap cleanup EXIT
# Prometheus multiprocess mode: each prefork child writes metrics to this
# shared directory; the HTTP endpoint aggregates across all children.
# Without this, only one child (the one that binds port 8001) is scraped,
# causing Prometheus to see ~1/N of actual task events.
export PROMETHEUS_MULTIPROC_DIR="/tmp/posthog_celery_prometheus"
if [ "$with_scheduler" == "true" ]; then
./bin/docker-worker-beat &
fi
FLAGS=()
FLAGS+=("-Ofair")
FLAGS+=("-n node@%h")
[ "$with_gossip" == "false" ] && FLAGS+=("--without-gossip")
[ "$with_mingle" == "false" ] && FLAGS+=("--without-mingle")
[ "$with_heartbeat" == "false" ] && FLAGS+=("--without-heartbeat")
# On Heroku $WEB_CONCURRENCY contains suggested number of forks per dyno type
# https://github.com/heroku/heroku-buildpack-python/blob/main/vendor/WEB_CONCURRENCY.sh
[[ -n "${WEB_CONCURRENCY}" ]] && FLAGS+=" --concurrency $WEB_CONCURRENCY"
# Restart worker process after it processes this many tasks (to mitigate memory leaks)
[[ -n "${CELERY_MAX_TASKS_PER_CHILD}" ]] && FLAGS+=" --max-tasks-per-child $CELERY_MAX_TASKS_PER_CHILD"
# Restart worker process after it exceeds this much memory usage (to mitigate memory leaks)
[[ -n "${CELERY_MAX_MEMORY_PER_CHILD}" ]] && FLAGS+=" --max-memory-per-child $CELERY_MAX_MEMORY_PER_CHILD"
[[ -n "${CELERY_WORKER_PREFETCH_MULTIPLIER}" ]] && FLAGS+=" --prefetch-multiplier $CELERY_WORKER_PREFETCH_MULTIPLIER"
if [[ -z "${CELERY_WORKER_QUEUES}" ]]; then
source ./bin/celery-queues.env
fi
echo
echo "SKIP_ASYNC_MIGRATIONS_SETUP=0 celery -A posthog worker ${FLAGS[*]}"
echo
./bin/migrate-check
SKIP_ASYNC_MIGRATIONS_SETUP=0 CELERY_WORKER_QUEUES=$CELERY_WORKER_QUEUES celery -A posthog worker ${FLAGS[*]} &
# Exit if any processes exit, and exit with it's exit code
wait -n
exit $?