An open source, non-profit web search engine
services:
mwmbl-crawler:
build:
context: .
dockerfile: Dockerfile.crawler
image: mwmbl/crawler
container_name: mwmbl-crawler
pull_policy: always
depends_on:
- redis
environment:
# REQUIRED: API key
- MWMBL_API_KEY=${MWMBL_API_KEY}
# REQUIRED: Contact info for website administrators
- MWMBL_CONTACT_INFO=${MWMBL_CONTACT_INFO}
# Set value of ./mwmbl/crawler/env_vars.py
- CRAWLER_WORKERS=4
- CRAWL_THREADS=20
- CRAWL_DELAY_SECONDS=0.0
- PYTHONUNBUFFERED=1
- REDIS_URL=redis://redis:6379
volumes:
- crawler_data:/home/mwmbl/.mwmbl
# Compose applies deploy.restart_policy in swarm mode only, so what was here was no
# restart policy at all: a crawler that exited stayed exited. One did, for three days.
restart: unless-stopped
logging:
driver: "json-file"
options:
max-size: "50m"
max-file: "3"
# CPU restrictions if needed (deploy.resources.limits compose does apply):
# deploy:
# resources:
# limits:
# cpus: '0.50'
redis:
image: redis:alpine
restart: unless-stopped
# save "": no snapshots at all. What is in here is a work queue that allkeys-lru is
# already free to evict, so there is nothing worth the cost of saving it, and the cost
# was real - a crawler host was found with 8.5GB of temp-*.rdb next to a 209MB dump.rdb.
# A snapshot forks a Redis holding up to maxmemory and writes a second copy of the data
# beside the first, and every one of those children that the OOM killer took left its
# temp file behind for good. That filled the disk, which failed the next snapshot, which
# (before stop-writes-on-bgsave-error no, kept below for a save anyone triggers by hand)
# made Redis reject every write and stopped the crawler.
command: >
redis-server --maxmemory 2gb --maxmemory-policy allkeys-lru
--save "" --stop-writes-on-bgsave-error no
volumes:
- redis_data:/data
volumes:
redis_data:
crawler_data: