Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

2.0 KB
Raw Blame
services:
  mwmbl-crawler:
    build:
      context: .
      dockerfile: Dockerfile.crawler
    image: mwmbl/crawler
    container_name: mwmbl-crawler
    pull_policy: always
    depends_on:
      - redis
    environment:
      # REQUIRED: API key
      - MWMBL_API_KEY=${MWMBL_API_KEY}

      # REQUIRED: Contact info for website administrators
      - MWMBL_CONTACT_INFO=${MWMBL_CONTACT_INFO}

      # Set value of ./mwmbl/crawler/env_vars.py
      - CRAWLER_WORKERS=4
      - CRAWL_THREADS=20
      - CRAWL_DELAY_SECONDS=0.0
      - PYTHONUNBUFFERED=1
      - REDIS_URL=redis://redis:6379
    volumes:
      - crawler_data:/home/mwmbl/.mwmbl
    # Compose applies deploy.restart_policy in swarm mode only, so what was here was no
    # restart policy at all: a crawler that exited stayed exited. One did, for three days.
    restart: unless-stopped
    logging:
      driver: "json-file"
      options:
        max-size: "50m"
        max-file: "3"
    # CPU restrictions if needed (deploy.resources.limits compose does apply):
    #     deploy:
    #       resources:
    #         limits:
    #           cpus: '0.50'

  redis:
    image: redis:alpine
    restart: unless-stopped
    # save "": no snapshots at all. What is in here is a work queue that allkeys-lru is
    # already free to evict, so there is nothing worth the cost of saving it, and the cost
    # was real - a crawler host was found with 8.5GB of temp-*.rdb next to a 209MB dump.rdb.
    # A snapshot forks a Redis holding up to maxmemory and writes a second copy of the data
    # beside the first, and every one of those children that the OOM killer took left its
    # temp file behind for good. That filled the disk, which failed the next snapshot, which
    # (before stop-writes-on-bgsave-error no, kept below for a save anyone triggers by hand)
    # made Redis reject every write and stopped the crawler.
    command: >
      redis-server --maxmemory 2gb --maxmemory-policy allkeys-lru
      --save "" --stop-writes-on-bgsave-error no
    volumes:
      - redis_data:/data

volumes:
  redis_data:
  crawler_data: