Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Add curated domains to the crawl queue

Daoud Clarkea26fa6225fa4
diff --git a/mwmbl/main.py b/mwmbl/main.py
index aefe80a..a8ba2e7 100644
--- a/mwmbl/main.py
+++ b/mwmbl/main.py
@@ -5,16 +5,15 @@ import uvicorn
from django.core.management import call_command
from redis import Redis
-from mwmbl.count_urls import count_urls_continuously
-from mwmbl.indexer.update_urls import update_urls_continuously
-from mwmbl.redis_url_queue import RedisURLQueue
-
def run():
django.setup()
from django.conf import settings
from mwmbl import background
+ from mwmbl.redis_url_queue import RedisURLQueue
+ from mwmbl.count_urls import count_urls_continuously
+ from mwmbl.indexer.update_urls import update_urls_continuously
if settings.STATIC_ROOT:
call_command("collectstatic", "--clear", "--noinput")
diff --git a/mwmbl/redis_url_queue.py b/mwmbl/redis_url_queue.py
index 52bd0f3..7519183 100644
--- a/mwmbl/redis_url_queue.py
+++ b/mwmbl/redis_url_queue.py
@@ -11,6 +11,7 @@ from mwmbl.crawler.domains import DomainLinkDatabase, TOP_DOMAINS
from mwmbl.crawler.urls import FoundURL
from mwmbl.hn_top_domains_filtered import DOMAINS
from mwmbl.indexer.blacklist import get_blacklist_domains, is_domain_blacklisted
+from mwmbl.models import DomainSubmission
from mwmbl.settings import CORE_DOMAINS
@@ -29,10 +30,6 @@ MAX_URLS_PER_TOP_DOMAIN = 100
MAX_URLS_PER_OTHER_DOMAIN = 5
MAX_OTHER_DOMAINS = 10000
-MAX_BATCH_URLS_PER_CORE_DOMAIN = 100
-MAX_BATCH_URLS_PER_TOP_DOMAIN = 10
-MAX_BATCH_URLS_PER_OTHER_DOMAIN = 1
-
NUM_TOP_DOMAIN_URLS_TO_INCLUDE = 50
NUM_OTHER_URLS_TO_INCLUDE = 100
@@ -42,8 +39,8 @@ BATCH_SIZE = 100
SCORE_TIME_CONSTANT = 60 * 60 * 24 * 30 * 10
-def get_domain_max_urls(domain: str):
- if domain in CORE_DOMAINS:
+def get_domain_max_urls(domain: str, curated_domains: set[str]):
+ if domain in CORE_DOMAINS | curated_domains:
return MAX_URLS_PER_CORE_DOMAIN
elif domain in TOP_DOMAINS:
return MAX_URLS_PER_TOP_DOMAIN
@@ -51,12 +48,19 @@ def get_domain_max_urls(domain: str):
return MAX_URLS_PER_OTHER_DOMAIN
+def _get_curated_domains():
+ curated_domains = set(DomainSubmission.objects.filter(status="APPROVED").values_list('name', flat=True))
+ return curated_domains
+
+
class RedisURLQueue:
def __init__(self, redis: Redis):
self.redis = redis
self.black_listed_domains = get_blacklist_domains()
def queue_urls(self, found_urls: list[FoundURL]):
+ curated_domains = _get_curated_domains()
+ logger.info(f"Got {len(found_urls)} URLs, {len(curated_domains)} curated domains")
with DomainLinkDatabase() as link_db:
for url in found_urls:
time_since_crawled = (datetime.utcnow() - url.last_crawled
@@ -75,7 +79,7 @@ class RedisURLQueue:
logger.info(f"URL score: {url_score}, score multiplier: {score_multiplier} for domain {domain} and age {time_since_crawled}")
domain_score = link_db.get_domain_score(domain) + url_score
- max_urls = get_domain_max_urls(domain)
+ max_urls = get_domain_max_urls(domain, curated_domains)
self.redis.zadd(DOMAIN_URLS_KEY.format(domain=domain), {url.url: url_score})
self.redis.zremrangebyrank(DOMAIN_URLS_KEY.format(domain=domain), 0, -(max_urls + 1))
self.redis.zadd(DOMAIN_SCORE_KEY, {domain: domain_score}, gt=True)
@@ -90,9 +94,10 @@ class RedisURLQueue:
def get_batch(self) -> list[str]:
top_scoring_domains = set(self.redis.zrange(DOMAIN_SCORE_KEY, 0, 2000, desc=True))
top_other_domains = top_scoring_domains - DOMAINS.keys()
+ curated_domains = _get_curated_domains()
domains = list(CORE_DOMAINS)
- top_curated_domains = DOMAINS.keys() & top_scoring_domains
+ top_curated_domains = (DOMAINS.keys() & top_scoring_domains) | curated_domains
if len(top_curated_domains) > NUM_TOP_DOMAIN_URLS_TO_INCLUDE:
domains += random.sample(top_curated_domains, NUM_TOP_DOMAIN_URLS_TO_INCLUDE)
else:
@@ -107,7 +112,7 @@ class RedisURLQueue:
logger.info(f"Getting batch from domains {domains}")
# Add a random url as the root domain of one of DOMAINS
- random_domain = random.choice(list(DOMAINS.keys()))
+ random_domain = random.choice(list(DOMAINS.keys() | curated_domains))
urls = [f"https://{random_domain}/"]
# Pop the highest scoring URL from each domain