An open source, non-profit web search engine
Allow crawling links from unknown domains
Daoud Clarkee578d5578911
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.pyindex 408c4e6..ec790f6 100644--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@ -35,6 +35,7 @@ FILE_NAME_SUFFIX = '.json.gz'SCORE_FOR_ROOT_PATH = 0.1SCORE_FOR_DIFFERENT_DOMAIN = 1.0SCORE_FOR_SAME_DOMAIN = 0.01+UNKNOWN_DOMAIN_MULTIPLIER = 0.001router = APIRouter(prefix="/crawler", tags=["crawler"])@@ -142,15 +143,13 @@ def record_urls_in_database(batch: Union[Batch, HashedBatch], user_id_hash: str,for item in batch.items:if item.content is not None:crawled_page_domain = urlparse(item.url).netloc- if crawled_page_domain not in DOMAINS:- continue-+ score_multiplier = 1 if crawled_page_domain in DOMAINS else UNKNOWN_DOMAIN_MULTIPLIERfor link in item.content.links:parsed_link = urlparse(link)score = SCORE_FOR_SAME_DOMAIN if parsed_link.netloc == crawled_page_domain else SCORE_FOR_DIFFERENT_DOMAIN- url_scores[link] += score+ url_scores[link] += score * score_multiplierdomain = f'{parsed_link.scheme}://{parsed_link.netloc}/'- url_scores[domain] += SCORE_FOR_ROOT_PATH+ url_scores[domain] += SCORE_FOR_ROOT_PATH * score_multiplierfound_urls = [FoundURL(url, user_id_hash, score, URLStatus.NEW, timestamp) for url, score in url_scores.items()]if len(found_urls) > 0: