Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Allow crawling links from unknown domains

Daoud Clarkee578d5578911
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.py
index 408c4e6..ec790f6 100644
--- a/mwmbl/crawler/app.py
+++ b/mwmbl/crawler/app.py
@@ -35,6 +35,7 @@ FILE_NAME_SUFFIX = '.json.gz'
SCORE_FOR_ROOT_PATH = 0.1
SCORE_FOR_DIFFERENT_DOMAIN = 1.0
SCORE_FOR_SAME_DOMAIN = 0.01
+UNKNOWN_DOMAIN_MULTIPLIER = 0.001
router = APIRouter(prefix="/crawler", tags=["crawler"])
@@ -142,15 +143,13 @@ def record_urls_in_database(batch: Union[Batch, HashedBatch], user_id_hash: str,
for item in batch.items:
if item.content is not None:
crawled_page_domain = urlparse(item.url).netloc
- if crawled_page_domain not in DOMAINS:
- continue
-
+ score_multiplier = 1 if crawled_page_domain in DOMAINS else UNKNOWN_DOMAIN_MULTIPLIER
for link in item.content.links:
parsed_link = urlparse(link)
score = SCORE_FOR_SAME_DOMAIN if parsed_link.netloc == crawled_page_domain else SCORE_FOR_DIFFERENT_DOMAIN
- url_scores[link] += score
+ url_scores[link] += score * score_multiplier
domain = f'{parsed_link.scheme}://{parsed_link.netloc}/'
- url_scores[domain] += SCORE_FOR_ROOT_PATH
+ url_scores[domain] += SCORE_FOR_ROOT_PATH * score_multiplier
found_urls = [FoundURL(url, user_id_hash, score, URLStatus.NEW, timestamp) for url, score in url_scores.items()]
if len(found_urls) > 0: