An open source, non-profit web search engine
Fix some bugs in URL fetching query
Daoud Clarke7bd12c1ead30
diff --git a/mwmbl/crawler/urls.py b/mwmbl/crawler/urls.pyindex da677cb..4dc0db7 100644--- a/mwmbl/crawler/urls.py+++ b/mwmbl/crawler/urls.py@@ -139,16 +139,16 @@ class URLDatabase:select_sql = f"""SELECT host, (array_agg(url order by score desc))[:{MAX_URLS_PER_TOP_DOMAIN}] FROM url_and_hostsWHERE host IN %(domains)s- AND status IN ({URLStatus.NEW.value}) OR (+ AND (status = {URLStatus.NEW.value} OR (status = {URLStatus.ASSIGNED.value} AND updated < %(min_updated_date)s- )+ ))GROUP BY host"""others_sql = f"""SELECT DISTINCT ON (host) url FROM (SELECT * FROM url_and_hosts- WHERE status IN ({URLStatus.NEW.value}) OR (+ WHERE status = {URLStatus.NEW.value} OR (status = {URLStatus.ASSIGNED.value} AND updated < %(min_updated_date)s)ORDER BY score DESC LIMIT {MAX_OTHER_DOMAINS}) u@@ -174,9 +174,8 @@ class URLDatabase:results = []for host, urls in agg_results:- # There seems to be a bug in psql where we can get things we didn't ask for...- if host in domain_sample:- results += urls+ results += urls+logger.info(f"Got {len(results)} top domain results")with self.connection.cursor() as cursor:diff --git a/mwmbl/settings.py b/mwmbl/settings.pyindex a450bc3..6dc5553 100644--- a/mwmbl/settings.py+++ b/mwmbl/settings.py@@ -30,7 +30,7 @@ EXTRA_LINK_MULTIPLIER = 0.001UNKNOWN_DOMAIN_MULTIPLIER = 0.001EXCLUDED_DOMAINS = {'web.archive.org', 'forums.giantitp.com', 'www.crutchfield.com', 'plus.google.com'}CORE_DOMAINS = {- 'github.com,',+ 'github.com','en.wikipedia.org','stackoverflow.com','docs.google.com',