An open source, non-profit web search engine
Get URL scores in batches
Daoud Clarke2942d83673d0
diff --git a/mwmbl/crawler/urls.py b/mwmbl/crawler/urls.pyindex 1f60dd6..e4eddfd 100644--- a/mwmbl/crawler/urls.py+++ b/mwmbl/crawler/urls.py@@ -5,6 +5,7 @@ import osfrom dataclasses import dataclassfrom datetime import datetime, timedeltafrom enum import Enum+from typing import Iterablefrom psycopg2 import connectfrom psycopg2.extras import execute_values@@ -30,6 +31,15 @@ class URLStatus(Enum):CRAWLED = 100 # At least one user has crawled the URL+def batch(items: list, batch_size):+ """+ Adapted from https://stackoverflow.com/a/8290508+ """+ length = len(items)+ for ndx in range(0, length, batch_size):+ yield items[ndx:min(ndx + batch_size, length)]++@dataclassclass FoundURL:url: str@@ -137,11 +147,14 @@ class URLDatabase:SELECT url, score FROM urls WHERE url IN %(urls)s"""- with self.connection.cursor() as cursor:- cursor.execute(sql, {'urls': tuple(urls)})- results = cursor.fetchall()+ url_scores = {}+ for url_batch in batch(urls, 10000):+ with self.connection.cursor() as cursor:+ cursor.execute(sql, {'urls': tuple(url_batch)})+ results = cursor.fetchall()+ url_scores.update({result[0]: result[1] for result in results})- return {result[0]: result[1] for result in results}+ return url_scoresif __name__ == "__main__":