An open source, non-profit web search engine
Add endpoint to fetch a URL and return title and extract
Daoud Clarke0a4e1e4aee25
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.pyindex c7aa9de..3e1805b 100644--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@ -8,8 +8,12 @@ from typing import Unionfrom uuid import uuid4import boto3+import justextimport requestsfrom fastapi import HTTPException, APIRouter+from justext.core import html_to_dom, ParagraphMaker, classify_paragraphs, revise_paragraph_classification, \+ LENGTH_LOW_DEFAULT, STOPWORDS_LOW_DEFAULT, MAX_LINK_DENSITY_DEFAULT, NO_HEADINGS_DEFAULT, LENGTH_HIGH_DEFAULT, \+ STOPWORDS_HIGH_DEFAULT, MAX_HEADING_DISTANCE_DEFAULT, DEFAULT_ENCODING, DEFAULT_ENC_ERRORS, preprocessorfrom mwmbl.crawler.batch import Batch, NewBatchRequest, HashedBatchfrom mwmbl.crawler.urls import URLDatabase, FoundURL, URLStatus@@ -27,7 +31,7 @@ from mwmbl.settings import (PUBLIC_URL_PREFIX,PUBLIC_USER_ID_LENGTH,FILE_NAME_SUFFIX,- DATE_REGEX)+ DATE_REGEX, NUM_EXTRACT_CHARS, NUM_TITLE_CHARS)def get_bucket(name):@@ -45,6 +49,32 @@ def upload(data: bytes, name: str):last_batch = None+def justext_with_dom(html_text, stoplist, length_low=LENGTH_LOW_DEFAULT,+ length_high=LENGTH_HIGH_DEFAULT, stopwords_low=STOPWORDS_LOW_DEFAULT,+ stopwords_high=STOPWORDS_HIGH_DEFAULT, max_link_density=MAX_LINK_DENSITY_DEFAULT,+ max_heading_distance=MAX_HEADING_DISTANCE_DEFAULT, no_headings=NO_HEADINGS_DEFAULT,+ encoding=None, default_encoding=DEFAULT_ENCODING,+ enc_errors=DEFAULT_ENC_ERRORS):+ """+ Converts an HTML page into a list of classified paragraphs. Each paragraph+ is represented as instance of class ˙˙justext.paragraph.Paragraph˙˙.+ """+ dom = html_to_dom(html_text, default_encoding, encoding, enc_errors)++ titles = dom.xpath("//title")+ title = titles[0].text if len(titles) > 0 else None++ dom = preprocessor(dom)++ paragraphs = ParagraphMaker.make_paragraphs(dom)++ classify_paragraphs(paragraphs, stoplist, length_low, length_high,+ stopwords_low, stopwords_high, max_link_density, no_headings)+ revise_paragraph_classification(paragraphs, max_heading_distance)++ return paragraphs, title++def get_router(batch_cache: BatchCache, url_queue: Queue):router = APIRouter(prefix="/crawler", tags=["crawler"])@@ -54,6 +84,22 @@ def get_router(batch_cache: BatchCache, url_queue: Queue):url_db = URLDatabase(db.connection)return url_db.create_tables()+ @router.get('/fetch')+ def fetch_url(url: str):+ response = requests.get(url)+ paragraphs, title = justext_with_dom(response.content, justext.get_stoplist("English"))+ good_paragraphs = [p for p in paragraphs if p.class_type == 'good']++ extract = ' '.join([p.text for p in good_paragraphs])+ if len(extract) > NUM_EXTRACT_CHARS:+ extract = extract[:NUM_EXTRACT_CHARS - 1] + '…'++ return {+ 'url': url,+ 'title': title,+ 'extract': extract,+ }+@router.post('/batches/')def create_batch(batch: Batch):if len(batch.items) > MAX_BATCH_SIZE:diff --git a/mwmbl/settings.py b/mwmbl/settings.pyindex 9335314..b6ee8e6 100644--- a/mwmbl/settings.py+++ b/mwmbl/settings.py@@ -23,6 +23,9 @@ DATE_REGEX = re.compile(r'\d{4}-\d{2}-\d{2}')PUBLIC_URL_PREFIX = f'https://f004.backblazeb2.com/file/{BUCKET_NAME}/'FILE_NAME_SUFFIX = '.json.gz'+NUM_TITLE_CHARS = 65+NUM_EXTRACT_CHARS = 155+SCORE_FOR_ROOT_PATH = 0.1SCORE_FOR_DIFFERENT_DOMAIN = 1.0SCORE_FOR_SAME_DOMAIN = 0.01diff --git a/poetry.lock b/poetry.lockindex 5372b42..61c994b 100644--- a/poetry.lock+++ b/poetry.lock@@ -176,7 +176,7 @@ name = "en-core-web-sm"version = "3.2.0"description = "English pipeline optimized for CPU. Components: tok2vec, tagger, parser, senter, ner, attribute_ruler, lemmatizer."category = "main"-optional = true+optional = falsepython-versions = "*"[package.dependencies]@@ -185,6 +185,7 @@ spacy = ">=3.2.0,<3.3.0"[package.source]type = "url"url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.2.0/en_core_web_sm-3.2.0.tar.gz"+[[package]]name = "fastapi"version = "0.70.1"@@ -928,12 +929,12 @@ cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\cffi = ["cffi (>=1.11)"][extras]-indexer = ["ujson", "warcio", "idna", "beautifulsoup4", "lxml", "jusText", "langdetect", "pyarrow", "pyspark", "Levenshtein", "en-core-web-sm"]+indexer = ["ujson", "warcio", "idna", "beautifulsoup4", "lxml", "jusText", "langdetect", "pyarrow", "pyspark", "Levenshtein"][metadata]lock-version = "1.1"python-versions = ">=3.10,<3.11"-content-hash = "cd17d671d8fa708240c3bf7bf5240bcf4bbcebea4538e5ad091bbcf9d10c5574"+content-hash = "073bb40814f94368f5a6658acbff4b0b7cf720644593f016d1a28f7a6f86042d"[metadata.files]anyio = [diff --git a/pyproject.toml b/pyproject.tomlindex 53c1eb4..9f17bda 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -18,6 +18,7 @@ boto3 = "^1.20.37"requests = "^2.27.1"psycopg2-binary = "^2.9.3"spacy = "==3.2.1"+jusText = "==3.0.0"# Optional dependencies do not get installed by default. Look under tool.poetry.extras section# to see which extras to use.@@ -26,7 +27,6 @@ warcio = {version= "==1.7.4", optional = true}idna = {version= "==3.3", optional = true}beautifulsoup4 = {version= "==4.10.0", optional = true}lxml = {version= "==4.6.4", optional = true}-jusText = {version= "==3.0.0", optional = true}langdetect = {version= "==1.0.9", optional = true}pyarrow = {version= "==6.0.0", optional = true}pyspark = {version= "==3.2.0", optional = true}