An open source, non-profit web search engine
Merge branch 'master' into user-registration
Daoud Clarke23688bd3ad2e
diff --cc mwmbl/crawler/app.pyindex accfea2,c24cb00..cad5705--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@@ -32,8 -26,8 +31,9 @@@ from mwmbl.settings importPUBLIC_URL_PREFIX,PUBLIC_USER_ID_LENGTH,FILE_NAME_SUFFIX,- DATE_REGEX)+ DATE_REGEX, NUM_EXTRACT_CHARS, NUM_TITLE_CHARS)+from mwmbl.tinysearchengine.indexer import Document+ from mwmbl.url_queue import URLQueuedef get_bucket(name):@@@ -51,56 -45,11 +51,56 @@@ def upload(data: bytes, name: str)last_batch = None+def justext_with_dom(html_text, stoplist, length_low=LENGTH_LOW_DEFAULT,+ length_high=LENGTH_HIGH_DEFAULT, stopwords_low=STOPWORDS_LOW_DEFAULT,+ stopwords_high=STOPWORDS_HIGH_DEFAULT, max_link_density=MAX_LINK_DENSITY_DEFAULT,+ max_heading_distance=MAX_HEADING_DISTANCE_DEFAULT, no_headings=NO_HEADINGS_DEFAULT,+ encoding=None, default_encoding=DEFAULT_ENCODING,+ enc_errors=DEFAULT_ENC_ERRORS):+ """+ Converts an HTML page into a list of classified paragraphs. Each paragraph+ is represented as instance of class ˙˙justext.paragraph.Paragraph˙˙.+ """+ dom = html_to_dom(html_text, default_encoding, encoding, enc_errors)++ titles = dom.xpath("//title")+ title = titles[0].text if len(titles) > 0 else None++ dom = preprocessor(dom)++ paragraphs = ParagraphMaker.make_paragraphs(dom)++ classify_paragraphs(paragraphs, stoplist, length_low, length_high,+ stopwords_low, stopwords_high, max_link_density, no_headings)+ revise_paragraph_classification(paragraphs, max_heading_distance)++ return paragraphs, title++- def get_router(batch_cache: BatchCache, url_queue: Queue):+ def get_router(batch_cache: BatchCache, queued_batches: Queue):router = APIRouter(prefix="/crawler", tags=["crawler"])+ @router.on_event("startup")+ async def on_startup():+ with Database() as db:+ url_db = URLDatabase(db.connection)+ return url_db.create_tables()++ @router.get('/fetch')+ def fetch_url(url: str, query: str):+ response = requests.get(url)+ paragraphs, title = justext_with_dom(response.content, justext.get_stoplist("English"))+ good_paragraphs = [p for p in paragraphs if p.class_type == 'good']++ extract = ' '.join([p.text for p in good_paragraphs])+ if len(extract) > NUM_EXTRACT_CHARS:+ extract = extract[:NUM_EXTRACT_CHARS - 1] + '…'++ result = Document(title=title, url=url, extract=extract, score=0.0)+ return format_result(result, query)+@router.post('/batches/')- def create_batch(batch: Batch):+ def post_batch(batch: Batch):if len(batch.items) > MAX_BATCH_SIZE:raise HTTPException(400, f"Batch size too large (maximum {MAX_BATCH_SIZE}), got {len(batch.items)}")diff --cc mwmbl/main.pyindex 32b13c3,aae58eb..be60cb3--- a/mwmbl/main.py+++ b/mwmbl/main.py@@@ -13,7 -11,7 +12,8 @@@ from mwmbl import backgrounfrom mwmbl.crawler import app as crawlerfrom mwmbl.indexer.batch_cache import BatchCachefrom mwmbl.indexer.paths import INDEX_NAME, BATCH_DIR_NAME+from mwmbl.platform import user+ from mwmbl.indexer.update_urls import update_urls_continuouslyfrom mwmbl.tinysearchengine import searchfrom mwmbl.tinysearchengine.completer import Completerfrom mwmbl.tinysearchengine.indexer import TinyIndex, Document, PAGE_SIZE@@@ -80,14 -71,11 +81,14 @@@ def run()app.include_router(search_router)batch_cache = BatchCache(Path(args.data) / BATCH_DIR_NAME)- crawler_router = crawler.get_router(batch_cache, url_queue)+ crawler_router = crawler.get_router(batch_cache, queued_batches)app.include_router(crawler_router)+ user_router = user.create_router()+ app.include_router(user_router)+# Initialize uvicorn server using global app instance and server config params- uvicorn.run(app, host="0.0.0.0", port=5000)+ uvicorn.run(app, host="0.0.0.0", port=args.port)if __name__ == "__main__":