Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Merge branch 'master' into user-registration

Daoud Clarke23688bd3ad2e
diff --cc mwmbl/crawler/app.py
index accfea2,c24cb00..cad5705
--- a/mwmbl/crawler/app.py
+++ b/mwmbl/crawler/app.py
@@@ -32,8 -26,8 +31,9 @@@ from mwmbl.settings import
PUBLIC_URL_PREFIX,
PUBLIC_USER_ID_LENGTH,
FILE_NAME_SUFFIX,
- DATE_REGEX)
+ DATE_REGEX, NUM_EXTRACT_CHARS, NUM_TITLE_CHARS)
+from mwmbl.tinysearchengine.indexer import Document
+ from mwmbl.url_queue import URLQueue
def get_bucket(name):
@@@ -51,56 -45,11 +51,56 @@@ def upload(data: bytes, name: str)
last_batch = None
+def justext_with_dom(html_text, stoplist, length_low=LENGTH_LOW_DEFAULT,
+ length_high=LENGTH_HIGH_DEFAULT, stopwords_low=STOPWORDS_LOW_DEFAULT,
+ stopwords_high=STOPWORDS_HIGH_DEFAULT, max_link_density=MAX_LINK_DENSITY_DEFAULT,
+ max_heading_distance=MAX_HEADING_DISTANCE_DEFAULT, no_headings=NO_HEADINGS_DEFAULT,
+ encoding=None, default_encoding=DEFAULT_ENCODING,
+ enc_errors=DEFAULT_ENC_ERRORS):
+ """
+ Converts an HTML page into a list of classified paragraphs. Each paragraph
+ is represented as instance of class ˙˙justext.paragraph.Paragraph˙˙.
+ """
+ dom = html_to_dom(html_text, default_encoding, encoding, enc_errors)
+
+ titles = dom.xpath("//title")
+ title = titles[0].text if len(titles) > 0 else None
+
+ dom = preprocessor(dom)
+
+ paragraphs = ParagraphMaker.make_paragraphs(dom)
+
+ classify_paragraphs(paragraphs, stoplist, length_low, length_high,
+ stopwords_low, stopwords_high, max_link_density, no_headings)
+ revise_paragraph_classification(paragraphs, max_heading_distance)
+
+ return paragraphs, title
+
+
- def get_router(batch_cache: BatchCache, url_queue: Queue):
+ def get_router(batch_cache: BatchCache, queued_batches: Queue):
router = APIRouter(prefix="/crawler", tags=["crawler"])
+ @router.on_event("startup")
+ async def on_startup():
+ with Database() as db:
+ url_db = URLDatabase(db.connection)
+ return url_db.create_tables()
+
+ @router.get('/fetch')
+ def fetch_url(url: str, query: str):
+ response = requests.get(url)
+ paragraphs, title = justext_with_dom(response.content, justext.get_stoplist("English"))
+ good_paragraphs = [p for p in paragraphs if p.class_type == 'good']
+
+ extract = ' '.join([p.text for p in good_paragraphs])
+ if len(extract) > NUM_EXTRACT_CHARS:
+ extract = extract[:NUM_EXTRACT_CHARS - 1] + '…'
+
+ result = Document(title=title, url=url, extract=extract, score=0.0)
+ return format_result(result, query)
+
@router.post('/batches/')
- def create_batch(batch: Batch):
+ def post_batch(batch: Batch):
if len(batch.items) > MAX_BATCH_SIZE:
raise HTTPException(400, f"Batch size too large (maximum {MAX_BATCH_SIZE}), got {len(batch.items)}")
diff --cc mwmbl/main.py
index 32b13c3,aae58eb..be60cb3
--- a/mwmbl/main.py
+++ b/mwmbl/main.py
@@@ -13,7 -11,7 +12,8 @@@ from mwmbl import backgroun
from mwmbl.crawler import app as crawler
from mwmbl.indexer.batch_cache import BatchCache
from mwmbl.indexer.paths import INDEX_NAME, BATCH_DIR_NAME
+from mwmbl.platform import user
+ from mwmbl.indexer.update_urls import update_urls_continuously
from mwmbl.tinysearchengine import search
from mwmbl.tinysearchengine.completer import Completer
from mwmbl.tinysearchengine.indexer import TinyIndex, Document, PAGE_SIZE
@@@ -80,14 -71,11 +81,14 @@@ def run()
app.include_router(search_router)
batch_cache = BatchCache(Path(args.data) / BATCH_DIR_NAME)
- crawler_router = crawler.get_router(batch_cache, url_queue)
+ crawler_router = crawler.get_router(batch_cache, queued_batches)
app.include_router(crawler_router)
+ user_router = user.create_router()
+ app.include_router(user_router)
+
# Initialize uvicorn server using global app instance and server config params
- uvicorn.run(app, host="0.0.0.0", port=5000)
+ uvicorn.run(app, host="0.0.0.0", port=args.port)
if __name__ == "__main__":