An open source, non-profit web search engine
Add resolved URL and anchor text to links in batch
Daoud Clarke38fed25fa46a
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.pyindex 720160b..55ea1d5 100644--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@ -55,6 +55,10 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versio@router.post('/batches/')def post_batch(request, batch: Batch):+ """+ Store a batch of crawled pages.+ """+if len(batch.items) > MAX_BATCH_SIZE:raise HTTPException(400, f"Batch size too large (maximum {MAX_BATCH_SIZE}), got {len(batch.items)}")diff --git a/mwmbl/crawler/batch.py b/mwmbl/crawler/batch.pyindex 812d408..c9a43e5 100644--- a/mwmbl/crawler/batch.py+++ b/mwmbl/crawler/batch.py@@ -3,10 +3,19 @@ from typing import Optionalfrom ninja import Schema+class Link(Schema):+ url: str+ anchor_text: str+ link_type: str++class ItemContent(Schema):title: strextract: str- links: list[str]+ link_details: Optional[list[Link]]++ # Deprecated+ links: Optional[list[str]]extra_links: Optional[list[str]]links_only: Optional[bool]@@ -18,6 +27,7 @@ class ItemError(Schema):class Item(Schema):url: str+ resolved_url: Optional[str]status: Optional[int]timestamp: intcontent: Optional[ItemContent]diff --git a/mwmbl/crawler/stats.py b/mwmbl/crawler/stats.pyindex 932399e..ac35906 100644--- a/mwmbl/crawler/stats.py+++ b/mwmbl/crawler/stats.py@@ -109,7 +109,15 @@ class StatsManager:self.redis.zincrby(host_key, 1, host)- for link in item.content.links:+ links = []+ if item.content.links is not None:+ links += item.content.links+ if item.content.extra_links is not None:+ links += item.content.extra_links+ if item.content.link_details is not None:+ links += [link.url for link in item.content.link_details]++ for link in links:link_host = urlparse(link).netlocself.redis.zincrby(HOST_COUNT_LINK_KEY.format(date=date), 1, link_host)