Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Add resolved URL and anchor text to links in batch

Daoud Clarke38fed25fa46a
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.py
index 720160b..55ea1d5 100644
--- a/mwmbl/crawler/app.py
+++ b/mwmbl/crawler/app.py
@@ -55,6 +55,10 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versio
@router.post('/batches/')
def post_batch(request, batch: Batch):
+ """
+ Store a batch of crawled pages.
+ """
+
if len(batch.items) > MAX_BATCH_SIZE:
raise HTTPException(400, f"Batch size too large (maximum {MAX_BATCH_SIZE}), got {len(batch.items)}")
diff --git a/mwmbl/crawler/batch.py b/mwmbl/crawler/batch.py
index 812d408..c9a43e5 100644
--- a/mwmbl/crawler/batch.py
+++ b/mwmbl/crawler/batch.py
@@ -3,10 +3,19 @@ from typing import Optional
from ninja import Schema
+class Link(Schema):
+ url: str
+ anchor_text: str
+ link_type: str
+
+
class ItemContent(Schema):
title: str
extract: str
- links: list[str]
+ link_details: Optional[list[Link]]
+
+ # Deprecated
+ links: Optional[list[str]]
extra_links: Optional[list[str]]
links_only: Optional[bool]
@@ -18,6 +27,7 @@ class ItemError(Schema):
class Item(Schema):
url: str
+ resolved_url: Optional[str]
status: Optional[int]
timestamp: int
content: Optional[ItemContent]
diff --git a/mwmbl/crawler/stats.py b/mwmbl/crawler/stats.py
index 932399e..ac35906 100644
--- a/mwmbl/crawler/stats.py
+++ b/mwmbl/crawler/stats.py
@@ -109,7 +109,15 @@ class StatsManager:
self.redis.zincrby(host_key, 1, host)
- for link in item.content.links:
+ links = []
+ if item.content.links is not None:
+ links += item.content.links
+ if item.content.extra_links is not None:
+ links += item.content.extra_links
+ if item.content.link_details is not None:
+ links += [link.url for link in item.content.link_details]
+
+ for link in links:
link_host = urlparse(link).netloc
self.redis.zincrby(HOST_COUNT_LINK_KEY.format(date=date), 1, link_host)