An open source, non-profit web search engine
Fix last_crawled timestamp being in the future
Daoud Clarke4cb4e69bb522
diff --git a/mwmbl/indexer/index_batches.py b/mwmbl/indexer/index_batches.pyindex de7b578..8097d67 100644--- a/mwmbl/indexer/index_batches.py+++ b/mwmbl/indexer/index_batches.py@@ -40,7 +40,7 @@ def get_documents_from_batches(batches: Collection[HashedBatch]) -> Iterable[Docif item.content is not None and not item.content.links_only:yield Document(item.content.title, item.url, item.content.extract,- last_crawled=int(item.timestamp),+ last_crawled=int(item.timestamp / 1000),)