Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Fix broken tests

Daoud Clarkea76dfea52f15
diff --git a/pyproject.toml b/pyproject.toml
index cb8cba4..f24d8cd 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -77,3 +77,7 @@ build-backend = "poetry.core.masonry.api"
[tool.poetry.scripts]
mwmbl-tinysearchengine = "mwmbl.main:run"
+
+[tool.pytest.ini_options]
+log_cli = true
+log_cli_level = "INFO"
diff --git a/test/test_update_urls.py b/test/test_update_urls.py
index 387fbdf..1e89acd 100644
--- a/test/test_update_urls.py
+++ b/test/test_update_urls.py
@@ -1,3 +1,5 @@
+from collections import defaultdict
+
from mwmbl.indexer.update_urls import process_link
@@ -5,6 +7,7 @@ def test_process_link_normal():
url_scores = {"https://somesite.com/something.html": 0.0, "https://somesite.com/": 0.0}
url_timestamps = {}
url_users = {}
+ domain_links = defaultdict(set)
process_link(
user_id_hash="abc123",
@@ -13,16 +16,18 @@ def test_process_link_normal():
timestamp=1234,
url_timestamps=url_timestamps,
url_users=url_users,
- blacklist_domains=[]
+ blacklist_domains=[],
+ domain_links=domain_links,
)
- assert url_scores["https://somesite.com/something.html"] > 0.0
+ assert domain_links == {"somewhere.com": {"somesite.com"}}
def test_process_link_excludes_porn():
url_scores = {}
url_timestamps = {}
url_users = {}
+ domain_links = {}
process_link(
user_id_hash="abc123",
@@ -31,9 +36,11 @@ def test_process_link_excludes_porn():
timestamp=1234,
url_timestamps=url_timestamps,
url_users=url_users,
- blacklist_domains=[]
+ blacklist_domains=[],
+ domain_links=domain_links,
)
assert url_scores == {}
assert url_timestamps == {}
assert url_users == {}
+ assert domain_links == {}
diff --git a/test/test_url_queue.py b/test/test_url_queue.py
index 9c5d7ac..5d218e5 100644
--- a/test/test_url_queue.py
+++ b/test/test_url_queue.py
@@ -1,3 +1,4 @@
+import logging
from datetime import datetime
from queue import Queue
@@ -5,12 +6,13 @@ from mwmbl.crawler.urls import FoundURL, URLStatus
from mwmbl.url_queue import URLQueue
-def test_url_queue_empties():
+def test_url_queue_empties(caplog):
+ caplog.set_level(logging.INFO)
new_item_queue = Queue()
queued_batches = Queue()
url_queue = URLQueue(new_item_queue, queued_batches, min_top_domains=1)
- new_item_queue.put([FoundURL("https://google.com", "123", URLStatus.NEW.value, datetime(2023, 1, 19))])
+ new_item_queue.put([FoundURL("https://google.com", "123", URLStatus.NEW, datetime(2023, 1, 19))])
url_queue.update()
@@ -19,15 +21,16 @@ def test_url_queue_empties():
assert items == ["https://google.com"]
-def test_url_queue_multiple_puts():
+def test_url_queue_multiple_puts(caplog):
+ caplog.set_level(logging.INFO)
new_item_queue = Queue()
queued_batches = Queue()
url_queue = URLQueue(new_item_queue, queued_batches, min_top_domains=1)
- new_item_queue.put([FoundURL("https://google.com", "123", URLStatus.NEW.value, datetime(2023, 1, 19))])
+ new_item_queue.put([FoundURL("https://google.com", "123", URLStatus.NEW, datetime(2023, 1, 19))])
url_queue.update()
- new_item_queue.put([FoundURL("https://www.supermemo.com", "124", URLStatus.NEW.value, datetime(2023, 1, 20))])
+ new_item_queue.put([FoundURL("https://www.supermemo.com", "124", URLStatus.NEW, datetime(2023, 1, 20))])
url_queue.update()
items = queued_batches.get(block=False)