Skip to content

Bulk asynchronous downloader for multiple file hosts

tests: add test for csv logs

NTFSvolumeaec2db580ffe
diff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.py
index 74d5477c5..ce96b7594 100644
--- a/cyberdrop_dl/crawlers/_forum.py
+++ b/cyberdrop_dl/crawlers/_forum.py
@@ -427,7 +427,7 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):
scrape_item.add_children()
finally:
if post_url and post_url != thread.url:
- self.manager.logs.write_last_post_log(post_url)
+ self.manager.logs.write_last_forum_post(post_url)
def _iter_posts(self, thread: ThreadProtocol, soup: BeautifulSoup) -> Generator[ForumPost]:
for article in soup.select(self.SELECTORS.posts.article):
diff --git a/cyberdrop_dl/csv_logs.py b/cyberdrop_dl/csv_logs.py
index 2406ff6c9..f0ef38974 100644
--- a/cyberdrop_dl/csv_logs.py
+++ b/cyberdrop_dl/csv_logs.py
@@ -9,7 +9,6 @@ from pathlib import Path
from typing import TYPE_CHECKING, Any, Self
from cyberdrop_dl import constants
-from cyberdrop_dl.exceptions import get_origin
from cyberdrop_dl.filepath import sanitize_filename
from cyberdrop_dl.utils import json
@@ -17,9 +16,11 @@ if TYPE_CHECKING:
import datetime
from collections.abc import Iterable, Iterator
+ import yarl
+
from cyberdrop_dl.clients.response import AbstractResponse
from cyberdrop_dl.config import Config
- from cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem
+ from cyberdrop_dl.url_objects import AbsoluteHttpURL
logger = logging.getLogger(__name__)
@@ -29,7 +30,6 @@ _CSV_DELIMITER = ","
@dataclasses.dataclass(slots=True, kw_only=True)
class CSVFiles:
- main_log: Path
unsupported_urls: Path
download_errors: Path
scrape_errors: Path
@@ -52,20 +52,20 @@ class CSVLogsManager:
_responses_folder: Path = dataclasses.field(init=False)
def __post_init__(self) -> None:
- self._responses_folder = self.files.main_log.parent / "cdl_responses"
+ self._responses_folder = self.files.jsonl_file.parent / "cdl_responses"
@classmethod
def from_config(cls, config: Config) -> Self:
files = config.logs.files
- files = CSVFiles(
- main_log=files.main,
- unsupported_urls=files.unsupported,
- download_errors=files.download_errors,
- scrape_errors=files.scrape_errors,
- jsonl_file=files.jsonl_file,
- last_forum_post=files.last_forum_post,
+ return cls(
+ CSVFiles(
+ unsupported_urls=files.unsupported,
+ download_errors=files.download_errors,
+ scrape_errors=files.scrape_errors,
+ jsonl_file=files.jsonl_file,
+ last_forum_post=files.last_forum_post,
+ )
)
- return cls(files)
def delete_old_logs(self) -> None:
if self._ready:
@@ -89,52 +89,44 @@ class CSVLogsManager:
is_first_write = file not in self._has_headers
self._has_headers.add(file)
- def write() -> None:
- if is_first_write:
- file.parent.mkdir(parents=True, exist_ok=True)
-
- with file.open("a", encoding="utf8", newline="") as csv_file:
- writer = csv.DictWriter(
- csv_file,
- fieldnames=tuple(row),
- delimiter=_CSV_DELIMITER,
- quoting=csv.QUOTE_ALL,
- )
- if is_first_write:
- writer.writeheader()
- writer.writerow(row)
-
- await asyncio.to_thread(write)
-
- def write_unsupported(self, url: AbsoluteHttpURL, origin: AbsoluteHttpURL | None = None) -> None:
- """Writes to the unsupported urls log."""
+ await asyncio.to_thread(_write_to_csv, file, row, write_headers=is_first_write)
+
+ def write_unsupported(self, url: AbsoluteHttpURL, origin: yarl.URL | Path | None = None) -> None:
_ = self.task_group.create_task(self._write_to_csv(self.files.unsupported_urls, url=url, origin=origin))
- def write_last_post_log(self, url: AbsoluteHttpURL) -> None:
+ def write_last_forum_post(self, url: AbsoluteHttpURL) -> None:
_ = self.task_group.create_task(self._write_to_csv(self.files.last_forum_post, url=url))
- def write_download_error(self, media_item: MediaItem, error_message: str) -> None:
- """Writes to the download error log."""
- origin = get_origin(media_item)
+ def write_download_error(
+ self,
+ url: AbsoluteHttpURL,
+ referer: AbsoluteHttpURL,
+ error_message: str,
+ origin: yarl.URL | Path | None = None,
+ ) -> None:
_ = self.task_group.create_task(
self._write_to_csv(
self.files.download_errors,
- url=media_item.url,
+ url=url,
error=error_message,
- referer=media_item.referer,
+ referer=referer,
origin=origin,
)
)
def write_scrape_error(
self,
- url: AbsoluteHttpURL | str,
+ url: yarl.URL | str,
error_message: str,
- origin: AbsoluteHttpURL | Path | None = None,
+ origin: yarl.URL | Path | None = None,
) -> None:
- """Writes to the scrape error log."""
_ = self.task_group.create_task(
- self._write_to_csv(self.files.scrape_errors, url=url, error=error_message, origin=origin)
+ self._write_to_csv(
+ self.files.scrape_errors,
+ url=url,
+ error=error_message,
+ origin=origin,
+ )
)
def write_response(
@@ -176,3 +168,19 @@ def _prepare_resp_file(folder: Path, url: AbsoluteHttpURL, created_at: datetime.
path_safe_url = sanitize_filename(Path(str(url)).as_posix().replace("/", "-"))
filename = f"{path_safe_url[:max_stem_len]}_{log_date}{ext}"
return folder / filename
+
+
+def _write_to_csv(file: Path, row: dict[str, object], *, write_headers: bool) -> None:
+ if write_headers:
+ file.parent.mkdir(parents=True, exist_ok=True)
+
+ with file.open("a", encoding="utf8", newline="") as csv_file:
+ writer = csv.DictWriter(
+ csv_file,
+ fieldnames=tuple(row),
+ delimiter=_CSV_DELIMITER,
+ quoting=csv.QUOTE_ALL,
+ )
+ if write_headers:
+ writer.writeheader()
+ writer.writerow(row)
diff --git a/cyberdrop_dl/utils/errors.py b/cyberdrop_dl/utils/errors.py
index 0412614c3..82c709fd4 100644
--- a/cyberdrop_dl/utils/errors.py
+++ b/cyberdrop_dl/utils/errors.py
@@ -180,7 +180,7 @@ def _log_error( # noqa: PLR0913
item: ScrapeItem | MediaItem | yarl.URL,
app_error: CDLAppError,
exc: BaseException | None,
- origin: ScrapeItem | MediaItem | yarl.URL | Path | None,
+ origin: yarl.URL | Path | None,
) -> None:
origin = origin or get_origin(item)
is_downloader = bool(getattr(self, "log_prefix", False))
@@ -190,13 +190,13 @@ def _log_error( # noqa: PLR0913
f"{self.log_prefix} Failed: {item.url} ({app_error.msg}) \n -> Referer: {item.referer}",
exc_info=exc,
)
- self.manager.logs.write_download_error(item, app_error.csv_msg)
+ self.manager.logs.write_download_error(item.url, item.referer, app_error.csv_msg, origin)
self.manager.scrape_mapper.tui.files.stats.failed += 1
self.manager.scrape_mapper.tui.download_errors.add(app_error.ui_error)
return
logger.error(f"Scrape Failed: {url} ({app_error.msg})", exc_info=exc)
- self.manager.logs.write_scrape_error(url, app_error.csv_msg, origin) # pyright: ignore[reportArgumentType]
+ self.manager.logs.write_scrape_error(url, app_error.csv_msg, origin)
self.manager.scrape_mapper.tui.scrape_errors.add(app_error.ui_error)
diff --git a/tests/test_csv_logs.py b/tests/test_csv_logs.py
index 6662deb5d..8903a75f0 100644
--- a/tests/test_csv_logs.py
+++ b/tests/test_csv_logs.py
@@ -3,7 +3,7 @@ from pathlib import Path
import pytest
-from cyberdrop_dl import csv_logs
+from cyberdrop_dl.csv_logs import _prepare_resp_file, _write_to_csv
from cyberdrop_dl.url_objects import AbsoluteHttpURL
now = datetime.datetime(2026, 5, 8, tzinfo=datetime.UTC)
@@ -27,6 +27,65 @@ now = datetime.datetime(2026, 5, 8, tzinfo=datetime.UTC)
],
)
def test_prepare_resp_filename(url: str, expected: str) -> None:
- result = csv_logs._prepare_resp_file(Path("/"), AbsoluteHttpURL(url), now)
+ result = _prepare_resp_file(Path("/"), AbsoluteHttpURL(url), now)
assert result.as_posix().count("/") == 1
assert result.as_posix() == expected
+
+
+class TestWriteToCsv:
+ @pytest.fixture
+ def row(self) -> dict[str, object]:
+ return {"url": "https://example.com", "origin": "URLs.txt"}
+
+ def test_parent_dirs_are_created_when_write_headers_true(self, tmp_path: Path, row: dict[str, object]) -> None:
+ file = tmp_path / "subfolder" / "errors.csv"
+ _write_to_csv(file, row, write_headers=True)
+ assert file.exists()
+
+ def test_parent_dirs_are_not_created_when_write_headers_false(self, tmp_path: Path, row: dict[str, object]) -> None:
+ file = tmp_path / "subfolder" / "errors.csv"
+ with pytest.raises(FileNotFoundError):
+ _write_to_csv(file, row, write_headers=False)
+
+ def test_file_is_always_appended_to(self, tmp_path: Path, row: dict[str, object]) -> None:
+ file = tmp_path / "subfolder" / "errors.csv"
+ file.parent.mkdir()
+ header = "<<TEST_HEADER>>"
+ file.write_text(header + "\n")
+ _write_to_csv(file, row, write_headers=False)
+
+ lines = file.read_text().splitlines()
+ assert len(lines) == 2
+ assert lines[0] == header
+ assert lines[1] == '"https://example.com","URLs.txt"'
+
+ def test_write_headers_true(self, tmp_path: Path, row: dict[str, object]) -> None:
+ file = tmp_path / "subfolder" / "errors.csv"
+ _write_to_csv(file, row, write_headers=True)
+ assert file.read_text().splitlines()[0] == '"url","origin"'
+
+ def test_write_headers_false(self, tmp_path: Path, row: dict[str, object]) -> None:
+ file = tmp_path / "errors.csv"
+ _write_to_csv(file, row, write_headers=False)
+ assert file.read_text().splitlines()[0] == '"https://example.com","URLs.txt"'
+
+ def test_row_w_various_data_types(self, tmp_path: Path) -> None:
+ file = tmp_path / "subfolder" / "errors.csv"
+
+ class CustomType:
+ def __init__(self, a: str) -> None:
+ self.a: str = a
+
+ _write_to_csv(
+ file,
+ row={
+ "string": "hello",
+ "integer": 42,
+ "float": 3.14,
+ "boolean": True,
+ "none_value": None,
+ "list_value": [1, 2, 3],
+ "custom_type": CustomType("test"),
+ },
+ write_headers=True,
+ )