Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
tests: add test for csv logs
NTFSvolumeaec2db580ffe
diff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.pyindex 74d5477c5..ce96b7594 100644--- a/cyberdrop_dl/crawlers/_forum.py+++ b/cyberdrop_dl/crawlers/_forum.py@@ -427,7 +427,7 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):scrape_item.add_children()finally:if post_url and post_url != thread.url:- self.manager.logs.write_last_post_log(post_url)+ self.manager.logs.write_last_forum_post(post_url)def _iter_posts(self, thread: ThreadProtocol, soup: BeautifulSoup) -> Generator[ForumPost]:for article in soup.select(self.SELECTORS.posts.article):diff --git a/cyberdrop_dl/csv_logs.py b/cyberdrop_dl/csv_logs.pyindex 2406ff6c9..f0ef38974 100644--- a/cyberdrop_dl/csv_logs.py+++ b/cyberdrop_dl/csv_logs.py@@ -9,7 +9,6 @@ from pathlib import Pathfrom typing import TYPE_CHECKING, Any, Selffrom cyberdrop_dl import constants-from cyberdrop_dl.exceptions import get_originfrom cyberdrop_dl.filepath import sanitize_filenamefrom cyberdrop_dl.utils import json@@ -17,9 +16,11 @@ if TYPE_CHECKING:import datetimefrom collections.abc import Iterable, Iterator+ import yarl+from cyberdrop_dl.clients.response import AbstractResponsefrom cyberdrop_dl.config import Config- from cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem+ from cyberdrop_dl.url_objects import AbsoluteHttpURLlogger = logging.getLogger(__name__)@@ -29,7 +30,6 @@ _CSV_DELIMITER = ","@dataclasses.dataclass(slots=True, kw_only=True)class CSVFiles:- main_log: Pathunsupported_urls: Pathdownload_errors: Pathscrape_errors: Path@@ -52,20 +52,20 @@ class CSVLogsManager:_responses_folder: Path = dataclasses.field(init=False)def __post_init__(self) -> None:- self._responses_folder = self.files.main_log.parent / "cdl_responses"+ self._responses_folder = self.files.jsonl_file.parent / "cdl_responses"@classmethoddef from_config(cls, config: Config) -> Self:files = config.logs.files- files = CSVFiles(- main_log=files.main,- unsupported_urls=files.unsupported,- download_errors=files.download_errors,- scrape_errors=files.scrape_errors,- jsonl_file=files.jsonl_file,- last_forum_post=files.last_forum_post,+ return cls(+ CSVFiles(+ unsupported_urls=files.unsupported,+ download_errors=files.download_errors,+ scrape_errors=files.scrape_errors,+ jsonl_file=files.jsonl_file,+ last_forum_post=files.last_forum_post,+ ))- return cls(files)def delete_old_logs(self) -> None:if self._ready:@@ -89,52 +89,44 @@ class CSVLogsManager:is_first_write = file not in self._has_headersself._has_headers.add(file)- def write() -> None:- if is_first_write:- file.parent.mkdir(parents=True, exist_ok=True)-- with file.open("a", encoding="utf8", newline="") as csv_file:- writer = csv.DictWriter(- csv_file,- fieldnames=tuple(row),- delimiter=_CSV_DELIMITER,- quoting=csv.QUOTE_ALL,- )- if is_first_write:- writer.writeheader()- writer.writerow(row)-- await asyncio.to_thread(write)-- def write_unsupported(self, url: AbsoluteHttpURL, origin: AbsoluteHttpURL | None = None) -> None:- """Writes to the unsupported urls log."""+ await asyncio.to_thread(_write_to_csv, file, row, write_headers=is_first_write)++ def write_unsupported(self, url: AbsoluteHttpURL, origin: yarl.URL | Path | None = None) -> None:_ = self.task_group.create_task(self._write_to_csv(self.files.unsupported_urls, url=url, origin=origin))- def write_last_post_log(self, url: AbsoluteHttpURL) -> None:+ def write_last_forum_post(self, url: AbsoluteHttpURL) -> None:_ = self.task_group.create_task(self._write_to_csv(self.files.last_forum_post, url=url))- def write_download_error(self, media_item: MediaItem, error_message: str) -> None:- """Writes to the download error log."""- origin = get_origin(media_item)+ def write_download_error(+ self,+ url: AbsoluteHttpURL,+ referer: AbsoluteHttpURL,+ error_message: str,+ origin: yarl.URL | Path | None = None,+ ) -> None:_ = self.task_group.create_task(self._write_to_csv(self.files.download_errors,- url=media_item.url,+ url=url,error=error_message,- referer=media_item.referer,+ referer=referer,origin=origin,))def write_scrape_error(self,- url: AbsoluteHttpURL | str,+ url: yarl.URL | str,error_message: str,- origin: AbsoluteHttpURL | Path | None = None,+ origin: yarl.URL | Path | None = None,) -> None:- """Writes to the scrape error log."""_ = self.task_group.create_task(- self._write_to_csv(self.files.scrape_errors, url=url, error=error_message, origin=origin)+ self._write_to_csv(+ self.files.scrape_errors,+ url=url,+ error=error_message,+ origin=origin,+ ))def write_response(@@ -176,3 +168,19 @@ def _prepare_resp_file(folder: Path, url: AbsoluteHttpURL, created_at: datetime.path_safe_url = sanitize_filename(Path(str(url)).as_posix().replace("/", "-"))filename = f"{path_safe_url[:max_stem_len]}_{log_date}{ext}"return folder / filename+++def _write_to_csv(file: Path, row: dict[str, object], *, write_headers: bool) -> None:+ if write_headers:+ file.parent.mkdir(parents=True, exist_ok=True)++ with file.open("a", encoding="utf8", newline="") as csv_file:+ writer = csv.DictWriter(+ csv_file,+ fieldnames=tuple(row),+ delimiter=_CSV_DELIMITER,+ quoting=csv.QUOTE_ALL,+ )+ if write_headers:+ writer.writeheader()+ writer.writerow(row)diff --git a/cyberdrop_dl/utils/errors.py b/cyberdrop_dl/utils/errors.pyindex 0412614c3..82c709fd4 100644--- a/cyberdrop_dl/utils/errors.py+++ b/cyberdrop_dl/utils/errors.py@@ -180,7 +180,7 @@ def _log_error( # noqa: PLR0913item: ScrapeItem | MediaItem | yarl.URL,app_error: CDLAppError,exc: BaseException | None,- origin: ScrapeItem | MediaItem | yarl.URL | Path | None,+ origin: yarl.URL | Path | None,) -> None:origin = origin or get_origin(item)is_downloader = bool(getattr(self, "log_prefix", False))@@ -190,13 +190,13 @@ def _log_error( # noqa: PLR0913f"{self.log_prefix} Failed: {item.url} ({app_error.msg}) \n -> Referer: {item.referer}",exc_info=exc,)- self.manager.logs.write_download_error(item, app_error.csv_msg)+ self.manager.logs.write_download_error(item.url, item.referer, app_error.csv_msg, origin)self.manager.scrape_mapper.tui.files.stats.failed += 1self.manager.scrape_mapper.tui.download_errors.add(app_error.ui_error)returnlogger.error(f"Scrape Failed: {url} ({app_error.msg})", exc_info=exc)- self.manager.logs.write_scrape_error(url, app_error.csv_msg, origin) # pyright: ignore[reportArgumentType]+ self.manager.logs.write_scrape_error(url, app_error.csv_msg, origin)self.manager.scrape_mapper.tui.scrape_errors.add(app_error.ui_error)diff --git a/tests/test_csv_logs.py b/tests/test_csv_logs.pyindex 6662deb5d..8903a75f0 100644--- a/tests/test_csv_logs.py+++ b/tests/test_csv_logs.py@@ -3,7 +3,7 @@ from pathlib import Pathimport pytest-from cyberdrop_dl import csv_logs+from cyberdrop_dl.csv_logs import _prepare_resp_file, _write_to_csvfrom cyberdrop_dl.url_objects import AbsoluteHttpURLnow = datetime.datetime(2026, 5, 8, tzinfo=datetime.UTC)@@ -27,6 +27,65 @@ now = datetime.datetime(2026, 5, 8, tzinfo=datetime.UTC)],)def test_prepare_resp_filename(url: str, expected: str) -> None:- result = csv_logs._prepare_resp_file(Path("/"), AbsoluteHttpURL(url), now)+ result = _prepare_resp_file(Path("/"), AbsoluteHttpURL(url), now)assert result.as_posix().count("/") == 1assert result.as_posix() == expected+++class TestWriteToCsv:+ @pytest.fixture+ def row(self) -> dict[str, object]:+ return {"url": "https://example.com", "origin": "URLs.txt"}++ def test_parent_dirs_are_created_when_write_headers_true(self, tmp_path: Path, row: dict[str, object]) -> None:+ file = tmp_path / "subfolder" / "errors.csv"+ _write_to_csv(file, row, write_headers=True)+ assert file.exists()++ def test_parent_dirs_are_not_created_when_write_headers_false(self, tmp_path: Path, row: dict[str, object]) -> None:+ file = tmp_path / "subfolder" / "errors.csv"+ with pytest.raises(FileNotFoundError):+ _write_to_csv(file, row, write_headers=False)++ def test_file_is_always_appended_to(self, tmp_path: Path, row: dict[str, object]) -> None:+ file = tmp_path / "subfolder" / "errors.csv"+ file.parent.mkdir()+ header = "<<TEST_HEADER>>"+ file.write_text(header + "\n")+ _write_to_csv(file, row, write_headers=False)++ lines = file.read_text().splitlines()+ assert len(lines) == 2+ assert lines[0] == header+ assert lines[1] == '"https://example.com","URLs.txt"'++ def test_write_headers_true(self, tmp_path: Path, row: dict[str, object]) -> None:+ file = tmp_path / "subfolder" / "errors.csv"+ _write_to_csv(file, row, write_headers=True)+ assert file.read_text().splitlines()[0] == '"url","origin"'++ def test_write_headers_false(self, tmp_path: Path, row: dict[str, object]) -> None:+ file = tmp_path / "errors.csv"+ _write_to_csv(file, row, write_headers=False)+ assert file.read_text().splitlines()[0] == '"https://example.com","URLs.txt"'++ def test_row_w_various_data_types(self, tmp_path: Path) -> None:+ file = tmp_path / "subfolder" / "errors.csv"++ class CustomType:+ def __init__(self, a: str) -> None:+ self.a: str = a++ _write_to_csv(+ file,+ row={+ "string": "hello",+ "integer": 42,+ "float": 3.14,+ "boolean": True,+ "none_value": None,+ "list_value": [1, 2, 3],+ "custom_type": CustomType("test"),+ },+ write_headers=True,+ )