Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: special case for __json__ logs (#121)
NTFSvolumed29ac6026a7d
diff --git a/cyberdrop_dl/clients/__init__.py b/cyberdrop_dl/clients/__init__.pyindex 4d86764c5..d1d31bb57 100644--- a/cyberdrop_dl/clients/__init__.py+++ b/cyberdrop_dl/clients/__init__.py@@ -119,7 +119,7 @@ class HTTPClient:exc = eraisefinally:- logger.debug("Finishing {} request [id={}]\n{}", method, request_id, resp)+ logger.debug("Finishing {} request [id={}]\n{}", method, request_id, str(resp))if self._save_responses_to_disk:_ = self.client_manager.manager.task_group.create_task(asyncio.to_thread(diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex d5648e9d4..aa01529f7 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -461,8 +461,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):if not self.manager.config.files.dump_json:return- data = [media_item.__json__()]- await self.manager.logs.write_jsonl(data)+ await self.manager.logs.write_jsonl([media_item.serialize()])@finalasync def check_complete(self, url: AbsoluteHttpURL, referer: AbsoluteHttpURL) -> bool:diff --git a/cyberdrop_dl/data_structures/url_objects.py b/cyberdrop_dl/data_structures/url_objects.pyindex 6a14fdb5a..3a28c2f9f 100644--- a/cyberdrop_dl/data_structures/url_objects.py+++ b/cyberdrop_dl/data_structures/url_objects.py@@ -237,12 +237,12 @@ class MediaItem:else:self._task_id = task_id- def __json__(self) -> dict[str, Any]:+ def serialize(self) -> dict[str, Any]:me = asdict(self)if self.hash:me["hash"] = f"xxh128:{self.hash}"for name in ("fallbacks", "_task_id", "is_segment", "parent_media_item"):- _ = me.pop(name)+ del me[name]return mediff --git a/cyberdrop_dl/ffmpeg.py b/cyberdrop_dl/ffmpeg.pyindex 2221d9830..6f3503764 100644--- a/cyberdrop_dl/ffmpeg.py+++ b/cyberdrop_dl/ffmpeg.py@@ -255,8 +255,8 @@ class Stream(DictDataclass):}@classmethod- def from_dict(cls, stream_info: dict[str, Any]) -> Self:- return cls(**cls.validate(stream_info))+ def from_dict(cls, data: dict[str, Any], /, **overrides: Any) -> Self:+ return cls(**cls.validate(data))@dataclasses.dataclass(slots=True, kw_only=True)diff --git a/cyberdrop_dl/logs.py b/cyberdrop_dl/logs.pyindex 59c9b7ecf..afe2d38ee 100644--- a/cyberdrop_dl/logs.py+++ b/cyberdrop_dl/logs.py@@ -30,6 +30,7 @@ _USER_NAME = Path.home().name_DEFAULT_CONSOLE_WIDTH = 240_MAIN_LOG_LISTENER: ContextVar[QueueListener] = ContextVar("_MAIN_LOGGER_LISTENER")MAIN_LOG_FILE: ContextVar[Path] = ContextVar("_MAIN_LOGGER_FILE")+LOG_TO_CONSOLE: ContextVar[bool] = ContextVar("LOG_TO_CONSOLE", default=True)if TYPE_CHECKING:@@ -37,8 +38,6 @@ if TYPE_CHECKING:from rich.console import ConsoleRenderable-LOG_TO_CONSOLE: ContextVar[bool] = ContextVar("LOG_TO_CONSOLE", default=True)-class RedactedConsole(Console):"""Custom console to remove username from logs"""@@ -73,12 +72,14 @@ class JsonLogRecord(logging.LogRecord):return msg@staticmethod- def _proccess_msg(msg: object) -> object:- if callable(dump := getattr(msg, "model_dump_json", None)):+ def _proccess_msg(obj: object) -> object:+ if callable(dump := getattr(obj, "model_dump_json", None)):return dump(indent=2, ensure_ascii=False)- if isinstance(msg, dict):- return json.dumps(msg, indent=2, ensure_ascii=False, default=str)- return msg+ if callable(dump := getattr(obj, "__json__", None)):+ return json.dumps(dump(), indent=2, ensure_ascii=False, default=str)+ if isinstance(obj, dict):+ return json.dumps(obj, indent=2, ensure_ascii=False, default=str)+ return objlogging.setLogRecordFactory(JsonLogRecord)@@ -252,7 +253,7 @@ def log_spacer(char: str = "-") -> None:@contextlib.contextmanager-def setup_console_logging(level: int = logging.DEBUG) -> Generator[None]:+def setup_console_logging(level: int = logging.INFO) -> Generator[None]:handler = LogHandler(level, show_time=False)logger.setLevel(logging.DEBUG)handler.addFilter(lambda _: LOG_TO_CONSOLE.get())diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.pyindex b76923ad2..cc4b7b6f2 100644--- a/cyberdrop_dl/managers/manager.py+++ b/cyberdrop_dl/managers/manager.py@@ -184,7 +184,7 @@ class Manager:"ffmpeg version": ffmpeg.get_ffmpeg_version(),"ffprobe version": ffmpeg.get_ffprobe_version(),}- logger.info(args_info)+ logger.debug(args_info)async def close(self) -> None:diff --git a/cyberdrop_dl/storage/_psutil.py b/cyberdrop_dl/storage/_psutil.pyindex bb99390ea..920afbeb6 100644--- a/cyberdrop_dl/storage/_psutil.py+++ b/cyberdrop_dl/storage/_psutil.py@@ -33,22 +33,16 @@ class DiskPartition:fstype: str = dataclasses.field(compare=False)opts: str = dataclasses.field(compare=False)-[email protected](frozen=True, slots=True, order=True)-class DiskPartitionStats:- partition: DiskPartition- free_space: ByteSize-- def __str__(self) -> str:- free_space = self.free_space.human_readable(decimal=True)- stats = dataclasses.asdict(self.partition) | {"free_space": free_space}- return ", ".join(f"'{k}': '{v}'" for k, v in stats.items())+ def __json__(self) -> dict[str, str]:+ return {k: str(v) for k, v in dataclasses.asdict(self).items()}class _Stats:+ def __json__(self) -> tuple[dict[str, str], ...]:+ return tuple(_partition_stats())+def __str__(self) -> str:- info = "\n".join(f" {stats!s}" for stats in _partition_stats())- return f"Storage status:\n {info}"+ return str(self.__json__())async def _get_free_space(path: Path) -> int:@@ -89,7 +83,7 @@ async def has_sufficient_space(folder: Path, /, required_free_space: int) -> boofree_space = _free_space[mount] = await _get_free_space(mount)logger.info(f"A new mountpoint ('{mount!s}') will be used for '{folder}'")- logger.info(_Stats())+ logger.info("Storage status \n{}", _Stats())return free_space == -1 or free_space > required_free_space@@ -120,11 +114,13 @@ def _partitions() -> tuple[DiskPartition, ...]:return tuple(_PARTITIONS)-def _partition_stats() -> Generator[DiskPartitionStats]:+def _partition_stats() -> Generator[dict[str, str]]:for partition in _partitions():free_space = _free_space.get(partition.mountpoint)if free_space is not None:- yield DiskPartitionStats(partition, ByteSize(free_space))+ stats = partition.__json__()+ stats["free_space"] = ByteSize(free_space).human_readable(decimal=True)+ yield statsdef clear_cache() -> None:@@ -153,7 +149,7 @@ async def start_loop() -> None:await update()if last_check % _LOG_PERIOD == 0:- logger.debug(_Stats())+ logger.debug("Storage status \n{}", _Stats())await asyncio.sleep(_CHECK_PERIOD)