Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: special case for __json__ logs (#121)

NTFSvolumed29ac6026a7d
diff --git a/cyberdrop_dl/clients/__init__.py b/cyberdrop_dl/clients/__init__.py
index 4d86764c5..d1d31bb57 100644
--- a/cyberdrop_dl/clients/__init__.py
+++ b/cyberdrop_dl/clients/__init__.py
@@ -119,7 +119,7 @@ class HTTPClient:
exc = e
raise
finally:
- logger.debug("Finishing {} request [id={}]\n{}", method, request_id, resp)
+ logger.debug("Finishing {} request [id={}]\n{}", method, request_id, str(resp))
if self._save_responses_to_disk:
_ = self.client_manager.manager.task_group.create_task(
asyncio.to_thread(
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index d5648e9d4..aa01529f7 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -461,8 +461,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
if not self.manager.config.files.dump_json:
return
- data = [media_item.__json__()]
- await self.manager.logs.write_jsonl(data)
+ await self.manager.logs.write_jsonl([media_item.serialize()])
@final
async def check_complete(self, url: AbsoluteHttpURL, referer: AbsoluteHttpURL) -> bool:
diff --git a/cyberdrop_dl/data_structures/url_objects.py b/cyberdrop_dl/data_structures/url_objects.py
index 6a14fdb5a..3a28c2f9f 100644
--- a/cyberdrop_dl/data_structures/url_objects.py
+++ b/cyberdrop_dl/data_structures/url_objects.py
@@ -237,12 +237,12 @@ class MediaItem:
else:
self._task_id = task_id
- def __json__(self) -> dict[str, Any]:
+ def serialize(self) -> dict[str, Any]:
me = asdict(self)
if self.hash:
me["hash"] = f"xxh128:{self.hash}"
for name in ("fallbacks", "_task_id", "is_segment", "parent_media_item"):
- _ = me.pop(name)
+ del me[name]
return me
diff --git a/cyberdrop_dl/ffmpeg.py b/cyberdrop_dl/ffmpeg.py
index 2221d9830..6f3503764 100644
--- a/cyberdrop_dl/ffmpeg.py
+++ b/cyberdrop_dl/ffmpeg.py
@@ -255,8 +255,8 @@ class Stream(DictDataclass):
}
@classmethod
- def from_dict(cls, stream_info: dict[str, Any]) -> Self:
- return cls(**cls.validate(stream_info))
+ def from_dict(cls, data: dict[str, Any], /, **overrides: Any) -> Self:
+ return cls(**cls.validate(data))
@dataclasses.dataclass(slots=True, kw_only=True)
diff --git a/cyberdrop_dl/logs.py b/cyberdrop_dl/logs.py
index 59c9b7ecf..afe2d38ee 100644
--- a/cyberdrop_dl/logs.py
+++ b/cyberdrop_dl/logs.py
@@ -30,6 +30,7 @@ _USER_NAME = Path.home().name
_DEFAULT_CONSOLE_WIDTH = 240
_MAIN_LOG_LISTENER: ContextVar[QueueListener] = ContextVar("_MAIN_LOGGER_LISTENER")
MAIN_LOG_FILE: ContextVar[Path] = ContextVar("_MAIN_LOGGER_FILE")
+LOG_TO_CONSOLE: ContextVar[bool] = ContextVar("LOG_TO_CONSOLE", default=True)
if TYPE_CHECKING:
@@ -37,8 +38,6 @@ if TYPE_CHECKING:
from rich.console import ConsoleRenderable
-LOG_TO_CONSOLE: ContextVar[bool] = ContextVar("LOG_TO_CONSOLE", default=True)
-
class RedactedConsole(Console):
"""Custom console to remove username from logs"""
@@ -73,12 +72,14 @@ class JsonLogRecord(logging.LogRecord):
return msg
@staticmethod
- def _proccess_msg(msg: object) -> object:
- if callable(dump := getattr(msg, "model_dump_json", None)):
+ def _proccess_msg(obj: object) -> object:
+ if callable(dump := getattr(obj, "model_dump_json", None)):
return dump(indent=2, ensure_ascii=False)
- if isinstance(msg, dict):
- return json.dumps(msg, indent=2, ensure_ascii=False, default=str)
- return msg
+ if callable(dump := getattr(obj, "__json__", None)):
+ return json.dumps(dump(), indent=2, ensure_ascii=False, default=str)
+ if isinstance(obj, dict):
+ return json.dumps(obj, indent=2, ensure_ascii=False, default=str)
+ return obj
logging.setLogRecordFactory(JsonLogRecord)
@@ -252,7 +253,7 @@ def log_spacer(char: str = "-") -> None:
@contextlib.contextmanager
-def setup_console_logging(level: int = logging.DEBUG) -> Generator[None]:
+def setup_console_logging(level: int = logging.INFO) -> Generator[None]:
handler = LogHandler(level, show_time=False)
logger.setLevel(logging.DEBUG)
handler.addFilter(lambda _: LOG_TO_CONSOLE.get())
diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.py
index b76923ad2..cc4b7b6f2 100644
--- a/cyberdrop_dl/managers/manager.py
+++ b/cyberdrop_dl/managers/manager.py
@@ -184,7 +184,7 @@ class Manager:
"ffmpeg version": ffmpeg.get_ffmpeg_version(),
"ffprobe version": ffmpeg.get_ffprobe_version(),
}
- logger.info(args_info)
+ logger.debug(args_info)
async def close(self) -> None:
diff --git a/cyberdrop_dl/storage/_psutil.py b/cyberdrop_dl/storage/_psutil.py
index bb99390ea..920afbeb6 100644
--- a/cyberdrop_dl/storage/_psutil.py
+++ b/cyberdrop_dl/storage/_psutil.py
@@ -33,22 +33,16 @@ class DiskPartition:
fstype: str = dataclasses.field(compare=False)
opts: str = dataclasses.field(compare=False)
-
[email protected](frozen=True, slots=True, order=True)
-class DiskPartitionStats:
- partition: DiskPartition
- free_space: ByteSize
-
- def __str__(self) -> str:
- free_space = self.free_space.human_readable(decimal=True)
- stats = dataclasses.asdict(self.partition) | {"free_space": free_space}
- return ", ".join(f"'{k}': '{v}'" for k, v in stats.items())
+ def __json__(self) -> dict[str, str]:
+ return {k: str(v) for k, v in dataclasses.asdict(self).items()}
class _Stats:
+ def __json__(self) -> tuple[dict[str, str], ...]:
+ return tuple(_partition_stats())
+
def __str__(self) -> str:
- info = "\n".join(f" {stats!s}" for stats in _partition_stats())
- return f"Storage status:\n {info}"
+ return str(self.__json__())
async def _get_free_space(path: Path) -> int:
@@ -89,7 +83,7 @@ async def has_sufficient_space(folder: Path, /, required_free_space: int) -> boo
free_space = _free_space[mount] = await _get_free_space(mount)
logger.info(f"A new mountpoint ('{mount!s}') will be used for '{folder}'")
- logger.info(_Stats())
+ logger.info("Storage status \n{}", _Stats())
return free_space == -1 or free_space > required_free_space
@@ -120,11 +114,13 @@ def _partitions() -> tuple[DiskPartition, ...]:
return tuple(_PARTITIONS)
-def _partition_stats() -> Generator[DiskPartitionStats]:
+def _partition_stats() -> Generator[dict[str, str]]:
for partition in _partitions():
free_space = _free_space.get(partition.mountpoint)
if free_space is not None:
- yield DiskPartitionStats(partition, ByteSize(free_space))
+ stats = partition.__json__()
+ stats["free_space"] = ByteSize(free_space).human_readable(decimal=True)
+ yield stats
def clear_cache() -> None:
@@ -153,7 +149,7 @@ async def start_loop() -> None:
await update()
if last_check % _LOG_PERIOD == 0:
- logger.debug(_Stats())
+ logger.debug("Storage status \n{}", _Stats())
await asyncio.sleep(_CHECK_PERIOD)