Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: factor out resp check logic
NTFSvolumeb2913aa6e24a
diff --git a/cyberdrop_dl/clients/downloads.py b/cyberdrop_dl/clients/downloads.pyindex a6f0265e7..17b4ebc1a 100644--- a/cyberdrop_dl/clients/downloads.py+++ b/cyberdrop_dl/clients/downloads.py@@ -11,6 +11,7 @@ from aiohttp import hdrsfrom cyberdrop_dl import aio, constants, ffmpeg, storagefrom cyberdrop_dl.clients import etag+from cyberdrop_dl.clients.http import check_http_statusfrom cyberdrop_dl.constants import FileExt, HashModefrom cyberdrop_dl.exceptions import DownloadError, InvalidContentTypeError, SlowDownloadErrorfrom cyberdrop_dl.hasher import compute_in_place_hash@@ -107,42 +108,21 @@ class DownloadClient:resume_point: int,resp: AbstractResponse[Any],) -> bool:- if resp.status == HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE:- await aio.unlink(media_item.partial_file)-- etag.check(resp.headers)- await self.http_client.check_http_status(resp)-- if not media_item.is_segment and (content_type := _get_content_type(resp.headers)):- _check_content_type(content_type, media_item.ext)-- media_item.size = _get_content_length(resp.headers)+ await _check_response(media_item, resp, resume_point)+ media_item.size = _get_content_length(resp.headers) + resume_point+ _set_upload_date(media_item, resp.headers)if not media_item.path:- _check_content_length(resp.headers)downloaded = await self._predownload_skip(media_item, domain)if downloaded is not None:return downloaded- if resp.status != HTTPStatus.PARTIAL_CONTENT:- await aio.unlink(media_item.partial_file, missing_ok=True)-- if (- not media_item.is_segment- and not media_item.uploaded_at- and (last_modified := _get_last_modified(resp.headers))- ):- logger.warning(- f"Unable to parse upload date for {media_item.url}, using `{hdrs.LAST_MODIFIED}` header as file datetime"- )- media_item.uploaded_at = last_modified-hook = self._make_hook(media_item, resume_point)if resume_point:hook.advance(resume_point)with hook:await self._append_content(media_item, hook, resp)- return True+ return Truedef _make_hook(self, media_item: MediaItem, resume_point: int) -> ProgressHook:if media_item.is_segment:@@ -185,7 +165,7 @@ class DownloadClient:await aio.unlink(media_item.partial_file, missing_ok=True)raise DownloadError(HTTPStatus.INTERNAL_SERVER_ERROR, "File is empty")- assert media_item.size+ assert media_item.size is not Noneif size < media_item.size:await aio.unlink(media_item.partial_file, missing_ok=True)raise DownloadError(@@ -407,10 +387,10 @@ def _get_last_modified(headers: Mapping[str, str]) -> int | None:def _is_html_or_text(content_type: str) -> bool:- return any(s in content_type for s in ("html", "text"))+ return "html" in content_type or "text" in content_type-def _check_content_length(headers: Mapping[str, str]) -> None:+def _check_for_placeholder_files(headers: Mapping[str, str]) -> None:match headers.get(hdrs.CONTENT_TYPE):case "video/mp4":match headers.get(hdrs.CONTENT_LENGTH):@@ -487,10 +467,41 @@ def make_speed_checker(media_item: MediaItem, hook: ProgressHook, speed_thresholreturn check_download_speed-def _get_content_length(headers: Mapping[str, str]) -> int:+def _get_content_length(headers: Mapping[str, str], *, _required: bool = False) -> int:try:return int(headers[hdrs.CONTENT_LENGTH])except KeyError:- return 0+ if not _required:+ return 0msg = f"Download response has no `{hdrs.CONTENT_LENGTH}` header. Refusing to download"raise DownloadError(HTTPStatus.LENGTH_REQUIRED, msg, retry=False) from None+++def _set_upload_date(media_item: MediaItem, headers: Mapping[str, str]) -> None:+ if not media_item.is_segment and not media_item.uploaded_at and (last_modified := _get_last_modified(headers)):+ logger.warning(+ "Unable to parse upload date for %s, using `%s` header as file datetime",+ media_item.url,+ hdrs.LAST_MODIFIED,+ )+ media_item.uploaded_at = last_modified+++async def _check_response(media_item: MediaItem, resp: AbstractResponse[Any], resume_point: int) -> None:+ if resp.status == HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE:+ logger.warning("Deleting partial file '%s'. Download is corrupted. Partial file is bigger that expected size")+ await aio.unlink(media_item.partial_file)++ etag.check(resp.headers)+ await check_http_status(resp)++ if not media_item.is_segment and (content_type := _get_content_type(resp.headers)):+ _check_content_type(content_type, media_item.ext)++ _check_for_placeholder_files(resp.headers)++ if resp.status != HTTPStatus.PARTIAL_CONTENT and resume_point:+ logger.warning(+ "Deleting partial file '%s'. Server did not acknowledge byte-range request", media_item.partial_file+ )+ await aio.unlink(media_item.partial_file)diff --git a/cyberdrop_dl/clients/http.py b/cyberdrop_dl/clients/http.pyindex af4469425..45b1f3677 100644--- a/cyberdrop_dl/clients/http.py+++ b/cyberdrop_dl/clients/http.py@@ -174,16 +174,6 @@ class HTTPClient:async for cookie in cookies.read_netscape_files(cookie_files):self.cookies.update_cookies(cookie)- async def check_http_status(self, response: AbstractResponse[Any]) -> None:- if HTTPStatus.OK <= response.status < HTTPStatus.BAD_REQUEST:- # Check DDosGuard even on successful pages- await ddos_guard.check_resp(response)- return-- await _check_json(response)- await ddos_guard.check_resp(response)- raise DownloadError(response.status)-@contextlib.asynccontextmanagerasync def request( # noqa: PLR0913self: object,@@ -211,7 +201,7 @@ class HTTPClient:request_params=request_params,) as resp:try:- await self.check_http_status(resp)+ await check_http_status(resp)except DDOSGuardError:await resp.aclose()if not self.flaresolverr:@@ -361,3 +351,14 @@ def _create_curl_session(config: Config) -> AsyncSession[CurlResponse]:timeout=config.network.curl_timeout,max_redirects=8,)+++async def check_http_status(response: AbstractResponse[Any]) -> None:+ if HTTPStatus.OK <= response.status < HTTPStatus.BAD_REQUEST:+ # Check DDosGuard even on successful pages+ await ddos_guard.check_resp(response)+ return++ await _check_json(response)+ await ddos_guard.check_resp(response)+ raise DownloadError(response.status)