Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
perf: bypass google virus warning for large files (gDrive) (#1262)
NTFSvolume409e754a428f
diff --git a/cyberdrop_dl/crawlers/google_drive.py b/cyberdrop_dl/crawlers/google_drive.pyindex d99b72ed5..eeed3f962 100644--- a/cyberdrop_dl/crawlers/google_drive.py+++ b/cyberdrop_dl/crawlers/google_drive.py@@ -32,258 +32,178 @@from __future__ import annotations-from dataclasses import dataclass-from typing import TYPE_CHECKING, ClassVar--from aiolimiter import AsyncLimiter+import asyncio+from typing import TYPE_CHECKING, ClassVar, castfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from cyberdrop_dl.exceptions import DownloadError, ScrapeError+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from collections.abc import Mapping-- from bs4 import BeautifulSoup-from cyberdrop_dl.data_structures.url_objects import ScrapeItem-VALID_FILE_URL_PARTS = "file", "document", "presentation", "spreadsheets"-ITEM_SELECTOR = "div.flip-entry-info > a"+# This versions numbers and restrictions are not documented and may actually be wrong+# These values are just from personal experience+_KNOWN_FILE_ID_VERSIONS = 0, 1+_DRIVE_ID_LEN = 28 # v0 uses 28, v1 uses 33+_DOCS_ID_LEN = 44 # v1 uses 44. I have not seen v0 doc URL-@dataclass(frozen=True, slots=True)-class GoogleDriveFolder:- id: str- title: str- items: tuple[str]+_PRIMARY_URL = AbsoluteHttpURL("https://drive.google.com")+_DOCS_URL = AbsoluteHttpURL("https://docs.google.com")+_FOLDER_ITEM_SELECTOR = "div.flip-entry-info > a[href]"+_DOC_FORMATS: dict[str, tuple[str, ...]] = {+ "spreadsheets": ("xslx", "ods", "html", "csv", "tsv"),+ "presentation": ("pptx", "odp"),+ "document": ("docx", "odt", "rtf", "txt", "epub", "pdf", "md", "zip"),+}-PRIMARY_URL = AbsoluteHttpURL("https://drive.google.com")++def _valid_formats_string() -> str:+ string = ""+ sep = "\n - "+ for doc, formats in sorted(_DOC_FORMATS.items()):+ default, *others = formats+ string += f"\n\n{doc}:{sep}"+ string += sep.join(sorted((f"{default} (default)", *others)))+ return stringclass GoogleDriveCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Files": "/file/d/<file_id>",+ "Folders": (+ "/drive/folders/<folder_id>",+ "/embeddedfolderview/<folder_id>",+ ),"Docs": "/document/d/<file_id>",- "Files": "/d/<file_id>",- "Folders": "/drive/folders/<folder_id>","Sheets": "/spreadsheets/d/<file_id>","Slides": "/presentation/d/<file_id>",+ "**NOTE**": (+ "You can download sheets, slides and docs in a custom format by using it as a query param.\n"+ "ex: https://docs.google.com/document/d/1ZzEzJbemBMPm46O2q5VcGNoPbqDu9AhhUc2djQbvbTY?format=ods\n"+ f"Valid Formats:{_valid_formats_string()}"+ ),}- SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "drive.google", "docs.google"- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "drive.google", "docs.google", "drive.usercontent.google.com"+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = _PRIMARY_URLDOMAIN: ClassVar[str] = "drive.google"FOLDER_DOMAIN: ClassVar[str] = "GoogleDrive"- def __post_init__(self) -> None:- self.request_limiter = AsyncLimiter(4, 6)-async def fetch(self, scrape_item: ScrapeItem) -> None:- if is_folder(scrape_item.url):- return await self.folder(scrape_item)-- if file_id := get_file_id(scrape_item.url):+ url = scrape_item.url+ if file_id := url.query.get("id"):return await self.file(scrape_item, file_id)- raise ValueError-- @error_handling_wrapper- async def folder(self, scrape_item: ScrapeItem) -> None:- folder = await self.get_folder_details(scrape_item)- title = self.create_title(folder.title, folder.id)- scrape_item.setup_as_album(title, album_id=folder.id)- results = await self.get_album_results(folder.id)-- subfolders = []- for link_str in folder.items:- link = self.parse_url(link_str)- if is_folder(link):- subfolders.append(link)- continue- if not self.check_album_results(link, results):- new_scrape_item = scrape_item.create_child(link)- self.manager.task_group.create_task(self.run(new_scrape_item))- scrape_item.add_children()-- for folder_link in subfolders:- new_scrape_item = scrape_item.create_child(folder_link)- self.manager.task_group.create_task(self.run(new_scrape_item))- scrape_item.add_children()-- async def get_folder_details(self, scrape_item: ScrapeItem) -> GoogleDriveFolder:- folder_id = get_folder_id(scrape_item.url)- download_url = get_download_url(folder_id, folder=True)- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, download_url)-- title: str = css.select_one_get_text(soup, "title")- children = []- for item in soup.select(ITEM_SELECTOR):- link = item.get("href")- if not link:- continue- children.append(link)- children = tuple(children)- return GoogleDriveFolder(folder_id, title, children)-- @error_handling_wrapper- async def file(self, scrape_item: ScrapeItem, file_id: str = "") -> None:- file_id = file_id or get_file_id(scrape_item.url)- if not file_id:- raise ScrapeError(422)-- canonical_url = get_canonical_url(file_id)- if await self.check_complete_from_referer(canonical_url):- return-- scrape_item.url = canonical_url- download_url = get_download_url(file_id)- link, filename = await self.get_file_url_and_name(download_url, file_id)- filename, ext = self.get_filename_and_ext(filename or link.name)- await self.handle_file(canonical_url, scrape_item, filename, ext, debrid_link=link)-- async def get_file_url_and_name(self, url: AbsoluteHttpURL, file_id: str) -> tuple[AbsoluteHttpURL, str | None]:- soup = last_error = response = None- current_url: AbsoluteHttpURL | None = url- try_file_open_url = True-- while current_url:- current_url, filename = await self.add_filename(current_url)- if filename:- return current_url, filename-+ def next_to(name: str):try:- async with self.request_limiter:- response, soup = await self.client._get_response_and_soup(self.DOMAIN, current_url)-- except DownloadError as e:- last_error = e- if e.status == 500 and try_file_open_url:- current_url = AbsoluteHttpURL(f"https://drive.google.com/open?id={file_id}")- try_file_open_url = False- continue-- if not soup:- raise last_error or ScrapeError(400)+ index = url.parts.index(name)+ return url.parts[index + 1]+ except (ValueError, IndexError):+ return- if docs_url := get_docs_url(soup, file_id):- current_url = docs_url- continue+ if folder_id := (next_to("folders") or next_to("embeddedfolderview")):+ return await self.folder(scrape_item, folder_id)- if response and response.content_disposition and response.content_disposition.filename:- return current_url, response.content_disposition.filename+ if file_id := next_to("d"):+ if (first := url.parts[1]) in _DOC_FORMATS:+ doc = first+ else:+ doc = None+ return await self.file(scrape_item, file_id, doc)- current_url = self.get_url_from_download_button(soup)- if not current_url:- break-- return await self.add_filename(current_url)-- raise ScrapeError(422)+ raise ValueError- async def add_filename(self, url: AbsoluteHttpURL) -> tuple[AbsoluteHttpURL, str | None]:+ @error_handling_wrapper+ async def folder(self, scrape_item: ScrapeItem, folder_id: str) -> None:+ embeded_folder_url = (self.PRIMARY_URL / "embeddedfolderview").with_query(id=folder_id)async with self.request_limiter:- response = await self.client._get_head(self.DOMAIN, url)- location = response.headers.get("location")- if location:- link = self.parse_url(location)- return await self.add_filename(link)- if response.content_disposition and response.content_disposition.filename:- return url, response.content_disposition.filename- return url, None-- def get_url_from_download_button(self, soup: BeautifulSoup) -> AbsoluteHttpURL | None:- form = soup.select_one("#download-form")- if not form:- return None-- url_str: str = css.get_attr(form, "action")- url: AbsoluteHttpURL = self.parse_url(url_str.replace("&", "&"))- query_params = dict(url.query)-- input_tags = soup.select('input[type="hidden"]')- for input_tag in input_tags:- query_params[css.get_attr(input_tag, "name")] = css.get_attr(input_tag, "value")-- return url.with_query(query_params)---# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~---def get_docs_url(soup: BeautifulSoup, file_id: str) -> AbsoluteHttpURL | None:- title: str = css.select_one_get_text(soup, "title")- if title.endswith(" - Google Docs"):- return AbsoluteHttpURL(f"https://docs.google.com/document/d/{file_id}/export?format=docx")- if title.endswith(" - Google Sheets"):- return AbsoluteHttpURL(f"https://docs.google.com/spreadsheets/d/{file_id}/export?format=xlsx")- if title.endswith(" - Google Slides"):- return AbsoluteHttpURL(f"https://docs.google.com/presentation/d/{file_id}/export?format=pptx")+ soup = await self.client.get_soup(self.DOMAIN, embeded_folder_url)+ folder_name = css.select_one_get_text(soup, "title")+ title = self.create_title(folder_name, folder_id)+ scrape_item.setup_as_album(title, album_id=folder_id)-def get_id_from_query(url: AbsoluteHttpURL) -> str | None:- if item_id := url.query.get("id"):- if len(item_id) == 1:- return item_id[1]- return item_id---def get_file_id(url: AbsoluteHttpURL) -> str:- if file_id := get_id_from_query(url):- return file_id-- if "d" in url.parts and any(p in url.parts for p in VALID_FILE_URL_PARTS):- file_id_index = url.parts.index("d") + 1- if len(url.parts) > file_id_index:- return url.parts[file_id_index]- return ""---def get_folder_id(url: AbsoluteHttpURL) -> str:- # URL should have been pre-validated with `is_folder`- folder_id = get_id_from_query(url)- if folder_id:- return folder_id-- folder_id_index = url.parts.index("folders") + 1- if len(url.parts) > folder_id_index:- return url.parts[folder_id_index]- return ""---# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~+ for index, (_, child) in enumerate(self.iter_tags(soup, _FOLDER_ITEM_SELECTOR), 1):+ new_scrape_item = scrape_item.create_child(child)+ self.create_task(self.run(new_scrape_item))+ scrape_item.add_children()+ if index % 200 == 0:+ await asyncio.sleep(0)+ async def file(self, scrape_item: ScrapeItem, file_id: str = "", doc: str | None = None) -> None:+ version = int(file_id[0])+ if version not in _KNOWN_FILE_ID_VERSIONS:+ msg = f"{scrape_item.url} has an unknown file_id {version = }. Falling back to download as normal file"+ self.log(msg, 30)+ return await self._drive_file(scrape_item, file_id)-def get_canonical_url(item_id: str, folder: bool = False) -> AbsoluteHttpURL:- if folder:- return AbsoluteHttpURL(f"https://drive.google.com/drive/folders/{item_id}")- return AbsoluteHttpURL(f"https://drive.google.com/file/d/{item_id}")+ if len(file_id) < _DRIVE_ID_LEN:+ msg = f"{scrape_item.url} has an invalid file_id. Needs to be at least {_DRIVE_ID_LEN} long"+ self.log(msg, 40)+ raise ValueError+ if len(file_id) < _DOCS_ID_LEN:+ return await self._drive_file(scrape_item, file_id)-def get_download_url(item_id: str, folder: bool = False) -> AbsoluteHttpURL:- if folder:- return AbsoluteHttpURL(f"https://drive.google.com/embeddedfolderview?id={item_id}")- return AbsoluteHttpURL(f"https://drive.google.com/uc?export=download&id={item_id}")+ return await self._docs_file(scrape_item, file_id, doc)+ async def _drive_file(self, scrape_item: ScrapeItem, file_id: str) -> None:+ scrape_item.url = _PRIMARY_URL / "file/d" / file_id+ export_url = (_PRIMARY_URL / "uc").with_query(id=file_id, export="download", confirm="True")+ return await self._file(scrape_item, export_url)-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~+ @error_handling_wrapper+ async def _docs_file(self, scrape_item: ScrapeItem, file_id: str, doc: str | None) -> None:+ if not doc:+ open_url = (_DOCS_URL / "open").with_query(id=file_id)+ url = await self._get_redirect_url(open_url)+ if (first := url.parts[1]) in _DOC_FORMATS:+ doc = first++ if not doc:+ raise ScrapeError(422, "Unable to identify google docs file type")++ format_ = scrape_item.url.query.get("format")+ proper_format = _get_proper_doc_format(doc, format_)+ if format_ and format_ != proper_format:+ msg = f"{scrape_item.url} with {format_ = } is not valid. Falling back to {proper_format}"+ self.log(msg, 30)++ scrape_item.url = (_DOCS_URL / doc / "d" / file_id).with_query(format=proper_format)+ export_url = (_DOCS_URL / doc / "export").with_query(id=file_id, format=proper_format)+ return await self._file(scrape_item, export_url)+ @error_handling_wrapper+ async def _file(self, scrape_item: ScrapeItem, export_url: AbsoluteHttpURL) -> None:+ if await self.check_complete_from_referer(scrape_item):+ return-def are_valid_headers(headers: Mapping[str, str]):- return "Content-Disposition" in headers and not is_html(headers)+ link, name = await self._get_file_info(export_url)+ filename, ext = self.get_filename_and_ext(name)+ await self.handle_file(scrape_item.url, scrape_item, name, ext, debrid_link=link, custom_filename=filename)+ async def _get_file_info(self, export_url: AbsoluteHttpURL) -> tuple[AbsoluteHttpURL, str]:+ # Use POST request to bypass "file is too large to scan. Would you still like to download this file" warning+ method = "GET" if export_url.host == _DOCS_URL.host else "POST"-def is_html(headers: Mapping[str, str]) -> bool:- content_type: str = headers.get("Content-Type", "").lower()- return any(s in content_type for s in ("html", "text"))+ # TODO: This request bypasses the config limiter. Use the new request method when PR #1251 is merged+ async with self.request_limiter, self.client._session.request(method, export_url) as resp:+ if not resp.ok or "html" in resp.content_type:+ await self.client.client_manager.check_http_status(resp)+ direct_url = cast("AbsoluteHttpURL", resp.url)+ filename: str = resp.content_disposition.filename # type: ignore[reportAssignmentType,reportOptionalMemberAccess]+ return direct_url, filename-def is_folder(url: AbsoluteHttpURL) -> bool:- return "/drive/folders/" in url.path or "embeddedfolderview" in url.parts+def _get_proper_doc_format(doc: str, format: str | None) -> str:+ valid_formats = _DOC_FORMATS[doc]+ if format in valid_formats:+ return format-def is_download_page(url: AbsoluteHttpURL) -> bool:- return url.name == "uc" or "usercontent" in url.host+ return valid_formats[0]diff --git a/tests/crawlers/test_cases/google_drive.py b/tests/crawlers/test_cases/google_drive.pynew file mode 100644index 000000000..caafc4234--- /dev/null+++ b/tests/crawlers/test_cases/google_drive.py@@ -0,0 +1,137 @@+DOMAIN = "drive.google"+TEST_CASES = [+ (+ # small file with no warning+ "https://drive.google.com/file/d/1F0YBsnQRvrMbK0p9UlnyLu88kqQ0j_F6/edit",+ [+ {+ "url": "ANY",+ "filename": "file-50MB.dat",+ "referer": "https://drive.google.com/file/d/1F0YBsnQRvrMbK0p9UlnyLu88kqQ0j_F6",+ "album_id": None,+ "datetime": None,+ }+ ],+ ),+ (+ # small file with no warning+ "https://drive.google.com/file/d/15WghIO0iwekXStmVWeK5HxC566iN41l6/view",+ [+ {+ "url": "ANY",+ "filename": "file-100MB.dat",+ "referer": "https://drive.google.com/file/d/15WghIO0iwekXStmVWeK5HxC566iN41l6",+ "album_id": None,+ "datetime": None,+ }+ ],+ ),+ (+ # file with warning do to large size (529M)+ "https://drive.usercontent.google.com/download?id=1fXgBupLzThTGLLsiYCHRQJixuDsR1bSI&export=download",+ [+ {+ "url": "ANY",+ "filename": "cifar10_stats.npz",+ "referer": "https://drive.google.com/file/d/1fXgBupLzThTGLLsiYCHRQJixuDsR1bSI",+ "album_id": None,+ "datetime": None,+ }+ ],+ ),+ (+ # Huge file with warning do to large size (9.8G), this test may fail.+ # Public download wihtout an account are limited to about 5G per day and they return 429 with that happends+ "https://drive.google.com/file/d/1WHv5Dm1GtrDZj-AxJZd3T-NMIBXty3eV/view",+ [+ {+ "url": "ANY",+ "filename": "bundle_cutouts_africa.zip",+ "referer": "https://drive.google.com/file/d/1WHv5Dm1GtrDZj-AxJZd3T-NMIBXty3eV",+ "album_id": None,+ "datetime": None,+ }+ ],+ ),+ (+ "https://docs.google.com/document/d/1ZzEzJbemBMPm46O2q5VcGNoPbqDu9AhhUc2djQbvbTY/edit",+ [+ {+ "url": "ANY",+ "filename": "test.docx",+ "referer": "https://docs.google.com/document/d/1ZzEzJbemBMPm46O2q5VcGNoPbqDu9AhhUc2djQbvbTY?format=docx",+ "album_id": None,+ "datetime": None,+ }+ ],+ ),+ (+ "https://docs.google.com/document/d/1ZzEzJbemBMPm46O2q5VcGNoPbqDu9AhhUc2djQbvbTY?format=txt",+ [+ {+ "url": "ANY",+ "filename": "test.txt",+ "referer": "https://docs.google.com/document/d/1ZzEzJbemBMPm46O2q5VcGNoPbqDu9AhhUc2djQbvbTY?format=txt",+ "album_id": None,+ "datetime": None,+ }+ ],+ ),+ (+ "https://docs.google.com/document/d/1ZzEzJbemBMPm46O2q5VcGNoPbqDu9AhhUc2djQbvbTY?format=xlsx",+ [+ {+ "url": "ANY",+ "filename": "test.docx",+ "referer": "https://docs.google.com/document/d/1ZzEzJbemBMPm46O2q5VcGNoPbqDu9AhhUc2djQbvbTY?format=docx",+ "album_id": None,+ "datetime": None,+ }+ ],+ ),+ (+ # This is a spreeadsheet but the id is a normal file id+ # We will not be able to download it with a custom format+ "https://docs.google.com/spreadsheets/d/1E3LpudUdUZycJpxSKK-c9-oIDuJoo5_7/edit?format=ods",+ [+ {+ "url": "ANY",+ "filename": "TK Online 1.5.xlsx",+ "referer": "https://drive.google.com/file/d/1E3LpudUdUZycJpxSKK-c9-oIDuJoo5_7",+ "album_id": None,+ "datetime": None,+ }+ ],+ ),+ (+ # v0 file id (28 chars)+ "https://drive.google.com/file/d/0ByeS4oOUV-49Zzh4R1J6R09zazQ/edit",+ [+ {+ "url": "ANY",+ "filename": "Big Buck Bunny.mp4",+ "referer": "https://drive.google.com/file/d/0ByeS4oOUV-49Zzh4R1J6R09zazQ",+ "album_id": None,+ "datetime": None,+ }+ ],+ ),+ (+ "https://drive.google.com/uc?id=1IP0o8dHcQrIHGgVyp0Ofvx2cGfLzyO1x",+ [+ {+ "url": "ANY",+ "filename": "My Buddy - Henry Burr - Gus Kahn - Walter Donaldson.mp3",+ "referer": "https://drive.google.com/file/d/1IP0o8dHcQrIHGgVyp0Ofvx2cGfLzyO1x",+ "album_id": None,+ "datetime": None,+ }+ ],+ ),+ (+ # Folder with +50 files+ "https://drive.google.com/drive/folders/1k8pgIaGw6PribxVqMgmDtlpzbUPJuzrX",+ [],+ 135,+ ),+]