Skip to content

Bulk asynchronous downloader for multiple file hosts

perf: bypass google virus warning for large files (gDrive) (#1262)

NTFSvolume409e754a428f
diff --git a/cyberdrop_dl/crawlers/google_drive.py b/cyberdrop_dl/crawlers/google_drive.py
index d99b72ed5..eeed3f962 100644
--- a/cyberdrop_dl/crawlers/google_drive.py
+++ b/cyberdrop_dl/crawlers/google_drive.py
@@ -32,258 +32,178 @@
from __future__ import annotations
-from dataclasses import dataclass
-from typing import TYPE_CHECKING, ClassVar
-
-from aiolimiter import AsyncLimiter
+import asyncio
+from typing import TYPE_CHECKING, ClassVar, cast
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.exceptions import DownloadError, ScrapeError
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from collections.abc import Mapping
-
- from bs4 import BeautifulSoup
-
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
-VALID_FILE_URL_PARTS = "file", "document", "presentation", "spreadsheets"
-ITEM_SELECTOR = "div.flip-entry-info > a"
+# This versions numbers and restrictions are not documented and may actually be wrong
+# These values are just from personal experience
+_KNOWN_FILE_ID_VERSIONS = 0, 1
+_DRIVE_ID_LEN = 28 # v0 uses 28, v1 uses 33
+_DOCS_ID_LEN = 44 # v1 uses 44. I have not seen v0 doc URL
-@dataclass(frozen=True, slots=True)
-class GoogleDriveFolder:
- id: str
- title: str
- items: tuple[str]
+_PRIMARY_URL = AbsoluteHttpURL("https://drive.google.com")
+_DOCS_URL = AbsoluteHttpURL("https://docs.google.com")
+_FOLDER_ITEM_SELECTOR = "div.flip-entry-info > a[href]"
+_DOC_FORMATS: dict[str, tuple[str, ...]] = {
+ "spreadsheets": ("xslx", "ods", "html", "csv", "tsv"),
+ "presentation": ("pptx", "odp"),
+ "document": ("docx", "odt", "rtf", "txt", "epub", "pdf", "md", "zip"),
+}
-PRIMARY_URL = AbsoluteHttpURL("https://drive.google.com")
+
+def _valid_formats_string() -> str:
+ string = ""
+ sep = "\n - "
+ for doc, formats in sorted(_DOC_FORMATS.items()):
+ default, *others = formats
+ string += f"\n\n{doc}:{sep}"
+ string += sep.join(sorted((f"{default} (default)", *others)))
+ return string
class GoogleDriveCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "Files": "/file/d/<file_id>",
+ "Folders": (
+ "/drive/folders/<folder_id>",
+ "/embeddedfolderview/<folder_id>",
+ ),
"Docs": "/document/d/<file_id>",
- "Files": "/d/<file_id>",
- "Folders": "/drive/folders/<folder_id>",
"Sheets": "/spreadsheets/d/<file_id>",
"Slides": "/presentation/d/<file_id>",
+ "**NOTE**": (
+ "You can download sheets, slides and docs in a custom format by using it as a query param.\n"
+ "ex: https://docs.google.com/document/d/1ZzEzJbemBMPm46O2q5VcGNoPbqDu9AhhUc2djQbvbTY?format=ods\n"
+ f"Valid Formats:{_valid_formats_string()}"
+ ),
}
- SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "drive.google", "docs.google"
- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "drive.google", "docs.google", "drive.usercontent.google.com"
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = _PRIMARY_URL
DOMAIN: ClassVar[str] = "drive.google"
FOLDER_DOMAIN: ClassVar[str] = "GoogleDrive"
- def __post_init__(self) -> None:
- self.request_limiter = AsyncLimiter(4, 6)
-
async def fetch(self, scrape_item: ScrapeItem) -> None:
- if is_folder(scrape_item.url):
- return await self.folder(scrape_item)
-
- if file_id := get_file_id(scrape_item.url):
+ url = scrape_item.url
+ if file_id := url.query.get("id"):
return await self.file(scrape_item, file_id)
- raise ValueError
-
- @error_handling_wrapper
- async def folder(self, scrape_item: ScrapeItem) -> None:
- folder = await self.get_folder_details(scrape_item)
- title = self.create_title(folder.title, folder.id)
- scrape_item.setup_as_album(title, album_id=folder.id)
- results = await self.get_album_results(folder.id)
-
- subfolders = []
- for link_str in folder.items:
- link = self.parse_url(link_str)
- if is_folder(link):
- subfolders.append(link)
- continue
- if not self.check_album_results(link, results):
- new_scrape_item = scrape_item.create_child(link)
- self.manager.task_group.create_task(self.run(new_scrape_item))
- scrape_item.add_children()
-
- for folder_link in subfolders:
- new_scrape_item = scrape_item.create_child(folder_link)
- self.manager.task_group.create_task(self.run(new_scrape_item))
- scrape_item.add_children()
-
- async def get_folder_details(self, scrape_item: ScrapeItem) -> GoogleDriveFolder:
- folder_id = get_folder_id(scrape_item.url)
- download_url = get_download_url(folder_id, folder=True)
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, download_url)
-
- title: str = css.select_one_get_text(soup, "title")
- children = []
- for item in soup.select(ITEM_SELECTOR):
- link = item.get("href")
- if not link:
- continue
- children.append(link)
- children = tuple(children)
- return GoogleDriveFolder(folder_id, title, children)
-
- @error_handling_wrapper
- async def file(self, scrape_item: ScrapeItem, file_id: str = "") -> None:
- file_id = file_id or get_file_id(scrape_item.url)
- if not file_id:
- raise ScrapeError(422)
-
- canonical_url = get_canonical_url(file_id)
- if await self.check_complete_from_referer(canonical_url):
- return
-
- scrape_item.url = canonical_url
- download_url = get_download_url(file_id)
- link, filename = await self.get_file_url_and_name(download_url, file_id)
- filename, ext = self.get_filename_and_ext(filename or link.name)
- await self.handle_file(canonical_url, scrape_item, filename, ext, debrid_link=link)
-
- async def get_file_url_and_name(self, url: AbsoluteHttpURL, file_id: str) -> tuple[AbsoluteHttpURL, str | None]:
- soup = last_error = response = None
- current_url: AbsoluteHttpURL | None = url
- try_file_open_url = True
-
- while current_url:
- current_url, filename = await self.add_filename(current_url)
- if filename:
- return current_url, filename
-
+ def next_to(name: str):
try:
- async with self.request_limiter:
- response, soup = await self.client._get_response_and_soup(self.DOMAIN, current_url)
-
- except DownloadError as e:
- last_error = e
- if e.status == 500 and try_file_open_url:
- current_url = AbsoluteHttpURL(f"https://drive.google.com/open?id={file_id}")
- try_file_open_url = False
- continue
-
- if not soup:
- raise last_error or ScrapeError(400)
+ index = url.parts.index(name)
+ return url.parts[index + 1]
+ except (ValueError, IndexError):
+ return
- if docs_url := get_docs_url(soup, file_id):
- current_url = docs_url
- continue
+ if folder_id := (next_to("folders") or next_to("embeddedfolderview")):
+ return await self.folder(scrape_item, folder_id)
- if response and response.content_disposition and response.content_disposition.filename:
- return current_url, response.content_disposition.filename
+ if file_id := next_to("d"):
+ if (first := url.parts[1]) in _DOC_FORMATS:
+ doc = first
+ else:
+ doc = None
+ return await self.file(scrape_item, file_id, doc)
- current_url = self.get_url_from_download_button(soup)
- if not current_url:
- break
-
- return await self.add_filename(current_url)
-
- raise ScrapeError(422)
+ raise ValueError
- async def add_filename(self, url: AbsoluteHttpURL) -> tuple[AbsoluteHttpURL, str | None]:
+ @error_handling_wrapper
+ async def folder(self, scrape_item: ScrapeItem, folder_id: str) -> None:
+ embeded_folder_url = (self.PRIMARY_URL / "embeddedfolderview").with_query(id=folder_id)
async with self.request_limiter:
- response = await self.client._get_head(self.DOMAIN, url)
- location = response.headers.get("location")
- if location:
- link = self.parse_url(location)
- return await self.add_filename(link)
- if response.content_disposition and response.content_disposition.filename:
- return url, response.content_disposition.filename
- return url, None
-
- def get_url_from_download_button(self, soup: BeautifulSoup) -> AbsoluteHttpURL | None:
- form = soup.select_one("#download-form")
- if not form:
- return None
-
- url_str: str = css.get_attr(form, "action")
- url: AbsoluteHttpURL = self.parse_url(url_str.replace("&amp;", "&"))
- query_params = dict(url.query)
-
- input_tags = soup.select('input[type="hidden"]')
- for input_tag in input_tags:
- query_params[css.get_attr(input_tag, "name")] = css.get_attr(input_tag, "value")
-
- return url.with_query(query_params)
-
-
-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-
-def get_docs_url(soup: BeautifulSoup, file_id: str) -> AbsoluteHttpURL | None:
- title: str = css.select_one_get_text(soup, "title")
- if title.endswith(" - Google Docs"):
- return AbsoluteHttpURL(f"https://docs.google.com/document/d/{file_id}/export?format=docx")
- if title.endswith(" - Google Sheets"):
- return AbsoluteHttpURL(f"https://docs.google.com/spreadsheets/d/{file_id}/export?format=xlsx")
- if title.endswith(" - Google Slides"):
- return AbsoluteHttpURL(f"https://docs.google.com/presentation/d/{file_id}/export?format=pptx")
+ soup = await self.client.get_soup(self.DOMAIN, embeded_folder_url)
+ folder_name = css.select_one_get_text(soup, "title")
+ title = self.create_title(folder_name, folder_id)
+ scrape_item.setup_as_album(title, album_id=folder_id)
-def get_id_from_query(url: AbsoluteHttpURL) -> str | None:
- if item_id := url.query.get("id"):
- if len(item_id) == 1:
- return item_id[1]
- return item_id
-
-
-def get_file_id(url: AbsoluteHttpURL) -> str:
- if file_id := get_id_from_query(url):
- return file_id
-
- if "d" in url.parts and any(p in url.parts for p in VALID_FILE_URL_PARTS):
- file_id_index = url.parts.index("d") + 1
- if len(url.parts) > file_id_index:
- return url.parts[file_id_index]
- return ""
-
-
-def get_folder_id(url: AbsoluteHttpURL) -> str:
- # URL should have been pre-validated with `is_folder`
- folder_id = get_id_from_query(url)
- if folder_id:
- return folder_id
-
- folder_id_index = url.parts.index("folders") + 1
- if len(url.parts) > folder_id_index:
- return url.parts[folder_id_index]
- return ""
-
-
-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ for index, (_, child) in enumerate(self.iter_tags(soup, _FOLDER_ITEM_SELECTOR), 1):
+ new_scrape_item = scrape_item.create_child(child)
+ self.create_task(self.run(new_scrape_item))
+ scrape_item.add_children()
+ if index % 200 == 0:
+ await asyncio.sleep(0)
+ async def file(self, scrape_item: ScrapeItem, file_id: str = "", doc: str | None = None) -> None:
+ version = int(file_id[0])
+ if version not in _KNOWN_FILE_ID_VERSIONS:
+ msg = f"{scrape_item.url} has an unknown file_id {version = }. Falling back to download as normal file"
+ self.log(msg, 30)
+ return await self._drive_file(scrape_item, file_id)
-def get_canonical_url(item_id: str, folder: bool = False) -> AbsoluteHttpURL:
- if folder:
- return AbsoluteHttpURL(f"https://drive.google.com/drive/folders/{item_id}")
- return AbsoluteHttpURL(f"https://drive.google.com/file/d/{item_id}")
+ if len(file_id) < _DRIVE_ID_LEN:
+ msg = f"{scrape_item.url} has an invalid file_id. Needs to be at least {_DRIVE_ID_LEN} long"
+ self.log(msg, 40)
+ raise ValueError
+ if len(file_id) < _DOCS_ID_LEN:
+ return await self._drive_file(scrape_item, file_id)
-def get_download_url(item_id: str, folder: bool = False) -> AbsoluteHttpURL:
- if folder:
- return AbsoluteHttpURL(f"https://drive.google.com/embeddedfolderview?id={item_id}")
- return AbsoluteHttpURL(f"https://drive.google.com/uc?export=download&id={item_id}")
+ return await self._docs_file(scrape_item, file_id, doc)
+ async def _drive_file(self, scrape_item: ScrapeItem, file_id: str) -> None:
+ scrape_item.url = _PRIMARY_URL / "file/d" / file_id
+ export_url = (_PRIMARY_URL / "uc").with_query(id=file_id, export="download", confirm="True")
+ return await self._file(scrape_item, export_url)
-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ @error_handling_wrapper
+ async def _docs_file(self, scrape_item: ScrapeItem, file_id: str, doc: str | None) -> None:
+ if not doc:
+ open_url = (_DOCS_URL / "open").with_query(id=file_id)
+ url = await self._get_redirect_url(open_url)
+ if (first := url.parts[1]) in _DOC_FORMATS:
+ doc = first
+
+ if not doc:
+ raise ScrapeError(422, "Unable to identify google docs file type")
+
+ format_ = scrape_item.url.query.get("format")
+ proper_format = _get_proper_doc_format(doc, format_)
+ if format_ and format_ != proper_format:
+ msg = f"{scrape_item.url} with {format_ = } is not valid. Falling back to {proper_format}"
+ self.log(msg, 30)
+
+ scrape_item.url = (_DOCS_URL / doc / "d" / file_id).with_query(format=proper_format)
+ export_url = (_DOCS_URL / doc / "export").with_query(id=file_id, format=proper_format)
+ return await self._file(scrape_item, export_url)
+ @error_handling_wrapper
+ async def _file(self, scrape_item: ScrapeItem, export_url: AbsoluteHttpURL) -> None:
+ if await self.check_complete_from_referer(scrape_item):
+ return
-def are_valid_headers(headers: Mapping[str, str]):
- return "Content-Disposition" in headers and not is_html(headers)
+ link, name = await self._get_file_info(export_url)
+ filename, ext = self.get_filename_and_ext(name)
+ await self.handle_file(scrape_item.url, scrape_item, name, ext, debrid_link=link, custom_filename=filename)
+ async def _get_file_info(self, export_url: AbsoluteHttpURL) -> tuple[AbsoluteHttpURL, str]:
+ # Use POST request to bypass "file is too large to scan. Would you still like to download this file" warning
+ method = "GET" if export_url.host == _DOCS_URL.host else "POST"
-def is_html(headers: Mapping[str, str]) -> bool:
- content_type: str = headers.get("Content-Type", "").lower()
- return any(s in content_type for s in ("html", "text"))
+ # TODO: This request bypasses the config limiter. Use the new request method when PR #1251 is merged
+ async with self.request_limiter, self.client._session.request(method, export_url) as resp:
+ if not resp.ok or "html" in resp.content_type:
+ await self.client.client_manager.check_http_status(resp)
+ direct_url = cast("AbsoluteHttpURL", resp.url)
+ filename: str = resp.content_disposition.filename # type: ignore[reportAssignmentType,reportOptionalMemberAccess]
+ return direct_url, filename
-def is_folder(url: AbsoluteHttpURL) -> bool:
- return "/drive/folders/" in url.path or "embeddedfolderview" in url.parts
+def _get_proper_doc_format(doc: str, format: str | None) -> str:
+ valid_formats = _DOC_FORMATS[doc]
+ if format in valid_formats:
+ return format
-def is_download_page(url: AbsoluteHttpURL) -> bool:
- return url.name == "uc" or "usercontent" in url.host
+ return valid_formats[0]
diff --git a/tests/crawlers/test_cases/google_drive.py b/tests/crawlers/test_cases/google_drive.py
new file mode 100644
index 000000000..caafc4234
--- /dev/null
+++ b/tests/crawlers/test_cases/google_drive.py
@@ -0,0 +1,137 @@
+DOMAIN = "drive.google"
+TEST_CASES = [
+ (
+ # small file with no warning
+ "https://drive.google.com/file/d/1F0YBsnQRvrMbK0p9UlnyLu88kqQ0j_F6/edit",
+ [
+ {
+ "url": "ANY",
+ "filename": "file-50MB.dat",
+ "referer": "https://drive.google.com/file/d/1F0YBsnQRvrMbK0p9UlnyLu88kqQ0j_F6",
+ "album_id": None,
+ "datetime": None,
+ }
+ ],
+ ),
+ (
+ # small file with no warning
+ "https://drive.google.com/file/d/15WghIO0iwekXStmVWeK5HxC566iN41l6/view",
+ [
+ {
+ "url": "ANY",
+ "filename": "file-100MB.dat",
+ "referer": "https://drive.google.com/file/d/15WghIO0iwekXStmVWeK5HxC566iN41l6",
+ "album_id": None,
+ "datetime": None,
+ }
+ ],
+ ),
+ (
+ # file with warning do to large size (529M)
+ "https://drive.usercontent.google.com/download?id=1fXgBupLzThTGLLsiYCHRQJixuDsR1bSI&export=download",
+ [
+ {
+ "url": "ANY",
+ "filename": "cifar10_stats.npz",
+ "referer": "https://drive.google.com/file/d/1fXgBupLzThTGLLsiYCHRQJixuDsR1bSI",
+ "album_id": None,
+ "datetime": None,
+ }
+ ],
+ ),
+ (
+ # Huge file with warning do to large size (9.8G), this test may fail.
+ # Public download wihtout an account are limited to about 5G per day and they return 429 with that happends
+ "https://drive.google.com/file/d/1WHv5Dm1GtrDZj-AxJZd3T-NMIBXty3eV/view",
+ [
+ {
+ "url": "ANY",
+ "filename": "bundle_cutouts_africa.zip",
+ "referer": "https://drive.google.com/file/d/1WHv5Dm1GtrDZj-AxJZd3T-NMIBXty3eV",
+ "album_id": None,
+ "datetime": None,
+ }
+ ],
+ ),
+ (
+ "https://docs.google.com/document/d/1ZzEzJbemBMPm46O2q5VcGNoPbqDu9AhhUc2djQbvbTY/edit",
+ [
+ {
+ "url": "ANY",
+ "filename": "test.docx",
+ "referer": "https://docs.google.com/document/d/1ZzEzJbemBMPm46O2q5VcGNoPbqDu9AhhUc2djQbvbTY?format=docx",
+ "album_id": None,
+ "datetime": None,
+ }
+ ],
+ ),
+ (
+ "https://docs.google.com/document/d/1ZzEzJbemBMPm46O2q5VcGNoPbqDu9AhhUc2djQbvbTY?format=txt",
+ [
+ {
+ "url": "ANY",
+ "filename": "test.txt",
+ "referer": "https://docs.google.com/document/d/1ZzEzJbemBMPm46O2q5VcGNoPbqDu9AhhUc2djQbvbTY?format=txt",
+ "album_id": None,
+ "datetime": None,
+ }
+ ],
+ ),
+ (
+ "https://docs.google.com/document/d/1ZzEzJbemBMPm46O2q5VcGNoPbqDu9AhhUc2djQbvbTY?format=xlsx",
+ [
+ {
+ "url": "ANY",
+ "filename": "test.docx",
+ "referer": "https://docs.google.com/document/d/1ZzEzJbemBMPm46O2q5VcGNoPbqDu9AhhUc2djQbvbTY?format=docx",
+ "album_id": None,
+ "datetime": None,
+ }
+ ],
+ ),
+ (
+ # This is a spreeadsheet but the id is a normal file id
+ # We will not be able to download it with a custom format
+ "https://docs.google.com/spreadsheets/d/1E3LpudUdUZycJpxSKK-c9-oIDuJoo5_7/edit?format=ods",
+ [
+ {
+ "url": "ANY",
+ "filename": "TK Online 1.5.xlsx",
+ "referer": "https://drive.google.com/file/d/1E3LpudUdUZycJpxSKK-c9-oIDuJoo5_7",
+ "album_id": None,
+ "datetime": None,
+ }
+ ],
+ ),
+ (
+ # v0 file id (28 chars)
+ "https://drive.google.com/file/d/0ByeS4oOUV-49Zzh4R1J6R09zazQ/edit",
+ [
+ {
+ "url": "ANY",
+ "filename": "Big Buck Bunny.mp4",
+ "referer": "https://drive.google.com/file/d/0ByeS4oOUV-49Zzh4R1J6R09zazQ",
+ "album_id": None,
+ "datetime": None,
+ }
+ ],
+ ),
+ (
+ "https://drive.google.com/uc?id=1IP0o8dHcQrIHGgVyp0Ofvx2cGfLzyO1x",
+ [
+ {
+ "url": "ANY",
+ "filename": "My Buddy - Henry Burr - Gus Kahn - Walter Donaldson.mp3",
+ "referer": "https://drive.google.com/file/d/1IP0o8dHcQrIHGgVyp0Ofvx2cGfLzyO1x",
+ "album_id": None,
+ "datetime": None,
+ }
+ ],
+ ),
+ (
+ # Folder with +50 files
+ "https://drive.google.com/drive/folders/1k8pgIaGw6PribxVqMgmDtlpzbUPJuzrX",
+ [],
+ 135,
+ ),
+]