Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
from pathlib import Path
from cyberdrop_dl import __version__
from cyberdrop_dl.commands import supported_sites
from cyberdrop_dl.constants import CDL_USER_AGENT
REPO_ROOT = Path(__file__).parents[2]
CLI_ARGUMENTS_MD = REPO_ROOT / "docs/reference/cli-arguments.md"
SUPPORTED_SITES_MD = REPO_ROOT / "docs/reference/supported-websites.md"
NETWORK_MD = REPO_ROOT / "docs/reference/config/network.md"
def _write_if_updated(path: Path, old_content: str, new_content: str) -> None:
relative = path.relative_to(REPO_ROOT)
if old_content != new_content:
path.write_text(new_content, encoding="utf8")
msg = f"Updated:'{relative}'"
else:
msg = f"Did not change: '{relative}'"
print(msg)
def _make_supported_sites_markdown_table() -> str:
return "".join(
(
"\n",
"## Supported sites",
"\n\n",
f"List of sites supported by cyberdrop-dl-patched as of version {__version__}",
"\n\n",
supported_sites.as_markdown(),
"\n",
)
)
def _replace_content(text: str, marker: str, new_content: str) -> str:
start_marker = f"<!-- START_{marker} -->"
start = text.index(start_marker) + len(start_marker)
end = text.index(f"<!-- END_{marker} -->")
return text[:start] + "\n" + new_content + "\n" + text[end:]
def _get_help_message() -> str:
from rich.console import Console
from cyberdrop_dl.cli import app
with Console(record=True, width=100) as console:
app.help_print([], console=console)
console.line()
console.rule(style="blue")
console.line()
app.help_print(["download"], console=console)
return console.export_text()
def _update_file(file: Path, new_content: str, *, marker: str) -> None:
new_content = _replace_content(
old_content := file.read_text(encoding="utf8"),
marker=marker,
new_content=new_content,
)
_write_if_updated(file, old_content, new_content)
def _get_custom_ua_crawlers() -> list[str]:
from cyberdrop_dl.crawlers import Registry
return sorted(
c.FOLDER_DOMAIN
for c in Registry.get_crawlers()
if c.__http_config__
and c.__http_config__.headers
and CDL_USER_AGENT in c.__http_config__.headers.get("User-Agent", "")
)
def update_supported_sites() -> None:
_update_file(
SUPPORTED_SITES_MD,
_make_supported_sites_markdown_table(),
marker="SUPPORTED_SITES",
)
def update_cli_overview() -> None:
new_content = f"```shell\n{_get_help_message()}```"
_update_file(
CLI_ARGUMENTS_MD,
new_content,
marker="CLI_OVERVIEW",
)
def update_custom_ua_crawlers() -> None:
new_content = "- " + "\n- ".join(_get_custom_ua_crawlers())
_update_file(
NETWORK_MD,
new_content,
marker="CUSTOM_UA_CRAWLERS",
)
if __name__ == "__main__":
update_cli_overview()
update_supported_sites()
update_custom_ua_crawlers()