🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(cli): adding terminal command `scrapling install`
Karim shoair03b52aa74cb1
diff --git a/.bandit.yml b/.bandit.ymlindex 9d7a1f7..525749a 100644--- a/.bandit.yml+++ b/.bandit.yml@@ -5,3 +5,5 @@ skips:- B410- B113 # `Requests call without timeout` these requests are done in the benchmark and examples scripts only- B403 # We are using pickle for tests only+- B404 # Using subprocess library+- B602 # subprocess call with shell=True identifieddiff --git a/MANIFEST.in b/MANIFEST.inindex b8fbc5b..8c168dd 100644--- a/MANIFEST.in+++ b/MANIFEST.in@@ -4,7 +4,10 @@ include *.jsinclude scrapling/engines/toolbelt/bypasses/*.jsinclude scrapling/*.dbinclude scrapling/*.db*+include scrapling/*.db-*include scrapling/py.typed+include scrapling/.scrapling_dependencies_installed+include .scrapling_dependencies_installedrecursive-exclude * __pycache__recursive-exclude * *.py[co]\ No newline at end of filediff --git a/scrapling/cli.py b/scrapling/cli.pynew file mode 100644index 0000000..7d22f0c--- /dev/null+++ b/scrapling/cli.py@@ -0,0 +1,37 @@+import os+import subprocess+import sys+from pathlib import Path++import click+++def get_package_dir():+ return Path(os.path.dirname(__file__))+++def run_command(command, line):+ print(f"Installing {line}...")+ _ = subprocess.check_call(command, shell=True)+ # I meant to not use try except here++[email protected](help="Install all Scrapling's Fetchers dependencies")+def install():+ if not get_package_dir().joinpath(".scrapling_dependencies_installed").exists():+ run_command([sys.executable, "-m", "playwright", "install", 'chromium'], 'Playwright browsers')+ run_command([sys.executable, "-m", "playwright", "install-deps", 'chromium', 'firefox'], 'Playwright dependencies')+ run_command([sys.executable, "-m", "camoufox", "fetch", '--browserforge'], 'Camoufox browser and databases')+ # if no errors raised by above commands, then we add below file+ get_package_dir().joinpath(".scrapling_dependencies_installed").touch()+ else:+ print('The dependencies are already installed')+++def main():+ pass+++# Adding commands+main.add_command(install)diff --git a/setup.py b/setup.pyindex 45b88ad..f092a64 100644--- a/setup.py+++ b/setup.py@@ -20,6 +20,11 @@ setup(package_dir={"scrapling": "scrapling",},+ entry_points={+ 'console_scripts': [+ 'scrapling=scrapling.cli:main'+ ],+ },include_package_data=True,classifiers=["Operating System :: OS Independent",@@ -50,6 +55,7 @@ setup("requests>=2.3","lxml>=4.5","cssselect>=1.2",+ 'click',"w3lib","orjson>=3","tldextract",