Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(cli): adding terminal command `scrapling install`

Karim shoair03b52aa74cb1
diff --git a/.bandit.yml b/.bandit.yml
index 9d7a1f7..525749a 100644
--- a/.bandit.yml
+++ b/.bandit.yml
@@ -5,3 +5,5 @@ skips:
- B410
- B113 # `Requests call without timeout` these requests are done in the benchmark and examples scripts only
- B403 # We are using pickle for tests only
+- B404 # Using subprocess library
+- B602 # subprocess call with shell=True identified
diff --git a/MANIFEST.in b/MANIFEST.in
index b8fbc5b..8c168dd 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -4,7 +4,10 @@ include *.js
include scrapling/engines/toolbelt/bypasses/*.js
include scrapling/*.db
include scrapling/*.db*
+include scrapling/*.db-*
include scrapling/py.typed
+include scrapling/.scrapling_dependencies_installed
+include .scrapling_dependencies_installed
recursive-exclude * __pycache__
recursive-exclude * *.py[co]
\ No newline at end of file
diff --git a/scrapling/cli.py b/scrapling/cli.py
new file mode 100644
index 0000000..7d22f0c
--- /dev/null
+++ b/scrapling/cli.py
@@ -0,0 +1,37 @@
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import click
+
+
+def get_package_dir():
+ return Path(os.path.dirname(__file__))
+
+
+def run_command(command, line):
+ print(f"Installing {line}...")
+ _ = subprocess.check_call(command, shell=True)
+ # I meant to not use try except here
+
+
[email protected](help="Install all Scrapling's Fetchers dependencies")
+def install():
+ if not get_package_dir().joinpath(".scrapling_dependencies_installed").exists():
+ run_command([sys.executable, "-m", "playwright", "install", 'chromium'], 'Playwright browsers')
+ run_command([sys.executable, "-m", "playwright", "install-deps", 'chromium', 'firefox'], 'Playwright dependencies')
+ run_command([sys.executable, "-m", "camoufox", "fetch", '--browserforge'], 'Camoufox browser and databases')
+ # if no errors raised by above commands, then we add below file
+ get_package_dir().joinpath(".scrapling_dependencies_installed").touch()
+ else:
+ print('The dependencies are already installed')
+
+
+def main():
+ pass
+
+
+# Adding commands
+main.add_command(install)
diff --git a/setup.py b/setup.py
index 45b88ad..f092a64 100644
--- a/setup.py
+++ b/setup.py
@@ -20,6 +20,11 @@ setup(
package_dir={
"scrapling": "scrapling",
},
+ entry_points={
+ 'console_scripts': [
+ 'scrapling=scrapling.cli:main'
+ ],
+ },
include_package_data=True,
classifiers=[
"Operating System :: OS Independent",
@@ -50,6 +55,7 @@ setup(
"requests>=2.3",
"lxml>=4.5",
"cssselect>=1.2",
+ 'click',
"w3lib",
"orjson>=3",
"tldextract",