Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

build: Move non-primarily deps to optional extras

Karim shoair6b8d861b9230
diff --git a/pyproject.toml b/pyproject.toml
index eef2839..ad4550f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -58,7 +58,6 @@ classifiers = [
dependencies = [
"lxml>=6.0.0",
"cssselect>=1.3.0",
- "IPython>=8.37", # The last version that supports Python 3.10
"click>=8.2.1",
"orjson>=3.11.2",
"tldextract>=5.3.0",
@@ -68,8 +67,19 @@ dependencies = [
"camoufox>=0.4.11",
"geoip2>=5.1.0",
"msgspec>=0.19.0",
+]
+
+[project.optional-dependencies]
+ai = [
+ "mcp>=1.13.0",
"markdownify>=1.2.0",
- "mcp[cli]>=1.13.0",
+]
+shell = [
+ "IPython>=8.37", # The last version that supports Python 3.10
+ "markdownify>=1.2.0",
+]
+all = [
+ "scrapling[ai,shell]",
]
[project.urls]
diff --git a/tox.ini b/tox.ini
index 31f4684..d2f5b80 100644
--- a/tox.ini
+++ b/tox.ini
@@ -10,10 +10,8 @@ envlist = pre-commit,py{310,311,312,313}
usedevelop = True
changedir = tests
deps =
- playwright==1.52.0
- rebrowser-playwright==1.52.0
- camoufox
-r{toxinidir}/tests/requirements.txt
+extras = ai,shell
commands =
# Run browser tests without parallelization (avoid browser conflicts)
pytest --cov=scrapling --cov-report=xml -k "DynamicFetcher or StealthyFetcher" --verbose