Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(fetchers): Add backward compatibility for `custom_config` argument

Karim shoair0e1f3b9b0595
diff --git a/scrapling/fetchers/chrome.py b/scrapling/fetchers/chrome.py
index 44e7a44..a706826 100644
--- a/scrapling/fetchers/chrome.py
+++ b/scrapling/fetchers/chrome.py
@@ -52,12 +52,12 @@ class DynamicFetcher(BaseFetcher):
- additional_args: Additional arguments to be passed to Playwright's context as additional settings.
:return: A `Response` object.
"""
- # Get selector_config from kwargs if provided, otherwise use empty dict
- selector_config = kwargs.get("selector_config", {})
+ selector_config = kwargs.get("selector_config", {}) or kwargs.get(
+ "custom_config", {}
+ ) # Checking `custom_config` for backward compatibility
if not isinstance(selector_config, dict):
raise TypeError("Argument `selector_config` must be a dictionary.")
- # Merge selector_config with class defaults
kwargs["selector_config"] = {**cls._generate_parser_arguments(), **selector_config}
with DynamicSession(**kwargs) as session:
@@ -95,12 +95,12 @@ class DynamicFetcher(BaseFetcher):
- additional_args: Additional arguments to be passed to Playwright's context as additional settings.
:return: A `Response` object.
"""
- # Get selector_config from kwargs if provided, otherwise use empty dict
- selector_config = kwargs.get("selector_config", {})
+ selector_config = kwargs.get("selector_config", {}) or kwargs.get(
+ "custom_config", {}
+ ) # Checking `custom_config` for backward compatibility
if not isinstance(selector_config, dict):
raise TypeError("Argument `selector_config` must be a dictionary.")
- # Merge selector_config with class defaults
kwargs["selector_config"] = {**cls._generate_parser_arguments(), **selector_config}
async with AsyncDynamicSession(**kwargs) as session:
diff --git a/scrapling/fetchers/firefox.py b/scrapling/fetchers/firefox.py
index 825b917..a9361d6 100644
--- a/scrapling/fetchers/firefox.py
+++ b/scrapling/fetchers/firefox.py
@@ -45,12 +45,12 @@ class StealthyFetcher(BaseFetcher):
- additional_args: Additional arguments to be passed to Camoufox as additional settings.
:return: A `Response` object.
"""
- # Get selector_config from kwargs if provided, otherwise use empty dict
- selector_config = kwargs.get("selector_config", {})
+ selector_config = kwargs.get("selector_config", {}) or kwargs.get(
+ "custom_config", {}
+ ) # Checking `custom_config` for backward compatibility
if not isinstance(selector_config, dict):
raise TypeError("Argument `selector_config` must be a dictionary.")
- # Merge selector_config with class defaults
kwargs["selector_config"] = {**cls._generate_parser_arguments(), **selector_config}
with StealthySession(**kwargs) as engine:
@@ -90,12 +90,12 @@ class StealthyFetcher(BaseFetcher):
- additional_args: Additional arguments to be passed to Camoufox as additional settings.
:return: A `Response` object.
"""
- # Get selector_config from kwargs if provided, otherwise use empty dict
- selector_config = kwargs.get("selector_config", {})
+ selector_config = kwargs.get("selector_config", {}) or kwargs.get(
+ "custom_config", {}
+ ) # Checking `custom_config` for backward compatibility
if not isinstance(selector_config, dict):
raise TypeError("Argument `selector_config` must be a dictionary.")
- # Merge selector_config with class defaults
kwargs["selector_config"] = {**cls._generate_parser_arguments(), **selector_config}
async with AsyncStealthySession(**kwargs) as engine: