Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(shell): Fixing a bug with content converting

Karim shoair2d704b2a8b9d
diff --git a/scrapling/cli.py b/scrapling/cli.py
index 48e99bf..12b96c6 100644
--- a/scrapling/cli.py
+++ b/scrapling/cli.py
@@ -32,8 +32,8 @@ def __ParseJSONData(json_string: Optional[str] = None) -> Optional[Dict[str, Any
try:
return json_loads(json_string)
- except JSONDecodeError as e: # pragma: no cover
- raise ValueError(f"Invalid JSON data '{json_string}': {e}")
+ except JSONDecodeError as err: # pragma: no cover
+ raise ValueError(f"Invalid JSON data '{json_string}': {err}")
def __Request_and_Save(
@@ -65,8 +65,8 @@ def __ParseExtractArguments(
for key, value in _CookieParser(cookies):
try:
parsed_cookies[key] = value
- except Exception as e:
- raise ValueError(f"Could not parse cookies '{cookies}': {e}")
+ except Exception as err:
+ raise ValueError(f"Could not parse cookies '{cookies}': {err}")
parsed_json = __ParseJSONData(json)
parsed_params = {}
diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py
index 4cca171..f9f790f 100644
--- a/scrapling/core/shell.py
+++ b/scrapling/core/shell.py
@@ -545,7 +545,7 @@ class Convertor:
for page in pages:
match extraction_type:
case "markdown":
- yield cls._convert_to_markdown(page.body)
+ yield cls._convert_to_markdown(page.html_content)
case "html":
yield page.body
case "text":