Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(extract): Adding new command to CLI options + Optimizations

Karim shoair1c9e48b1c6eb
diff --git a/scrapling/cli.py b/scrapling/cli.py
index 00e7c3f..9ea0ed0 100644
--- a/scrapling/cli.py
+++ b/scrapling/cli.py
@@ -1,21 +1,75 @@
-import os
-import sys
-import subprocess
from pathlib import Path
+from subprocess import check_output
+from sys import executable as python_executable
-from click import command, option, Choice, group
+from scrapling.core.utils import log
+from scrapling.core.shell import Convertor, _CookieParser
+from scrapling.fetchers import Fetcher, DynamicFetcher, StealthyFetcher
+
+from orjson import loads as json_loads, JSONDecodeError
+from click import command, option, Choice, group, argument
+
+__OUTPUT_FILE_HELP__ = "Output file path can be HTML content, Markdown of the HTML content, or the text content. Use file extensions (`.html`/`.md`/`.txt`) respectively."
def get_package_dir():
- return Path(os.path.dirname(__file__))
+ return Path(__file__).parent
def run_command(cmd, line):
print(f"Installing {line}...")
- _ = subprocess.check_call(cmd, shell=False) # nosec B603
+ _ = check_output(cmd, shell=False) # nosec B603
# I meant to not use try except here
+def parse_headers(header_strings):
+ """Parse header strings into a dictionary"""
+ headers = {}
+ for header in header_strings:
+ if ":" in header:
+ key, value = header.split(":", 1)
+ headers[key.strip()] = value.strip()
+ else:
+ log.warning(f"Invalid header format '{header}', should be 'Key: Value'")
+ return headers
+
+
+def parse_cookies(cookie_string):
+ """Parse cookie string into a dictionary"""
+ if not cookie_string:
+ return {}
+
+ try:
+ cookies = {key: value for key, value in _CookieParser(cookie_string)}
+ except Exception as e:
+ raise ValueError(f"Could not parse cookies '{cookie_string}': {e}")
+
+ return cookies
+
+
+def parse_json_data(json_string):
+ """Parse JSON string into a Python object"""
+ if not json_string:
+ return None
+
+ try:
+ return json_loads(json_string)
+ except JSONDecodeError as e:
+ raise ValueError(f"Invalid JSON data '{json_string}': {e}")
+
+
+def make_request_and_save(fetcher_func, url, output_file, css_selector=None, **kwargs):
+ """Make a request using the specified fetcher function and save the result"""
+ # Handle relative paths - convert to an absolute path based on the current working directory
+ output_path = Path(output_file)
+ if not output_path.is_absolute():
+ output_path = Path.cwd() / output_file
+
+ response = fetcher_func(url, **kwargs)
+ Convertor.write_content_to_file(response, str(output_path), css_selector)
+ log.info(f"Content successfully saved to '{output_path}'")
+
+
@command(help="Install all Scrapling's Fetchers dependencies")
@option(
"-f",
@@ -32,15 +86,22 @@ def install(force):
or not get_package_dir().joinpath(".scrapling_dependencies_installed").exists()
):
run_command(
- [sys.executable, "-m", "playwright", "install", "chromium"],
+ [python_executable, "-m", "playwright", "install", "chromium"],
"Playwright browsers",
)
run_command(
- [sys.executable, "-m", "playwright", "install-deps", "chromium", "firefox"],
+ [
+ python_executable,
+ "-m",
+ "playwright",
+ "install-deps",
+ "chromium",
+ "firefox",
+ ],
"Playwright dependencies",
)
run_command(
- [sys.executable, "-m", "camoufox", "fetch", "--browserforge"],
+ [python_executable, "-m", "camoufox", "fetch", "--browserforge"],
"Camoufox browser and databases",
)
# if no errors raised by the above commands, then we add the below file
@@ -77,6 +138,720 @@ def shell(code, level):
console.start()
+def parse_extract_arguments(headers, cookies, params, json=None):
+ """Parse arguments for extract command"""
+ parsed_headers = parse_headers(headers)
+ parsed_cookies = parse_cookies(cookies)
+ parsed_json = parse_json_data(json)
+ parsed_params = {}
+ for param in params:
+ if "=" in param:
+ key, value = param.split("=", 1)
+ parsed_params[key] = value
+
+ return parsed_headers, parsed_cookies, parsed_params, parsed_json
+
+
+@group(
+ help="Fetch web pages using various fetchers and extract full/selected HTML content as HTML, Markdown, or extract text content."
+)
+def extract():
+ """Extract content from web pages and save to files"""
+ pass
+
+
+ help=f"Perform a GET request and save content to file.\n\n{__OUTPUT_FILE_HELP__}"
+)
+@argument("url", required=True)
+@argument("output_file", required=True)
+@option(
+ "--headers",
+ "-H",
+ multiple=True,
+ help='HTTP headers in format "Key: Value" (can be used multiple times)',
+)
+@option("--cookies", help='Cookies string in format "name1=value1; name2=value2"')
+@option(
+ "--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)"
+)
+@option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
+@option(
+ "--css-selector",
+ "-s",
+ help="CSS selector to extract specific content from the page. It resolves to the first match if multiple matches are found.",
+)
+@option(
+ "--params",
+ "-p",
+ multiple=True,
+ help='Query parameters in format "key=value" (can be used multiple times)',
+)
+@option(
+ "--follow-redirects/--no-follow-redirects",
+ default=True,
+ help="Whether to follow redirects (default: True)",
+)
+@option(
+ "--verify/--no-verify",
+ default=True,
+ help="Whether to verify SSL certificates (default: True)",
+)
+@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).")
+@option(
+ "--stealthy-headers/--no-stealthy-headers",
+ default=True,
+ help="Use stealthy browser headers (default: True)",
+)
+def get(
+ url,
+ output_file,
+ headers,
+ cookies,
+ timeout,
+ proxy,
+ css_selector,
+ params,
+ follow_redirects,
+ verify,
+ impersonate,
+ stealthy_headers,
+):
+ """
+ Perform a GET request and save content to file.
+
+ :param url: Target URL for the request.
+ :param output_file: Output file path (.md for Markdown, .html for HTML).
+ :param headers: HTTP headers to include in the request.
+ :param cookies: Cookies to use in the request.
+ :param timeout: Number of seconds to wait before timing out.
+ :param proxy: Proxy URL to use. (Format: "http://username:password@localhost:8030")
+ :param css_selector: CSS selector to extract specific content.
+ :param params: Query string parameters for the request.
+ :param follow_redirects: Whether to follow redirects.
+ :param verify: Whether to verify HTTPS certificates.
+ :param impersonate: Browser version to impersonate.
+ :param stealthy_headers: If enabled, creates and adds real browser headers.
+ """
+
+ # Parse parameters
+ parsed_headers, parsed_cookies, parsed_params, _ = parse_extract_arguments(
+ headers, cookies, params
+ )
+
+ # Build request arguments
+ kwargs = {
+ "headers": parsed_headers if parsed_headers else None,
+ "cookies": parsed_cookies if parsed_cookies else None,
+ "timeout": timeout,
+ "follow_redirects": follow_redirects,
+ "verify": verify,
+ "stealthy_headers": stealthy_headers,
+ "impersonate": impersonate,
+ }
+
+ if parsed_params:
+ kwargs["params"] = parsed_params
+ if proxy:
+ kwargs["proxy"] = proxy
+
+ make_request_and_save(Fetcher.get, url, output_file, css_selector, **kwargs)
+
+
+ help=f"Perform a POST request and save content to file.\n\n{__OUTPUT_FILE_HELP__}"
+)
+@argument("url", required=True)
+@argument("output_file", required=True)
+@option(
+ "--data",
+ "-d",
+ help='Form data to include in the request body (as string, ex: "param1=value1&param2=value2")',
+)
+@option("--json", "-j", help="JSON data to include in the request body (as string)")
+@option(
+ "--headers",
+ "-H",
+ multiple=True,
+ help='HTTP headers in format "Key: Value" (can be used multiple times)',
+)
+@option("--cookies", help='Cookies string in format "name1=value1; name2=value2"')
+@option(
+ "--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)"
+)
+@option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
+@option(
+ "--css-selector",
+ "-s",
+ help="CSS selector to extract specific content from the page",
+)
+@option(
+ "--params",
+ "-p",
+ multiple=True,
+ help='Query parameters in format "key=value" (can be used multiple times)',
+)
+@option(
+ "--follow-redirects/--no-follow-redirects",
+ default=True,
+ help="Whether to follow redirects (default: True)",
+)
+@option(
+ "--verify/--no-verify",
+ default=True,
+ help="Whether to verify SSL certificates (default: True)",
+)
+@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).")
+@option(
+ "--stealthy-headers/--no-stealthy-headers",
+ default=True,
+ help="Use stealthy browser headers (default: True)",
+)
+def post(
+ url,
+ output_file,
+ data,
+ json,
+ headers,
+ cookies,
+ timeout,
+ proxy,
+ css_selector,
+ params,
+ follow_redirects,
+ verify,
+ impersonate,
+ stealthy_headers,
+):
+ """
+ Perform a POST request and save content to file.
+
+ :param url: Target URL for the request.
+ :param output_file: Output file path (.md for Markdown, .html for HTML).
+ :param data: Form data to include in the request body. (as string, ex: "param1=value1&param2=value2")
+ :param json: A JSON serializable object to include in the body of the request.
+ :param headers: Headers to include in the request.
+ :param cookies: Cookies to use in the request.
+ :param timeout: Number of seconds to wait before timing out.
+ :param proxy: Proxy URL to use.
+ :param css_selector: CSS selector to extract specific content.
+ :param params: Query string parameters for the request.
+ :param follow_redirects: Whether to follow redirects.
+ :param verify: Whether to verify HTTPS certificates.
+ :param impersonate: Browser version to impersonate.
+ :param stealthy_headers: If enabled, creates and adds real browser headers.
+ """
+
+ # Parse parameters
+ parsed_headers, parsed_cookies, parsed_params, parsed_json = (
+ parse_extract_arguments(headers, cookies, params, json)
+ )
+
+ # Build request arguments
+ kwargs = {
+ "headers": parsed_headers if parsed_headers else None,
+ "cookies": parsed_cookies if parsed_cookies else None,
+ "timeout": timeout,
+ "follow_redirects": follow_redirects,
+ "verify": verify,
+ "stealthy_headers": stealthy_headers,
+ "impersonate": impersonate,
+ }
+
+ if data:
+ kwargs["data"] = data
+ if parsed_json:
+ kwargs["json"] = parsed_json
+ if parsed_params:
+ kwargs["params"] = parsed_params
+ if proxy:
+ kwargs["proxy"] = proxy
+
+ make_request_and_save(Fetcher.post, url, output_file, css_selector, **kwargs)
+
+
+ help=f"Perform a PUT request and save content to file.\n\n{__OUTPUT_FILE_HELP__}"
+)
+@argument("url", required=True)
+@argument("output_file", required=True)
+@option("--data", "-d", help="Form data to include in the request body")
+@option("--json", "-j", help="JSON data to include in the request body (as string)")
+@option(
+ "--headers",
+ "-H",
+ multiple=True,
+ help='HTTP headers in format "Key: Value" (can be used multiple times)',
+)
+@option("--cookies", help='Cookies string in format "name1=value1; name2=value2"')
+@option(
+ "--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)"
+)
+@option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
+@option(
+ "--css-selector",
+ "-s",
+ help="CSS selector to extract specific content from the page",
+)
+@option(
+ "--params",
+ "-p",
+ multiple=True,
+ help='Query parameters in format "key=value" (can be used multiple times)',
+)
+@option(
+ "--follow-redirects/--no-follow-redirects",
+ default=True,
+ help="Whether to follow redirects (default: True)",
+)
+@option(
+ "--verify/--no-verify",
+ default=True,
+ help="Whether to verify SSL certificates (default: True)",
+)
+@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).")
+@option(
+ "--stealthy-headers/--no-stealthy-headers",
+ default=True,
+ help="Use stealthy browser headers (default: True)",
+)
+def put(
+ url,
+ output_file,
+ data,
+ json,
+ headers,
+ cookies,
+ timeout,
+ proxy,
+ css_selector,
+ params,
+ follow_redirects,
+ verify,
+ impersonate,
+ stealthy_headers,
+):
+ """
+ Perform a PUT request and save content to file.
+
+ :param url: Target URL for the request.
+ :param output_file: Output file path (.md for Markdown, .html for HTML).
+ :param data: Form data to include in the request body.
+ :param json: A JSON serializable object to include in the body of the request.
+ :param headers: Headers to include in the request.
+ :param cookies: Cookies to use in the request.
+ :param timeout: Number of seconds to wait before timing out.
+ :param proxy: Proxy URL to use.
+ :param css_selector: CSS selector to extract specific content.
+ :param params: Query string parameters for the request.
+ :param follow_redirects: Whether to follow redirects.
+ :param verify: Whether to verify HTTPS certificates.
+ :param impersonate: Browser version to impersonate.
+ :param stealthy_headers: If enabled, creates and adds real browser headers.
+ """
+
+ # Parse parameters
+ parsed_headers, parsed_cookies, parsed_params, parsed_json = (
+ parse_extract_arguments(headers, cookies, params, json)
+ )
+
+ # Build request arguments
+ kwargs = {
+ "headers": parsed_headers if parsed_headers else None,
+ "cookies": parsed_cookies if parsed_cookies else None,
+ "timeout": timeout,
+ "follow_redirects": follow_redirects,
+ "verify": verify,
+ "stealthy_headers": stealthy_headers,
+ "impersonate": impersonate,
+ }
+
+ if data:
+ kwargs["data"] = data
+ if parsed_json:
+ kwargs["json"] = parsed_json
+ if parsed_params:
+ kwargs["params"] = parsed_params
+ if proxy:
+ kwargs["proxy"] = proxy
+
+ make_request_and_save(Fetcher.put, url, output_file, css_selector, **kwargs)
+
+
+ help=f"Perform a DELETE request and save content to file.\n\n{__OUTPUT_FILE_HELP__}"
+)
+@argument("url", required=True)
+@argument("output_file", required=True)
+@option(
+ "--headers",
+ "-H",
+ multiple=True,
+ help='HTTP headers in format "Key: Value" (can be used multiple times)',
+)
+@option("--cookies", help='Cookies string in format "name1=value1; name2=value2"')
+@option(
+ "--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)"
+)
+@option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
+@option(
+ "--css-selector",
+ "-s",
+ help="CSS selector to extract specific content from the page",
+)
+@option(
+ "--params",
+ "-p",
+ multiple=True,
+ help='Query parameters in format "key=value" (can be used multiple times)',
+)
+@option(
+ "--follow-redirects/--no-follow-redirects",
+ default=True,
+ help="Whether to follow redirects (default: True)",
+)
+@option(
+ "--verify/--no-verify",
+ default=True,
+ help="Whether to verify SSL certificates (default: True)",
+)
+@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).")
+@option(
+ "--stealthy-headers/--no-stealthy-headers",
+ default=True,
+ help="Use stealthy browser headers (default: True)",
+)
+def delete(
+ url,
+ output_file,
+ headers,
+ cookies,
+ timeout,
+ proxy,
+ css_selector,
+ params,
+ follow_redirects,
+ verify,
+ impersonate,
+ stealthy_headers,
+):
+ """
+ Perform a DELETE request and save content to file.
+
+ :param url: Target URL for the request.
+ :param output_file: Output file path (.md for Markdown, .html for HTML).
+ :param headers: Headers to include in the request.
+ :param cookies: Cookies to use in the request.
+ :param timeout: Number of seconds to wait before timing out.
+ :param proxy: Proxy URL to use.
+ :param css_selector: CSS selector to extract specific content.
+ :param params: Query string parameters for the request.
+ :param follow_redirects: Whether to follow redirects.
+ :param verify: Whether to verify HTTPS certificates.
+ :param impersonate: Browser version to impersonate.
+ :param stealthy_headers: If enabled, creates and adds real browser headers.
+ """
+
+ # Parse parameters
+ parsed_headers, parsed_cookies, parsed_params, _ = parse_extract_arguments(
+ headers, cookies, params
+ )
+
+ # Build request arguments
+ kwargs = {
+ "headers": parsed_headers if parsed_headers else None,
+ "cookies": parsed_cookies if parsed_cookies else None,
+ "timeout": timeout,
+ "follow_redirects": follow_redirects,
+ "verify": verify,
+ "stealthy_headers": stealthy_headers,
+ "impersonate": impersonate,
+ }
+
+ if parsed_params:
+ kwargs["params"] = parsed_params
+ if proxy:
+ kwargs["proxy"] = proxy
+
+ make_request_and_save(Fetcher.delete, url, output_file, css_selector, **kwargs)
+
+
+ help=f"Use DynamicFetcher to fetch content with browser automation.\n\n{__OUTPUT_FILE_HELP__}"
+)
+@argument("url", required=True)
+@argument("output_file", required=True)
+@option(
+ "--headless/--no-headless",
+ default=True,
+ help="Run browser in headless mode (default: True)",
+)
+@option(
+ "--disable-resources/--enable-resources",
+ default=False,
+ help="Drop unnecessary resources for speed boost (default: False)",
+)
+@option(
+ "--network-idle/--no-network-idle",
+ default=False,
+ help="Wait for network idle (default: False)",
+)
+@option(
+ "--timeout",
+ type=int,
+ default=30000,
+ help="Timeout in milliseconds (default: 30000)",
+)
+@option(
+ "--wait",
+ type=int,
+ default=0,
+ help="Additional wait time in milliseconds after page load (default: 0)",
+)
+@option(
+ "--css-selector",
+ "-s",
+ help="CSS selector to extract specific content from the page",
+)
+@option("--wait-selector", help="CSS selector to wait for before proceeding")
+@option("--locale", default="en-US", help="Browser locale (default: en-US)")
+@option(
+ "--stealth/--no-stealth", default=False, help="Enable stealth mode (default: False)"
+)
+@option(
+ "--hide-canvas/--show-canvas",
+ default=False,
+ help="Add noise to canvas operations (default: False)",
+)
+@option(
+ "--disable-webgl/--enable-webgl",
+ default=False,
+ help="Disable WebGL support (default: False)",
+)
+@option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
+@option(
+ "--extra-headers",
+ "-H",
+ multiple=True,
+ help='Extra headers in format "Key: Value" (can be used multiple times)',
+)
+def fetch(
+ url,
+ output_file,
+ headless,
+ disable_resources,
+ network_idle,
+ timeout,
+ wait,
+ css_selector,
+ wait_selector,
+ locale,
+ stealth,
+ hide_canvas,
+ disable_webgl,
+ proxy,
+ extra_headers,
+):
+ """
+ Opens up a browser and fetch content using DynamicFetcher.
+
+ :param url: Target url.
+ :param output_file: Output file path (.md for Markdown, .html for HTML).
+ :param headless: Run the browser in headless/hidden or headful/visible mode.
+ :param disable_resources: Drop requests of unnecessary resources for a speed boost.
+ :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
+ :param timeout: The timeout in milliseconds that is used in all operations and waits through the page.
+ :param wait: The time (milliseconds) the fetcher will wait after everything finishes before returning.
+ :param css_selector: CSS selector to extract specific content.
+ :param wait_selector: Wait for a specific CSS selector to be in a specific state.
+ :param locale: Set the locale for the browser.
+ :param stealth: Enables stealth mode.
+ :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
+ :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
+ :param proxy: The proxy to be used with requests.
+ :param extra_headers: Extra headers to add to the request.
+ """
+
+ # Parse parameters
+ parsed_headers = parse_headers(extra_headers)
+
+ # Build request arguments
+ kwargs = {
+ "headless": headless,
+ "disable_resources": disable_resources,
+ "network_idle": network_idle,
+ "timeout": timeout,
+ "locale": locale,
+ "stealth": stealth,
+ "hide_canvas": hide_canvas,
+ "disable_webgl": disable_webgl,
+ }
+
+ if wait > 0:
+ kwargs["wait"] = wait
+ if wait_selector:
+ kwargs["wait_selector"] = wait_selector
+ if proxy:
+ kwargs["proxy"] = proxy
+ if parsed_headers:
+ kwargs["extra_headers"] = parsed_headers
+
+ make_request_and_save(
+ DynamicFetcher.fetch, url, output_file, css_selector, **kwargs
+ )
+
+
+ help=f"Use StealthyFetcher to fetch content with advanced stealth features.\n\n{__OUTPUT_FILE_HELP__}"
+)
+@argument("url", required=True)
+@argument("output_file", required=True)
+@option(
+ "--headless/--no-headless",
+ default=True,
+ help="Run browser in headless mode (default: True)",
+)
+@option(
+ "--block-images/--allow-images",
+ default=False,
+ help="Block image loading (default: False)",
+)
+@option(
+ "--disable-resources/--enable-resources",
+ default=False,
+ help="Drop unnecessary resources for speed boost (default: False)",
+)
+@option(
+ "--block-webrtc/--allow-webrtc",
+ default=False,
+ help="Block WebRTC entirely (default: False)",
+)
+@option(
+ "--humanize/--no-humanize",
+ default=False,
+ help="Humanize cursor movement (default: False)",
+)
+@option(
+ "--solve-cloudflare/--no-solve-cloudflare",
+ default=False,
+ help="Solve Cloudflare challenges (default: False)",
+)
+@option("--allow-webgl/--block-webgl", default=True, help="Allow WebGL (default: True)")
+@option(
+ "--network-idle/--no-network-idle",
+ default=False,
+ help="Wait for network idle (default: False)",
+)
+@option(
+ "--disable-ads/--allow-ads",
+ default=False,
+ help="Install uBlock Origin addon (default: False)",
+)
+@option(
+ "--timeout",
+ type=int,
+ default=30000,
+ help="Timeout in milliseconds (default: 30000)",
+)
+@option(
+ "--wait",
+ type=int,
+ default=0,
+ help="Additional wait time in milliseconds after page load (default: 0)",
+)
+@option(
+ "--css-selector",
+ "-s",
+ help="CSS selector to extract specific content from the page",
+)
+@option("--wait-selector", help="CSS selector to wait for before proceeding")
+@option(
+ "--geoip/--no-geoip",
+ default=False,
+ help="Use IP geolocation for timezone/locale (default: False)",
+)
+@option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
+@option(
+ "--extra-headers",
+ "-H",
+ multiple=True,
+ help='Extra headers in format "Key: Value" (can be used multiple times)',
+)
+def stealthy_fetch(
+ url,
+ output_file,
+ headless,
+ block_images,
+ disable_resources,
+ block_webrtc,
+ humanize,
+ solve_cloudflare,
+ allow_webgl,
+ network_idle,
+ disable_ads,
+ timeout,
+ wait,
+ css_selector,
+ wait_selector,
+ geoip,
+ proxy,
+ extra_headers,
+):
+ """
+ Opens up a browser with advanced stealth features and fetch content using StealthyFetcher.
+
+ :param url: Target url.
+ :param output_file: Output file path (.md for Markdown, .html for HTML).
+ :param headless: Run the browser in headless/hidden, virtual screen mode, or headful/visible mode.
+ :param block_images: Prevent the loading of images through Firefox preferences.
+ :param disable_resources: Drop requests of unnecessary resources for a speed boost.
+ :param block_webrtc: Blocks WebRTC entirely.
+ :param humanize: Humanize the cursor movement.
+ :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page.
+ :param allow_webgl: Allow WebGL (recommended to keep enabled).
+ :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
+ :param disable_ads: Install the uBlock Origin addon on the browser.
+ :param timeout: The timeout in milliseconds that is used in all operations and waits through the page.
+ :param wait: The time (milliseconds) the fetcher will wait after everything finishes before returning.
+ :param css_selector: CSS selector to extract specific content.
+ :param wait_selector: Wait for a specific CSS selector to be in a specific state.
+ :param geoip: Automatically use IP's longitude, latitude, timezone, country, locale.
+ :param proxy: The proxy to be used with requests.
+ :param extra_headers: Extra headers to add to the request.
+ """
+
+ # Parse parameters
+ parsed_headers = parse_headers(extra_headers)
+
+ # Build request arguments
+ kwargs = {
+ "headless": headless,
+ "block_images": block_images,
+ "disable_resources": disable_resources,
+ "block_webrtc": block_webrtc,
+ "humanize": humanize,
+ "solve_cloudflare": solve_cloudflare,
+ "allow_webgl": allow_webgl,
+ "network_idle": network_idle,
+ "disable_ads": disable_ads,
+ "timeout": timeout,
+ "geoip": geoip,
+ }
+
+ if wait > 0:
+ kwargs["wait"] = wait
+ if wait_selector:
+ kwargs["wait_selector"] = wait_selector
+ if proxy:
+ kwargs["proxy"] = proxy
+ if parsed_headers:
+ kwargs["extra_headers"] = parsed_headers
+
+ make_request_and_save(
+ StealthyFetcher.fetch, url, output_file, css_selector, **kwargs
+ )
+
+
@group()
def main():
pass
@@ -85,3 +860,4 @@ def main():
# Adding commands
main.add_command(install)
main.add_command(shell)
+main.add_command(extract)
diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py
index b56a063..63ec304 100644
--- a/scrapling/core/shell.py
+++ b/scrapling/core/shell.py
@@ -1,4 +1,5 @@
# -*- coding: utf-8 -*-
+from re import sub as re_sub
from sys import stderr
from functools import wraps
from http import cookies as Cookie
@@ -24,6 +25,7 @@ from IPython.terminal.embed import InteractiveShellEmbed
from orjson import loads as json_loads, JSONDecodeError
from scrapling import __version__
+from scrapling.core.custom_types import TextHandler
from scrapling.core.utils import log
from scrapling.parser import Adaptor, Adaptors
from scrapling.core._types import List, Optional, Dict, Tuple, Any, Union
@@ -63,6 +65,14 @@ Request = namedtuple(
)
+def _CookieParser(cookie_string):
+ # Errors will be handled on call so the log can be specified
+ cookie_parser = Cookie.SimpleCookie()
+ cookie_parser.load(cookie_string)
+ for key, morsel in cookie_parser.items():
+ yield key, morsel.value
+
+
# Suppress exit on error to handle parsing errors gracefully
class NoExitArgumentParser(ArgumentParser):
def error(self, message):
@@ -156,12 +166,11 @@ class CurlParser:
if header_key.lower() == "cookie":
try:
- cookie_parser = Cookie.SimpleCookie()
- cookie_parser.load(header_value)
- for key, morsel in cookie_parser.items():
- cookie_dict[key] = morsel.value
+ cookie_dict = {
+ key: value for key, value in _CookieParser(header_value)
+ }
except Exception as e:
- log.error(
+ raise ValueError(
f"Could not parse cookie string from -H '{header_value}': {e}"
)
else:
@@ -221,12 +230,9 @@ class CurlParser:
if parsed_args.cookie:
# We are focusing on the string format from DevTools.
try:
- cookie_parser = Cookie.SimpleCookie()
- cookie_parser.load(parsed_args.cookie)
- for key, morsel in cookie_parser.items():
- # Update the cookie dict, potentially overwriting
- # cookies with the same name from -H 'Cookie:'
- cookies[key] = morsel.value
+ for key, value in _CookieParser(parsed_args.cookie):
+ # Update the cookie dict, potentially overwriting cookies with the same name from -H 'cookie:'
+ cookies[key] = value
log.debug(f"Parsed cookies from -b argument: {list(cookies.keys())}")
except Exception as e:
log.error(
@@ -545,3 +551,47 @@ Type 'exit' or press Ctrl+D to exit.
return
ipython_shell()
+
+
+class Convertor:
+ """Utils for the extract shell command"""
+
+ @classmethod
+ def __convert_to_markdown(cls, body: TextHandler) -> str:
+ """Convert HTML content to Markdown"""
+ from markdownify import markdownify
+
+ return markdownify(body)
+
+ @classmethod
+ def write_content_to_file(
+ cls, page: Adaptor, filename: str, css_selector: Optional[str] = None
+ ) -> None:
+ """Write an Adaptor's content to a file"""
+ if not page or not isinstance(page, Adaptor):
+ raise TypeError("Input must be of type `Adaptor`")
+ elif not filename or not isinstance(filename, str) or not filename.strip():
+ raise ValueError("Filename must be provided")
+ elif not filename.endswith((".md", ".html", ".txt")):
+ raise ValueError(
+ "Unknown file type: filename must end with '.md', '.html', or '.txt'"
+ )
+ else:
+ body = page if not css_selector else page.css_first(css_selector)
+ with open(filename, "w", encoding="utf-8") as f:
+ if filename.endswith(".md"):
+ f.write(cls.__convert_to_markdown(body.body))
+ elif filename.endswith(".html"):
+ f.write(body.body)
+ elif filename.endswith(".txt"):
+ txt_content = body.get_all_text(strip=True)
+ for s in (
+ "\n",
+ "\r",
+ "\t",
+ " ",
+ ):
+ # Remove consecutive white-spaces
+ txt_content = re_sub(f"[{s}]+", s, txt_content)
+
+ f.write(txt_content)