Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(cli): Adding Scrapling Shell feature

Karim shoair49b7ae13f68e
diff --git a/scrapling/cli.py b/scrapling/cli.py
index 5b8e988..d1d4aa5 100644
--- a/scrapling/cli.py
+++ b/scrapling/cli.py
@@ -5,6 +5,8 @@ from pathlib import Path
import click
+from scrapling.core.shell import CustomShell
+
def get_package_dir():
return Path(os.path.dirname(__file__))
@@ -49,6 +51,32 @@ def install(force):
print("The dependencies are already installed")
[email protected](help="Interactive scraping console")
+ "-c",
+ "--code",
+ "code",
+ is_flag=False,
+ default="",
+ type=str,
+ help="Evaluate the code in the shell, print the result and exit",
+)
+ "-L",
+ "--loglevel",
+ "level",
+ is_flag=False,
+ default="debug",
+ type=click.Choice(
+ ["debug", "info", "warning", "error", "critical", "fatal"], case_sensitive=False
+ ),
+ help="Log level (default: DEBUG)",
+)
+def shell(code, level):
+ console = CustomShell(code=code, log_level=level)
+ console.start()
+
+
@click.group()
def main():
pass
@@ -56,3 +84,4 @@ def main():
# Adding commands
main.add_command(install)
+main.add_command(shell)
diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py
new file mode 100644
index 0000000..c3961e4
--- /dev/null
+++ b/scrapling/core/shell.py
@@ -0,0 +1,175 @@
+import os
+import logging
+import tempfile
+import webbrowser
+from functools import wraps
+
+from IPython.terminal.embed import InteractiveShellEmbed
+
+from scrapling import __version__
+from scrapling.core.utils import log
+from scrapling.parser import Adaptor, Adaptors
+from scrapling.fetchers import Fetcher, AsyncFetcher, PlayWrightFetcher, StealthyFetcher
+
+
+_known_logging_levels = {
+ "debug": logging.DEBUG,
+ "info": logging.INFO,
+ "warning": logging.WARNING,
+ "error": logging.ERROR,
+ "critical": logging.CRITICAL,
+ "fatal": logging.FATAL,
+}
+
+
+def show_page_in_browser(page):
+ if not page:
+ log.error("Input must be of type `Adaptor`")
+ return
+
+ fd, fname = tempfile.mkstemp(".html")
+ os.write(fd, page.body.encode("utf-8"))
+ os.close(fd)
+ webbrowser.open(f"file://{fname}")
+
+
+class CustomShell:
+ """A custom IPython shell with minimal dependencies"""
+
+ def __init__(self, code, log_level="debug"):
+ self.code = code
+ self.page = None
+ self.pages = Adaptors([])
+ log_level = log_level.strip().lower()
+
+ if _known_logging_levels.get(log_level):
+ self.log_level = _known_logging_levels[log_level]
+ else:
+ log.error(f'Unknown log level "{log_level}", defaulting to "DEBUG"')
+ self.log_level = logging.DEBUG
+
+ self.shell = None
+
+ # Initialize your application components
+ self.init_components()
+
+ def init_components(self):
+ """Initialize application components"""
+ # This is where you'd set up your application-specific objects
+ if self.log_level:
+ logging.getLogger("scrapling").setLevel(self.log_level)
+
+ settings = Fetcher.display_config()
+ _ = settings.pop("storage")
+ _ = settings.pop("storage_args")
+ log.info(f"Scrapling {__version__} shell started")
+ log.info(f"Logging level is set to '{logging.getLevelName(self.log_level)}'")
+ log.info(f"Fetchers' parsing settings: {settings}")
+
+ @staticmethod
+ def banner():
+ """Create a custom banner for the shell"""
+ return f"""
+-> Available Scrapling objects:
+ - Fetcher/AsyncFetcher
+ - PlayWrightFetcher
+ - StealthyFetcher
+ - Adaptor
+
+-> Useful shortcuts:
+ - {"get":<30} Shortcut for `Fetcher.get`
+ - {"post":<30} Shortcut for `Fetcher.post`
+ - {"put":<30} Shortcut for `Fetcher.put`
+ - {"delete":<30} Shortcut for `Fetcher.delete`
+ - {"fetch":<30} Shortcut for `PlayWrightFetcher.fetch`
+ - {"stealthy_fetch":<30} Shortcut for `StealthyFetcher.fetch`
+
+-> Useful commands
+ - {"page / response":<30} The response object of the last page you fetched
+ - {"pages":<30} Adaptors object of the last 5 response objects you fetched
+ - {"view(page)":<30} View page in a browser
+ - {"help()":<30} Show this help message (Shell help)
+
+Type 'exit' or press Ctrl+D to exit.
+ """
+
+ def update_page(self, result):
+ """Update current page and add to pages history"""
+ self.page = result
+ self.pages.append(result)
+ if len(self.pages) > 5:
+ self.pages.pop(0) # Remove oldest item
+
+ # Update in IPython namespace too
+ if self.shell:
+ self.shell.user_ns["page"] = self.page
+ self.shell.user_ns["response"] = self.page
+ self.shell.user_ns["pages"] = self.pages
+
+ return result
+
+ def create_wrapper(self, func):
+ """Create a wrapper that preserves function signature but updates page"""
+
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ result = func(*args, **kwargs)
+ return self.update_page(result)
+
+ return wrapper
+
+ def get_namespace(self):
+ """Create a namespace with application-specific objects"""
+
+ # Create wrapped versions of fetch functions
+ get = self.create_wrapper(Fetcher.get)
+ post = self.create_wrapper(Fetcher.post)
+ put = self.create_wrapper(Fetcher.put)
+ delete = self.create_wrapper(Fetcher.delete)
+ dynamic_fetch = self.create_wrapper(PlayWrightFetcher.fetch)
+ stealthy_fetch = self.create_wrapper(StealthyFetcher.fetch)
+
+ # Create the namespace dictionary
+ return {
+ "get": get,
+ "post": post,
+ "put": put,
+ "delete": delete,
+ "Fetcher": Fetcher,
+ "AsyncFetcher": AsyncFetcher,
+ "fetch": dynamic_fetch,
+ "PlayWrightFetcher": PlayWrightFetcher,
+ "stealthy_fetch": stealthy_fetch,
+ "StealthyFetcher": StealthyFetcher,
+ "Adaptor": Adaptor,
+ "page": self.page,
+ "response": self.page,
+ "pages": self.pages,
+ "view": show_page_in_browser,
+ "help": self.show_help,
+ }
+
+ def show_help(self):
+ """Show help information"""
+ print(self.banner())
+
+ def start(self):
+ """Start the interactive shell"""
+ # Create the shell
+ ipython_shell = InteractiveShellEmbed(banner1=self.banner(), exit_msg="Bye Bye")
+
+ # Store reference to the shell
+ self.shell = ipython_shell
+
+ # Get our namespace with application objects
+ namespace = self.get_namespace()
+
+ ipython_shell.user_ns.update(namespace)
+ # If a command was provided, execute it and exit
+ if self.code:
+ # Execute the command in the namespace
+ ipython_shell.run_cell(self.code, store_history=False)
+ return
+
+ # Start the shell with our namespace
+ ipython_shell(local_ns=namespace)
diff --git a/setup.py b/setup.py
index ea1f644..72b6585 100644
--- a/setup.py
+++ b/setup.py
@@ -52,6 +52,7 @@ setup(
install_requires=[
"lxml>=5.0",
"cssselect>=1.2",
+ "IPython",
"click",
"w3lib",
"orjson>=3",