🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(cli): Adding Scrapling Shell feature
Karim shoair49b7ae13f68e
diff --git a/scrapling/cli.py b/scrapling/cli.pyindex 5b8e988..d1d4aa5 100644--- a/scrapling/cli.py+++ b/scrapling/cli.py@@ -5,6 +5,8 @@ from pathlib import Pathimport click+from scrapling.core.shell import CustomShell+def get_package_dir():return Path(os.path.dirname(__file__))@@ -49,6 +51,32 @@ def install(force):print("The dependencies are already installed")[email protected](help="Interactive scraping console")+ "-c",+ "--code",+ "code",+ is_flag=False,+ default="",+ type=str,+ help="Evaluate the code in the shell, print the result and exit",+)+ "-L",+ "--loglevel",+ "level",+ is_flag=False,+ default="debug",+ type=click.Choice(+ ["debug", "info", "warning", "error", "critical", "fatal"], case_sensitive=False+ ),+ help="Log level (default: DEBUG)",+)+def shell(code, level):+ console = CustomShell(code=code, log_level=level)+ console.start()++@click.group()def main():pass@@ -56,3 +84,4 @@ def main():# Adding commandsmain.add_command(install)+main.add_command(shell)diff --git a/scrapling/core/shell.py b/scrapling/core/shell.pynew file mode 100644index 0000000..c3961e4--- /dev/null+++ b/scrapling/core/shell.py@@ -0,0 +1,175 @@+import os+import logging+import tempfile+import webbrowser+from functools import wraps++from IPython.terminal.embed import InteractiveShellEmbed++from scrapling import __version__+from scrapling.core.utils import log+from scrapling.parser import Adaptor, Adaptors+from scrapling.fetchers import Fetcher, AsyncFetcher, PlayWrightFetcher, StealthyFetcher+++_known_logging_levels = {+ "debug": logging.DEBUG,+ "info": logging.INFO,+ "warning": logging.WARNING,+ "error": logging.ERROR,+ "critical": logging.CRITICAL,+ "fatal": logging.FATAL,+}+++def show_page_in_browser(page):+ if not page:+ log.error("Input must be of type `Adaptor`")+ return++ fd, fname = tempfile.mkstemp(".html")+ os.write(fd, page.body.encode("utf-8"))+ os.close(fd)+ webbrowser.open(f"file://{fname}")+++class CustomShell:+ """A custom IPython shell with minimal dependencies"""++ def __init__(self, code, log_level="debug"):+ self.code = code+ self.page = None+ self.pages = Adaptors([])+ log_level = log_level.strip().lower()++ if _known_logging_levels.get(log_level):+ self.log_level = _known_logging_levels[log_level]+ else:+ log.error(f'Unknown log level "{log_level}", defaulting to "DEBUG"')+ self.log_level = logging.DEBUG++ self.shell = None++ # Initialize your application components+ self.init_components()++ def init_components(self):+ """Initialize application components"""+ # This is where you'd set up your application-specific objects+ if self.log_level:+ logging.getLogger("scrapling").setLevel(self.log_level)++ settings = Fetcher.display_config()+ _ = settings.pop("storage")+ _ = settings.pop("storage_args")+ log.info(f"Scrapling {__version__} shell started")+ log.info(f"Logging level is set to '{logging.getLevelName(self.log_level)}'")+ log.info(f"Fetchers' parsing settings: {settings}")++ @staticmethod+ def banner():+ """Create a custom banner for the shell"""+ return f"""+-> Available Scrapling objects:+ - Fetcher/AsyncFetcher+ - PlayWrightFetcher+ - StealthyFetcher+ - Adaptor++-> Useful shortcuts:+ - {"get":<30} Shortcut for `Fetcher.get`+ - {"post":<30} Shortcut for `Fetcher.post`+ - {"put":<30} Shortcut for `Fetcher.put`+ - {"delete":<30} Shortcut for `Fetcher.delete`+ - {"fetch":<30} Shortcut for `PlayWrightFetcher.fetch`+ - {"stealthy_fetch":<30} Shortcut for `StealthyFetcher.fetch`++-> Useful commands+ - {"page / response":<30} The response object of the last page you fetched+ - {"pages":<30} Adaptors object of the last 5 response objects you fetched+ - {"view(page)":<30} View page in a browser+ - {"help()":<30} Show this help message (Shell help)++Type 'exit' or press Ctrl+D to exit.+ """++ def update_page(self, result):+ """Update current page and add to pages history"""+ self.page = result+ self.pages.append(result)+ if len(self.pages) > 5:+ self.pages.pop(0) # Remove oldest item++ # Update in IPython namespace too+ if self.shell:+ self.shell.user_ns["page"] = self.page+ self.shell.user_ns["response"] = self.page+ self.shell.user_ns["pages"] = self.pages++ return result++ def create_wrapper(self, func):+ """Create a wrapper that preserves function signature but updates page"""++ @wraps(func)+ def wrapper(*args, **kwargs):+ result = func(*args, **kwargs)+ return self.update_page(result)++ return wrapper++ def get_namespace(self):+ """Create a namespace with application-specific objects"""++ # Create wrapped versions of fetch functions+ get = self.create_wrapper(Fetcher.get)+ post = self.create_wrapper(Fetcher.post)+ put = self.create_wrapper(Fetcher.put)+ delete = self.create_wrapper(Fetcher.delete)+ dynamic_fetch = self.create_wrapper(PlayWrightFetcher.fetch)+ stealthy_fetch = self.create_wrapper(StealthyFetcher.fetch)++ # Create the namespace dictionary+ return {+ "get": get,+ "post": post,+ "put": put,+ "delete": delete,+ "Fetcher": Fetcher,+ "AsyncFetcher": AsyncFetcher,+ "fetch": dynamic_fetch,+ "PlayWrightFetcher": PlayWrightFetcher,+ "stealthy_fetch": stealthy_fetch,+ "StealthyFetcher": StealthyFetcher,+ "Adaptor": Adaptor,+ "page": self.page,+ "response": self.page,+ "pages": self.pages,+ "view": show_page_in_browser,+ "help": self.show_help,+ }++ def show_help(self):+ """Show help information"""+ print(self.banner())++ def start(self):+ """Start the interactive shell"""+ # Create the shell+ ipython_shell = InteractiveShellEmbed(banner1=self.banner(), exit_msg="Bye Bye")++ # Store reference to the shell+ self.shell = ipython_shell++ # Get our namespace with application objects+ namespace = self.get_namespace()++ ipython_shell.user_ns.update(namespace)+ # If a command was provided, execute it and exit+ if self.code:+ # Execute the command in the namespace+ ipython_shell.run_cell(self.code, store_history=False)+ return++ # Start the shell with our namespace+ ipython_shell(local_ns=namespace)diff --git a/setup.py b/setup.pyindex ea1f644..72b6585 100644--- a/setup.py+++ b/setup.py@@ -52,6 +52,7 @@ setup(install_requires=["lxml>=5.0","cssselect>=1.2",+ "IPython","click","w3lib","orjson>=3",