🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
style: replacing `os` with `Pathlib` and small optimizations
Karim shoaire7cdd39695eb
diff --git a/scrapling/core/shell.py b/scrapling/core/shell.pyindex f11f386..8e95a9a 100644--- a/scrapling/core/shell.py+++ b/scrapling/core/shell.py@@ -6,7 +6,6 @@ from http import cookies as Cookiefrom collections import namedtuplefrom shlex import split as shlex_splitfrom tempfile import mkstemp as make_temp_file-from os import write as os_write, close as os_closefrom urllib.parse import urlparse, urlunparse, parse_qslfrom argparse import ArgumentParser, SUPPRESSfrom webbrowser import open as open_in_browser@@ -405,9 +404,10 @@ def show_page_in_browser(page: Selector):returntry:- fd, fname = make_temp_file(".html")- os_write(fd, page.body.encode("utf-8"))- os_close(fd)+ fd, fname = make_temp_file(prefix="scrapling_view_", suffix=".html")+ with open(fd, "w", encoding="utf-8") as f:+ f.write(page.body)+open_in_browser(f"file://{fname}")except IOError as e:log.error(f"Failed to write temporary file for viewing: {e}")diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.pyindex e6557ff..2426909 100644--- a/scrapling/engines/_browsers/_validators.py+++ b/scrapling/engines/_browsers/_validators.py@@ -1,13 +1,12 @@from msgspec import Struct, convert, ValidationErrorfrom urllib.parse import urlparse-from os.path import exists, isdir+from pathlib import Pathfrom scrapling.core._types import (Optional,Union,Dict,Callable,- Literal,List,SelectorWaitStates,)@@ -125,9 +124,10 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):self.addons = []else:for addon in self.addons:- if not exists(addon):+ addon_path = Path(addon)+ if not addon_path.exists():raise FileNotFoundError(f"Addon's path not found: {addon}")- elif not isdir(addon):+ elif not addon_path.is_dir():raise ValueError(f"Addon's path is not a folder, you need to pass a folder of the extracted addon: {addon}")diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.pyindex 95ed535..e03104c 100644--- a/scrapling/engines/toolbelt/navigation.py+++ b/scrapling/engines/toolbelt/navigation.py@@ -2,17 +2,20 @@Functions related to files and URLs"""-import os+from pathlib import Path+from functools import lru_cachefrom urllib.parse import urlencode, urlparsefrom playwright.async_api import Route as async_Routefrom msgspec import Struct, structs, convert, ValidationErrorfrom playwright.sync_api import Route+from scrapling.core.utils import logfrom scrapling.core._types import Dict, Optional, Union, Tuple-from scrapling.core.utils import log, lru_cachefrom scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES+__BYPASSES_DIR__ = Path(__file__).parent / "bypasses"+class ProxyDict(Struct):server: str@@ -129,5 +132,4 @@ def js_bypass_path(filename: str) -> str::param filename: The base filename of the JS file.:return: The full path of the JS file."""- current_directory = os.path.dirname(__file__)- return os.path.join(current_directory, "bypasses", filename)+ return str(__BYPASSES_DIR__ / filename)diff --git a/scrapling/parser.py b/scrapling/parser.pyindex 05d2384..459286a 100644--- a/scrapling/parser.py+++ b/scrapling/parser.py@@ -1,4 +1,4 @@-import os+from pathlib import Pathimport refrom inspect import signaturefrom difflib import SequenceMatcher@@ -39,6 +39,8 @@ from scrapling.core.storage import (from scrapling.core.translator import translator_instancefrom scrapling.core.utils import clean_spaces, flatten, html_forbidden, is_jsonable, log+__DEFAULT_DB_FILE__ = str(Path(__file__).parent / "elements_storage.db")+class Selector(SelectorsGeneration):__slots__ = (@@ -145,9 +147,7 @@ class Selector(SelectorsGeneration):else:if not storage_args:storage_args = {- "storage_file": os.path.join(- os.path.dirname(__file__), "elements_storage.db"- ),+ "storage_file": __DEFAULT_DB_FILE__,"url": url,}