Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

style: replacing `os` with `Pathlib` and small optimizations

Karim shoaire7cdd39695eb
diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py
index f11f386..8e95a9a 100644
--- a/scrapling/core/shell.py
+++ b/scrapling/core/shell.py
@@ -6,7 +6,6 @@ from http import cookies as Cookie
from collections import namedtuple
from shlex import split as shlex_split
from tempfile import mkstemp as make_temp_file
-from os import write as os_write, close as os_close
from urllib.parse import urlparse, urlunparse, parse_qsl
from argparse import ArgumentParser, SUPPRESS
from webbrowser import open as open_in_browser
@@ -405,9 +404,10 @@ def show_page_in_browser(page: Selector):
return
try:
- fd, fname = make_temp_file(".html")
- os_write(fd, page.body.encode("utf-8"))
- os_close(fd)
+ fd, fname = make_temp_file(prefix="scrapling_view_", suffix=".html")
+ with open(fd, "w", encoding="utf-8") as f:
+ f.write(page.body)
+
open_in_browser(f"file://{fname}")
except IOError as e:
log.error(f"Failed to write temporary file for viewing: {e}")
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py
index e6557ff..2426909 100644
--- a/scrapling/engines/_browsers/_validators.py
+++ b/scrapling/engines/_browsers/_validators.py
@@ -1,13 +1,12 @@
from msgspec import Struct, convert, ValidationError
from urllib.parse import urlparse
-from os.path import exists, isdir
+from pathlib import Path
from scrapling.core._types import (
Optional,
Union,
Dict,
Callable,
- Literal,
List,
SelectorWaitStates,
)
@@ -125,9 +124,10 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
self.addons = []
else:
for addon in self.addons:
- if not exists(addon):
+ addon_path = Path(addon)
+ if not addon_path.exists():
raise FileNotFoundError(f"Addon's path not found: {addon}")
- elif not isdir(addon):
+ elif not addon_path.is_dir():
raise ValueError(
f"Addon's path is not a folder, you need to pass a folder of the extracted addon: {addon}"
)
diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py
index 95ed535..e03104c 100644
--- a/scrapling/engines/toolbelt/navigation.py
+++ b/scrapling/engines/toolbelt/navigation.py
@@ -2,17 +2,20 @@
Functions related to files and URLs
"""
-import os
+from pathlib import Path
+from functools import lru_cache
from urllib.parse import urlencode, urlparse
from playwright.async_api import Route as async_Route
from msgspec import Struct, structs, convert, ValidationError
from playwright.sync_api import Route
+from scrapling.core.utils import log
from scrapling.core._types import Dict, Optional, Union, Tuple
-from scrapling.core.utils import log, lru_cache
from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
+__BYPASSES_DIR__ = Path(__file__).parent / "bypasses"
+
class ProxyDict(Struct):
server: str
@@ -129,5 +132,4 @@ def js_bypass_path(filename: str) -> str:
:param filename: The base filename of the JS file.
:return: The full path of the JS file.
"""
- current_directory = os.path.dirname(__file__)
- return os.path.join(current_directory, "bypasses", filename)
+ return str(__BYPASSES_DIR__ / filename)
diff --git a/scrapling/parser.py b/scrapling/parser.py
index 05d2384..459286a 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -1,4 +1,4 @@
-import os
+from pathlib import Path
import re
from inspect import signature
from difflib import SequenceMatcher
@@ -39,6 +39,8 @@ from scrapling.core.storage import (
from scrapling.core.translator import translator_instance
from scrapling.core.utils import clean_spaces, flatten, html_forbidden, is_jsonable, log
+__DEFAULT_DB_FILE__ = str(Path(__file__).parent / "elements_storage.db")
+
class Selector(SelectorsGeneration):
__slots__ = (
@@ -145,9 +147,7 @@ class Selector(SelectorsGeneration):
else:
if not storage_args:
storage_args = {
- "storage_file": os.path.join(
- os.path.dirname(__file__), "elements_storage.db"
- ),
+ "storage_file": __DEFAULT_DB_FILE__,
"url": url,
}