Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

chore: remove dead normalize_url definitions from utils

Rajan Chavada44968c5fe569
diff --git a/crawl4ai/utils.py b/crawl4ai/utils.py
index 89fb782d..279c2770 100644
--- a/crawl4ai/utils.py
+++ b/crawl4ai/utils.py
@@ -2212,25 +2212,6 @@ def fast_format_html(html_string):
return "\n".join(formatted)
-def normalize_url(href, base_url):
- """Normalize URLs to ensure consistent format"""
- from urllib.parse import urljoin, urlparse
-
- # Parse base URL to get components
- parsed_base = urlparse(base_url)
- if not parsed_base.scheme or not parsed_base.netloc:
- raise ValueError(f"Invalid base URL format: {base_url}")
-
- if parsed_base.scheme.lower() not in ["http", "https"]:
- # Handle special protocols
- raise ValueError(f"Invalid base URL format: {base_url}")
- cleaned_href = href.strip()
-
- # Use urljoin to handle all cases
- return urljoin(base_url, cleaned_href)
-
-
-
def normalize_url(
href: str,
@@ -2431,41 +2412,6 @@ def efficient_normalize_url_for_deep_crawl(href, base_url, preserve_https=False,
return normalized
-def normalize_url_tmp(href, base_url):
- """Normalize URLs to ensure consistent format"""
- # Extract protocol and domain from base URL
- try:
- base_parts = base_url.split("/")
- protocol = base_parts[0]
- domain = base_parts[2]
- except IndexError:
- raise ValueError(f"Invalid base URL format: {base_url}")
-
- # Handle special protocols
- special_protocols = {"mailto:", "tel:", "ftp:", "file:", "data:", "javascript:"}
- if any(href.lower().startswith(proto) for proto in special_protocols):
- return href.strip()
-
- # Handle anchor links
- if href.startswith("#"):
- return f"{base_url}{href}"
-
- # Handle protocol-relative URLs
- if href.startswith("//"):
- return f"{protocol}{href}"
-
- # Handle root-relative URLs
- if href.startswith("/"):
- return f"{protocol}//{domain}{href}"
-
- # Handle relative URLs
- if not href.startswith(("http://", "https://")):
- # Remove leading './' if present
- href = href.lstrip("./")
- return f"{protocol}//{domain}/{href}"
-
- return href.strip()
-
def quick_extract_links(html: str, base_url: str) -> Dict[str, List[Dict[str, str]]]:
"""