Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
chore: remove dead normalize_url definitions from utils
Rajan Chavada44968c5fe569
diff --git a/crawl4ai/utils.py b/crawl4ai/utils.pyindex 89fb782d..279c2770 100644--- a/crawl4ai/utils.py+++ b/crawl4ai/utils.py@@ -2212,25 +2212,6 @@ def fast_format_html(html_string):return "\n".join(formatted)-def normalize_url(href, base_url):- """Normalize URLs to ensure consistent format"""- from urllib.parse import urljoin, urlparse-- # Parse base URL to get components- parsed_base = urlparse(base_url)- if not parsed_base.scheme or not parsed_base.netloc:- raise ValueError(f"Invalid base URL format: {base_url}")-- if parsed_base.scheme.lower() not in ["http", "https"]:- # Handle special protocols- raise ValueError(f"Invalid base URL format: {base_url}")- cleaned_href = href.strip()-- # Use urljoin to handle all cases- return urljoin(base_url, cleaned_href)---def normalize_url(href: str,@@ -2431,41 +2412,6 @@ def efficient_normalize_url_for_deep_crawl(href, base_url, preserve_https=False,return normalized-def normalize_url_tmp(href, base_url):- """Normalize URLs to ensure consistent format"""- # Extract protocol and domain from base URL- try:- base_parts = base_url.split("/")- protocol = base_parts[0]- domain = base_parts[2]- except IndexError:- raise ValueError(f"Invalid base URL format: {base_url}")-- # Handle special protocols- special_protocols = {"mailto:", "tel:", "ftp:", "file:", "data:", "javascript:"}- if any(href.lower().startswith(proto) for proto in special_protocols):- return href.strip()-- # Handle anchor links- if href.startswith("#"):- return f"{base_url}{href}"-- # Handle protocol-relative URLs- if href.startswith("//"):- return f"{protocol}{href}"-- # Handle root-relative URLs- if href.startswith("/"):- return f"{protocol}//{domain}{href}"-- # Handle relative URLs- if not href.startswith(("http://", "https://")):- # Remove leading './' if present- href = href.lstrip("./")- return f"{protocol}//{domain}/{href}"-- return href.strip()-def quick_extract_links(html: str, base_url: str) -> Dict[str, List[Dict[str, str]]]:"""