Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
deprecate PruningContentFilter; make PruningContentFilterLXML the documented default
unclecode42ad7ecd49df
diff --git a/crawl4ai/content_filter_strategy.py b/crawl4ai/content_filter_strategy.pyindex 0909be33..04e521db 100644--- a/crawl4ai/content_filter_strategy.py+++ b/crawl4ai/content_filter_strategy.py@@ -1,6 +1,7 @@import inspectimport reimport time+import warningsfrom bs4 import BeautifulSoup, Tagfrom typing import List, Tuple, Dict, Optionalfrom rank_bm25 import BM25Okapi@@ -580,6 +581,19 @@ class PruningContentFilter(RelevantContentFilter):threshold (float): Fixed threshold value (default: 0.48)."""super().__init__(None)+ # Deprecation: the BeautifulSoup engine is superseded by the lxml one.+ # Only warn for direct use of this class, not when PruningContentFilterLXML+ # (a subclass) calls super().__init__.+ if type(self) is PruningContentFilter:+ warnings.warn(+ "PruningContentFilter (BeautifulSoup-based) is deprecated in favor of "+ "PruningContentFilterLXML, an lxml reimplementation that is ~10x faster "+ "with identical output. Switch to PruningContentFilterLXML now. In an "+ "upcoming release PruningContentFilter will become an alias for the lxml "+ "implementation, and PruningContentFilterLXML will be kept as a legacy alias.",+ DeprecationWarning,+ stacklevel=2,+ )self.min_word_threshold = min_word_thresholdself.threshold_type = threshold_typeself.threshold = threshold@@ -1088,3 +1102,17 @@ class LLMContentFilter(RelevantContentFilter):f"{i:<10} {usage.completion_tokens:>12,} "f"{usage.prompt_tokens:>12,} {usage.total_tokens:>12,}")+++def __getattr__(name):+ """Lazily re-export PruningContentFilterLXML from this module.++ The lxml engine lives in ``content_filter_strategy_lxml`` (which imports+ from this module). Exposing it here via PEP 562 lazy attribute access lets+ ``from crawl4ai.content_filter_strategy import PruningContentFilterLXML``+ work without creating an import-time circular dependency.+ """+ if name == "PruningContentFilterLXML":+ from .content_filter_strategy_lxml import PruningContentFilterLXML+ return PruningContentFilterLXML+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")diff --git a/docs/examples/browser_optimization_example.py b/docs/examples/browser_optimization_example.pyindex cf5e1307..25538803 100644--- a/docs/examples/browser_optimization_example.py+++ b/docs/examples/browser_optimization_example.py@@ -34,7 +34,7 @@ async def crawl_sequential(urls: List[str]):# Configure crawl settingscrawl_config = CrawlerRunConfig(markdown_generator=DefaultMarkdownGenerator(- # content_filter=PruningContentFilter(), In case you need fit_markdown+ # content_filter=PruningContentFilterLXML(), In case you need fit_markdown),)@@ -71,7 +71,7 @@ async def crawl_parallel(urls: List[str], max_concurrent: int = 3):crawl_config = CrawlerRunConfig(markdown_generator=DefaultMarkdownGenerator(- # content_filter=PruningContentFilter(), In case you need fit_markdown+ # content_filter=PruningContentFilterLXML(), In case you need fit_markdown),)diff --git a/docs/examples/demo_multi_config_clean.py b/docs/examples/demo_multi_config_clean.pyindex 09df71ce..085ad33c 100644--- a/docs/examples/demo_multi_config_clean.py+++ b/docs/examples/demo_multi_config_clean.py@@ -16,7 +16,7 @@ from crawl4ai import ()from crawl4ai.processors.pdf import PDFContentScrapingStrategyfrom crawl4ai.extraction_strategy import JsonCssExtractionStrategy-from crawl4ai.content_filter_strategy import PruningContentFilter+from crawl4ai.content_filter_strategy import PruningContentFilterLXMLfrom crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator@@ -171,7 +171,7 @@ async def demo_part2_practical_crawling():CrawlerRunConfig(url_matcher=["*/blog/*", "*/article/*", "*python.org*"],markdown_generator=DefaultMarkdownGenerator(- content_filter=PruningContentFilter(threshold=0.48)+ content_filter=PruningContentFilterLXML(threshold=0.48))),diff --git a/docs/examples/docker/demo_docker_api.py b/docs/examples/docker/demo_docker_api.pyindex 0a3d51af..e6c3b58d 100644--- a/docs/examples/docker/demo_docker_api.py+++ b/docs/examples/docker/demo_docker_api.py@@ -362,7 +362,7 @@ async def demo_markdown_pruning(client: httpx.AsyncClient):"type": "DefaultMarkdownGenerator","params": {"content_filter": {- "type": "PruningContentFilter",+ "type": "PruningContentFilterLXML","params": {"threshold": 0.6,"threshold_type": "relative"diff --git a/docs/examples/docker_config_obj.py b/docs/examples/docker_config_obj.pyindex 6ddf157a..2cff64fc 100644--- a/docs/examples/docker_config_obj.py+++ b/docs/examples/docker_config_obj.py@@ -1,4 +1,4 @@-from crawl4ai import BrowserConfig, CrawlerRunConfig, PruningContentFilter, DefaultMarkdownGenerator+from crawl4ai import BrowserConfig, CrawlerRunConfig, PruningContentFilterLXML, DefaultMarkdownGeneratorfrom crawl4ai.deep_crawling.filters import ContentTypeFilter, DomainFilterfrom crawl4ai.deep_crawling.scorers import KeywordRelevanceScorer, PathDepthScorerfrom crawl4ai.cache_context import CacheMode@@ -42,7 +42,7 @@ async def part1_basic_config():crawler_config = CrawlerRunConfig(word_count_threshold=200,markdown_generator=DefaultMarkdownGenerator(- content_filter=PruningContentFilter(threshold=0.5)+ content_filter=PruningContentFilterLXML(threshold=0.5)))@@ -90,7 +90,7 @@ async def part3_complex_structures():config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS,markdown_generator=DefaultMarkdownGenerator(- content_filter=PruningContentFilter()+ content_filter=PruningContentFilterLXML()),deep_crawl_strategy=BFSDeepCrawlStrategy(max_depth=5,@@ -172,7 +172,7 @@ async def part5_direct_api():"type": "DefaultMarkdownGenerator","params": {"content_filter": {- "type": "PruningContentFilter",+ "type": "PruningContentFilterLXML","params": {"threshold": 0.48,"threshold_type": "fixed"diff --git a/docs/examples/extraction_strategies_examples.py b/docs/examples/extraction_strategies_examples.pyindex 5f072f88..6717d89c 100644--- a/docs/examples/extraction_strategies_examples.py+++ b/docs/examples/extraction_strategies_examples.py@@ -17,7 +17,7 @@ from crawl4ai import (JsonCssExtractionStrategy,JsonXPathExtractionStrategy,)-from crawl4ai.content_filter_strategy import PruningContentFilter+from crawl4ai.content_filter_strategy import PruningContentFilterLXMLfrom crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator@@ -29,7 +29,7 @@ async def run_extraction(crawler: AsyncWebCrawler, url: str, strategy, name: strcache_mode=CacheMode.BYPASS,extraction_strategy=strategy,markdown_generator=DefaultMarkdownGenerator(- content_filter=PruningContentFilter() # For fit_markdown support+ content_filter=PruningContentFilterLXML() # For fit_markdown support),)diff --git a/docs/examples/hello_world.py b/docs/examples/hello_world.pyindex 2ba2e852..605c8885 100644--- a/docs/examples/hello_world.py+++ b/docs/examples/hello_world.py@@ -4,7 +4,7 @@ from crawl4ai import (BrowserConfig,CrawlerRunConfig,DefaultMarkdownGenerator,- PruningContentFilter,+ PruningContentFilterLXML,CrawlResult)@@ -17,7 +17,7 @@ async def main():async with AsyncWebCrawler(config=browser_config) as crawler:crawler_config = CrawlerRunConfig(markdown_generator=DefaultMarkdownGenerator(- content_filter=PruningContentFilter()+ content_filter=PruningContentFilterLXML()),)result: CrawlResult = await crawler.arun(diff --git a/docs/examples/hello_world_undetected.py b/docs/examples/hello_world_undetected.pyindex 83ce51ef..ae9d4a34 100644--- a/docs/examples/hello_world_undetected.py+++ b/docs/examples/hello_world_undetected.py@@ -4,7 +4,7 @@ from crawl4ai import (BrowserConfig,CrawlerRunConfig,DefaultMarkdownGenerator,- PruningContentFilter,+ PruningContentFilterLXML,CrawlResult,UndetectedAdapter)@@ -35,7 +35,7 @@ async def main():# Configure the crawlcrawler_config = CrawlerRunConfig(markdown_generator=DefaultMarkdownGenerator(- content_filter=PruningContentFilter()+ content_filter=PruningContentFilterLXML()),capture_console_messages=True, # Enable console capture to test adapter)diff --git a/docs/examples/quickstart.py b/docs/examples/quickstart.pyindex 9992a6bf..fbedcecc 100644--- a/docs/examples/quickstart.py+++ b/docs/examples/quickstart.py@@ -15,7 +15,7 @@ from bs4 import BeautifulSoupfrom pydantic import BaseModel, Fieldfrom crawl4ai import AsyncWebCrawler, CacheMode, BrowserConfig, CrawlerRunConfigfrom crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator-from crawl4ai.content_filter_strategy import PruningContentFilter+from crawl4ai.content_filter_strategy import PruningContentFilterLXMLfrom crawl4ai import (JsonCssExtractionStrategy,LLMExtractionStrategy,@@ -48,7 +48,7 @@ async def clean_content():excluded_tags=["nav", "footer", "aside"],remove_overlay_elements=True,markdown_generator=DefaultMarkdownGenerator(- content_filter=PruningContentFilter(+ content_filter=PruningContentFilterLXML(threshold=0.48, threshold_type="fixed", min_word_threshold=0),options={"ignore_links": True},diff --git a/docs/examples/quickstart_examples_set_1.py b/docs/examples/quickstart_examples_set_1.pyindex 078d1c4a..d9dabc2b 100644--- a/docs/examples/quickstart_examples_set_1.py+++ b/docs/examples/quickstart_examples_set_1.py@@ -10,7 +10,7 @@ from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode, CrawlResultfrom crawl4ai import RoundRobinProxyStrategyfrom crawl4ai import JsonCssExtractionStrategy, LLMExtractionStrategyfrom crawl4ai import LLMConfig-from crawl4ai import PruningContentFilter, BM25ContentFilter+from crawl4ai import PruningContentFilterLXML, BM25ContentFilterfrom crawl4ai import DefaultMarkdownGeneratorfrom crawl4ai import BFSDeepCrawlStrategy, DomainFilter, FilterChainfrom crawl4ai import BrowserConfig@@ -69,7 +69,7 @@ async def demo_fit_markdown():url = "https://en.wikipedia.org/wiki/Python_(programming_language)",config=CrawlerRunConfig(markdown_generator=DefaultMarkdownGenerator(- content_filter=PruningContentFilter()+ content_filter=PruningContentFilterLXML())),)diff --git a/docs/examples/quickstart_examples_set_2.py b/docs/examples/quickstart_examples_set_2.pyindex b12b084c..e78d5701 100644--- a/docs/examples/quickstart_examples_set_2.py+++ b/docs/examples/quickstart_examples_set_2.py@@ -15,7 +15,7 @@ from bs4 import BeautifulSoupfrom pydantic import BaseModel, Fieldfrom crawl4ai import AsyncWebCrawler, CacheMode, BrowserConfig, CrawlerRunConfigfrom crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator-from crawl4ai.content_filter_strategy import PruningContentFilter+from crawl4ai.content_filter_strategy import PruningContentFilterLXMLfrom crawl4ai import (JsonCssExtractionStrategy,LLMExtractionStrategy,@@ -48,7 +48,7 @@ async def clean_content():excluded_tags=["nav", "footer", "aside"],remove_overlay_elements=True,markdown_generator=DefaultMarkdownGenerator(- content_filter=PruningContentFilter(+ content_filter=PruningContentFilterLXML(threshold=0.48, threshold_type="fixed", min_word_threshold=0),options={"ignore_links": True},diff --git a/docs/examples/serp_api_project_11_feb.py b/docs/examples/serp_api_project_11_feb.pyindex 8f9312f7..2241bde0 100644--- a/docs/examples/serp_api_project_11_feb.py+++ b/docs/examples/serp_api_project_11_feb.py@@ -14,7 +14,7 @@ from crawl4ai import (CrawlerHub,CrawlResult,DefaultMarkdownGenerator,- PruningContentFilter,+ PruningContentFilterLXML,)from pathlib import Pathfrom pydantic import BaseModel@@ -35,7 +35,7 @@ async def hello_web():crawler_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS,markdown_generator=DefaultMarkdownGenerator(- content_filter=PruningContentFilter(+ content_filter=PruningContentFilterLXML(threshold=0.48, threshold_type="fixed", min_word_threshold=0)),diff --git a/docs/examples/session_id_example.py b/docs/examples/session_id_example.pyindex e49b7819..4781b705 100644--- a/docs/examples/session_id_example.py+++ b/docs/examples/session_id_example.py@@ -4,7 +4,7 @@ from crawl4ai import (BrowserConfig,CrawlerRunConfig,DefaultMarkdownGenerator,- PruningContentFilter,+ PruningContentFilterLXML,CrawlResult)diff --git a/docs/examples/url_seeder/bbc_sport_research_assistant.py b/docs/examples/url_seeder/bbc_sport_research_assistant.pyindex 51bcf179..8e797d1d 100644--- a/docs/examples/url_seeder/bbc_sport_research_assistant.py+++ b/docs/examples/url_seeder/bbc_sport_research_assistant.py@@ -53,7 +53,7 @@ from crawl4ai import (SeedingConfig,AsyncLogger)-from crawl4ai.content_filter_strategy import PruningContentFilter+from crawl4ai.content_filter_strategy import PruningContentFilterLXMLfrom crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator# LiteLLM for AI communication@@ -343,7 +343,7 @@ async def crawl_selected_urls(urls: List[str], query: str, config: ResearchConfi# Configure markdown generator with content filtermd_generator = DefaultMarkdownGenerator(- content_filter=PruningContentFilter(+ content_filter=PruningContentFilterLXML(threshold=0.48,threshold_type="dynamic",min_word_threshold=10diff --git a/docs/examples/url_seeder/tutorial_url_seeder.md b/docs/examples/url_seeder/tutorial_url_seeder.mdindex 4b9a2201..0cda65bb 100644--- a/docs/examples/url_seeder/tutorial_url_seeder.md+++ b/docs/examples/url_seeder/tutorial_url_seeder.md@@ -73,7 +73,7 @@ from crawl4ai import (SeedingConfig,AsyncLogger)-from crawl4ai.content_filter_strategy import PruningContentFilter+from crawl4ai.content_filter_strategy import PruningContentFilterLXMLfrom crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator# LiteLLM for AI capabilities@@ -357,7 +357,7 @@ async def crawl_selected_urls(# Configure intelligent content extraction# This removes ads, navigation, and other noisemd_generator = DefaultMarkdownGenerator(- content_filter=PruningContentFilter(+ content_filter=PruningContentFilterLXML(threshold=0.48, # Content relevance thresholdthreshold_type="dynamic", # Adapts to page structuremin_word_threshold=10 # Ignore tiny text blocksdiff --git a/docs/examples/url_seeder/url_seeder_quick_demo.py b/docs/examples/url_seeder/url_seeder_quick_demo.pyindex f48c5eec..acc73ef0 100644--- a/docs/examples/url_seeder/url_seeder_quick_demo.py+++ b/docs/examples/url_seeder/url_seeder_quick_demo.py@@ -6,7 +6,7 @@ Note: Uses context manager for automatic cleanup of resources."""import asyncio, osfrom crawl4ai import AsyncUrlSeeder, AsyncWebCrawler, SeedingConfig, CrawlerRunConfig, AsyncLogger, DefaultMarkdownGenerator-from crawl4ai.content_filter_strategy import PruningContentFilter+from crawl4ai.content_filter_strategy import PruningContentFilterLXMLCURRENT_DIR = os.path.dirname(os.path.abspath(__file__))@@ -39,7 +39,7 @@ async def discover_and_crawl():async with AsyncWebCrawler() as crawler:config = CrawlerRunConfig(markdown_generator=DefaultMarkdownGenerator(- content_filter=PruningContentFilter( # Smart filtering!+ content_filter=PruningContentFilterLXML( # Smart filtering!threshold=0.48, # Remove fluffthreshold_type="fixed",),diff --git a/docs/md_v2/advanced/multi-url-crawling.md b/docs/md_v2/advanced/multi-url-crawling.mdindex 2c924eff..994fbfcf 100644--- a/docs/md_v2/advanced/multi-url-crawling.md+++ b/docs/md_v2/advanced/multi-url-crawling.md@@ -418,7 +418,7 @@ When crawling diverse content types, you often need different configurations forfrom crawl4ai import AsyncWebCrawler, CrawlerRunConfig, MatchModefrom crawl4ai.processors.pdf import PDFContentScrapingStrategyfrom crawl4ai.extraction_strategy import JsonCssExtractionStrategy-from crawl4ai.content_filter_strategy import PruningContentFilter+from crawl4ai.content_filter_strategy import PruningContentFilterLXMLfrom crawl4ai.markdown_generation_strategy import DefaultMarkdownGeneratorasync def crawl_mixed_content():@@ -434,7 +434,7 @@ async def crawl_mixed_content():CrawlerRunConfig(url_matcher=["*/blog/*", "*/article/*"],markdown_generator=DefaultMarkdownGenerator(- content_filter=PruningContentFilter(threshold=0.48)+ content_filter=PruningContentFilterLXML(threshold=0.48))),diff --git a/docs/md_v2/advanced/undetected-browser.md b/docs/md_v2/advanced/undetected-browser.mdindex 32b73fc5..c709602f 100644--- a/docs/md_v2/advanced/undetected-browser.md+++ b/docs/md_v2/advanced/undetected-browser.md@@ -216,7 +216,7 @@ async def main():# Configure the crawlcrawler_config = CrawlerRunConfig(markdown_generator=DefaultMarkdownGenerator(- content_filter=PruningContentFilter()+ content_filter=PruningContentFilterLXML()),capture_console_messages=True, # Test adapter console capture)diff --git a/docs/md_v2/api/arun_many.md b/docs/md_v2/api/arun_many.mdindex 740dcc24..96324ae2 100644--- a/docs/md_v2/api/arun_many.md+++ b/docs/md_v2/api/arun_many.md@@ -105,7 +105,7 @@ Instead of using one config for all URLs, provide a list of configs with `url_mafrom crawl4ai import CrawlerRunConfig, MatchModefrom crawl4ai.processors.pdf import PDFContentScrapingStrategyfrom crawl4ai.extraction_strategy import JsonCssExtractionStrategy-from crawl4ai.content_filter_strategy import PruningContentFilter+from crawl4ai.content_filter_strategy import PruningContentFilterLXMLfrom crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator# PDF files - specialized extraction@@ -118,7 +118,7 @@ pdf_config = CrawlerRunConfig(blog_config = CrawlerRunConfig(url_matcher=["*/blog/*", "*/article/*", "*python.org*"],markdown_generator=DefaultMarkdownGenerator(- content_filter=PruningContentFilter(threshold=0.48)+ content_filter=PruningContentFilterLXML(threshold=0.48)))diff --git a/docs/md_v2/api/crawl-result.md b/docs/md_v2/api/crawl-result.mdindex 8890a704..f061836c 100644--- a/docs/md_v2/api/crawl-result.md+++ b/docs/md_v2/api/crawl-result.md@@ -159,7 +159,7 @@ print(result.markdown.raw_markdown[:200])print(result.markdown.fit_markdown)print(result.markdown.fit_html)```-**Important**: "Fit" content (in `fit_markdown`/`fit_html`) exists in result.markdown, only if you used a **filter** (like **PruningContentFilter** or **BM25ContentFilter**) within a `MarkdownGenerationStrategy`.+**Important**: "Fit" content (in `fit_markdown`/`fit_html`) exists in result.markdown, only if you used a **filter** (like **PruningContentFilterLXML** or **BM25ContentFilter**) within a `MarkdownGenerationStrategy`.---@@ -414,7 +414,7 @@ async def handle_result(result: CrawlResult):- `fit_markdown` and `fit_html` - No longer top-level properties in v0.5. Use `result.markdown.fit_markdown` and `result.markdown.fit_html`.2. **Fit Content**- - **`fit_markdown`** and **`fit_html`** appear in MarkdownGenerationResult, only if you used a content filter (like **PruningContentFilter** or **BM25ContentFilter**) inside your **MarkdownGenerationStrategy** or set them directly.+ - **`fit_markdown`** and **`fit_html`** appear in MarkdownGenerationResult, only if you used a content filter (like **PruningContentFilterLXML** or **BM25ContentFilter**) inside your **MarkdownGenerationStrategy** or set them directly.- If no filter is used, they remain `None`.3. **References & Citations**diff --git a/docs/md_v2/complete-sdk-reference.md b/docs/md_v2/complete-sdk-reference.mdindex aa0517b2..938111dc 100644--- a/docs/md_v2/complete-sdk-reference.md+++ b/docs/md_v2/complete-sdk-reference.md@@ -151,15 +151,15 @@ if __name__ == "__main__":## 4. Generating Markdown Output- **`result.markdown`**:- **`result.markdown.fit_markdown`**:- The same content after applying any configured **content filter** (e.g., `PruningContentFilter`).+ The same content after applying any configured **content filter** (e.g., `PruningContentFilterLXML`).### Example: Using a Filter with `DefaultMarkdownGenerator````pythonfrom crawl4ai import AsyncWebCrawler, CrawlerRunConfig-from crawl4ai.content_filter_strategy import PruningContentFilter+from crawl4ai.content_filter_strategy import PruningContentFilterLXMLfrom crawl4ai.markdown_generation_strategy import DefaultMarkdownGeneratormd_generator = DefaultMarkdownGenerator(- content_filter=PruningContentFilter(threshold=0.4, threshold_type="fixed")+ content_filter=PruningContentFilterLXML(threshold=0.4, threshold_type="fixed"))config = CrawlerRunConfig(@@ -172,7 +172,7 @@ async with AsyncWebCrawler() as crawler:print("Raw Markdown length:", len(result.markdown.raw_markdown))print("Fit Markdown length:", len(result.markdown.fit_markdown))```-**Note**: If you do **not** specify a content filter or markdown generator, you’ll typically see only the raw Markdown. `PruningContentFilter` may adds around `50ms` in processing time. We’ll dive deeper into these strategies in a dedicated **Markdown Generation** tutorial.+**Note**: If you do **not** specify a content filter or markdown generator, you’ll typically see only the raw Markdown. `PruningContentFilterLXML` may adds around `50ms` in processing time. We’ll dive deeper into these strategies in a dedicated **Markdown Generation** tutorial.## 5. Simple Data Extraction (CSS-based)```pythonfrom crawl4ai import JsonCssExtractionStrategy@@ -1006,7 +1006,7 @@ Instead of using one config for all URLs, provide a list of configs with `url_mafrom crawl4ai import CrawlerRunConfig, MatchModefrom crawl4ai.processors.pdf import PDFContentScrapingStrategyfrom crawl4ai.extraction_strategy import JsonCssExtractionStrategy-from crawl4ai.content_filter_strategy import PruningContentFilter+from crawl4ai.content_filter_strategy import PruningContentFilterLXMLfrom crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator# PDF files - specialized extraction@@ -1019,7 +1019,7 @@ pdf_config = CrawlerRunConfig(blog_config = CrawlerRunConfig(url_matcher=["*/blog/*", "*/article/*", "*python.org*"],markdown_generator=DefaultMarkdownGenerator(- content_filter=PruningContentFilter(threshold=0.48)+ content_filter=PruningContentFilterLXML(threshold=0.48)))@@ -1171,7 +1171,7 @@ print(result.markdown.raw_markdown[:200])print(result.markdown.fit_markdown)print(result.markdown.fit_html)```-**Important**: "Fit" content (in `fit_markdown`/`fit_html`) exists in result.markdown, only if you used a **filter** (like **PruningContentFilter** or **BM25ContentFilter**) within a `MarkdownGenerationStrategy`.+**Important**: "Fit" content (in `fit_markdown`/`fit_html`) exists in result.markdown, only if you used a **filter** (like **PruningContentFilterLXML** or **BM25ContentFilter**) within a `MarkdownGenerationStrategy`.## 4. Media & Links### 4.1 **`media`** *(Dict[str, List[Dict]])***What**: Contains info about discovered images, videos, or audio. Typically keys: `"images"`, `"videos"`, `"audios"`.@@ -1363,7 +1363,7 @@ async def handle_result(result: CrawlResult):- `markdown_v2` - Removed in v0.5. Accessing it raises `AttributeError`. Use `markdown`.- `fit_markdown` and `fit_html` - Removed as top-level `CrawlResult` properties in v0.5. Use `result.markdown.fit_markdown` and `result.markdown.fit_html`.2. **Fit Content**- - **`fit_markdown`** and **`fit_html`** appear in MarkdownGenerationResult, only if you used a content filter (like **PruningContentFilter** or **BM25ContentFilter**) inside your **MarkdownGenerationStrategy** or set them directly.+ - **`fit_markdown`** and **`fit_html`** appear in MarkdownGenerationResult, only if you used a content filter (like **PruningContentFilterLXML** or **BM25ContentFilter**) inside your **MarkdownGenerationStrategy** or set them directly.- If no filter is used, they remain `None`.3. **References & Citations**- If you enable link citations in your `DefaultMarkdownGenerator` (`options={"citations": True}`), you’ll see `markdown_with_citations` plus a **`references_markdown`** block. This helps large language models or academic-like referencing.@@ -2038,7 +2038,7 @@ The `arun()` method returns a `CrawlResult` object with several useful propertie```pythonconfig = CrawlerRunConfig(markdown_generator=DefaultMarkdownGenerator(- content_filter=PruningContentFilter(threshold=0.6),+ content_filter=PruningContentFilterLXML(threshold=0.6),options={"ignore_links": True}))@@ -2293,12 +2293,12 @@ config = CrawlerRunConfig(markdown_generator=md_generator)- **`bm25_threshold`**: Raise it to keep fewer blocks; lower it to keep more.- **`use_stemming`** *(default `True`)*: Whether to apply stemming to the query and content.- **`language (str)`**: Language for stemming (default: 'english').-### 5.2 PruningContentFilter-If you **don’t** have a specific query, or if you just want a robust “junk remover,” use `PruningContentFilter`. It analyzes text density, link density, HTML structure, and known patterns (like “nav,” “footer”) to systematically prune extraneous or repetitive sections.+### 5.2 PruningContentFilterLXML+If you **don’t** have a specific query, or if you just want a robust “junk remover,” use `PruningContentFilterLXML`. It analyzes text density, link density, HTML structure, and known patterns (like “nav,” “footer”) to systematically prune extraneous or repetitive sections.```python-from crawl4ai.content_filter_strategy import PruningContentFilter+from crawl4ai.content_filter_strategy import PruningContentFilterLXML-prune_filter = PruningContentFilter(+prune_filter = PruningContentFilterLXML(threshold=0.5,threshold_type="fixed", # or "dynamic"min_word_threshold=50@@ -2383,12 +2383,12 @@ When a content filter is active, the library produces two forms of markdown insiimport asynciofrom crawl4ai import AsyncWebCrawler, CrawlerRunConfigfrom crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator-from crawl4ai.content_filter_strategy import PruningContentFilter+from crawl4ai.content_filter_strategy import PruningContentFilterLXMLasync def main():config = CrawlerRunConfig(markdown_generator=DefaultMarkdownGenerator(- content_filter=PruningContentFilter(threshold=0.6),+ content_filter=PruningContentFilterLXML(threshold=0.6),options={"ignore_links": True}))@@ -2424,14 +2424,14 @@ print("FIT:\n", md_obj.fit_markdown)- Or feed `fit_markdown` into a vector database to reduce token usage.- `references_markdown` can help you keep track of link provenance.## 8. Combining Filters (BM25 + Pruning) in Two Passes-You might want to **prune out** noisy boilerplate first (with `PruningContentFilter`), and then **rank what’s left** against a user query (with `BM25ContentFilter`). You don’t have to crawl the page twice. Instead:-1. **First pass**: Apply `PruningContentFilter` directly to the raw HTML from `result.html` (the crawler’s downloaded HTML).+You might want to **prune out** noisy boilerplate first (with `PruningContentFilterLXML`), and then **rank what’s left** against a user query (with `BM25ContentFilter`). You don’t have to crawl the page twice. Instead:+1. **First pass**: Apply `PruningContentFilterLXML` directly to the raw HTML from `result.html` (the crawler’s downloaded HTML).2. **Second pass**: Take the pruned HTML (or text) from step 1, and feed it into `BM25ContentFilter`, focusing on a user query.### Two-Pass Example```pythonimport asynciofrom crawl4ai import AsyncWebCrawler, CrawlerRunConfig-from crawl4ai.content_filter_strategy import PruningContentFilter, BM25ContentFilter+from crawl4ai.content_filter_strategy import PruningContentFilterLXML, BM25ContentFilterfrom bs4 import BeautifulSoupasync def main():@@ -2450,8 +2450,8 @@ async def main():raw_html = result.html- # 2. First pass: PruningContentFilter on raw HTML- pruning_filter = PruningContentFilter(threshold=0.5, min_word_threshold=50)+ # 2. First pass: PruningContentFilterLXML on raw HTML+ pruning_filter = PruningContentFilterLXML(threshold=0.5, min_word_threshold=50)# filter_content returns a list of "text chunks" or cleaned HTML sectionspruned_chunks = pruning_filter.filter_content(raw_html)@@ -2518,17 +2518,24 @@ In **`CrawlerRunConfig`**, you can specify a **`content_filter`** to shape how c- **`result.markdown.fit_markdown`** (filtered or “fit” version)- **`result.markdown.fit_html`** (the corresponding HTML snippet that produced `fit_markdown`)### 1.2 Common Filters-## 2. PruningContentFilter+## 2. PruningContentFilterLXML++!!! warning "Deprecation"+ Use `PruningContentFilterLXML` (lxml engine, **~10× faster**, identical output). The legacy+ `PruningContentFilter` (BeautifulSoup) still works but emits a `DeprecationWarning`. In an+ upcoming release `PruningContentFilter` will become an alias for the lxml engine, with+ `PruningContentFilterLXML` kept as a legacy alias.+### 2.1 Usage Example```pythonimport asynciofrom crawl4ai import AsyncWebCrawler, CrawlerRunConfig-from crawl4ai.content_filter_strategy import PruningContentFilter+from crawl4ai.content_filter_strategy import PruningContentFilterLXMLfrom crawl4ai.markdown_generation_strategy import DefaultMarkdownGeneratorasync def main():# Step 1: Create a pruning filter- prune_filter = PruningContentFilter(+ prune_filter = PruningContentFilterLXML(# Lower → more content retained, higher → more content prunedthreshold=0.45,# "fixed" or "dynamic"@@ -2623,7 +2630,7 @@ If the content filter is **BM25**, you might see additional logic or references## 5. Code Patterns Recap### 5.1 Pruning```python-prune_filter = PruningContentFilter(+prune_filter = PruningContentFilterLXML(threshold=0.5,threshold_type="fixed",min_word_threshold=10@@ -2647,7 +2654,7 @@ config = CrawlerRunConfig(excluded_tags=["nav", "footer", "header"],exclude_external_links=True,markdown_generator=DefaultMarkdownGenerator(- content_filter=PruningContentFilter(threshold=0.5)+ content_filter=PruningContentFilterLXML(threshold=0.5)))```@@ -4698,7 +4705,7 @@ To keep context continuous across chunks, we can overlap them. E.g., `overlap_ra## 7. Input FormatBy default, **LLMExtractionStrategy** uses `input_format="markdown"`, meaning the **crawler’s final markdown** is fed to the LLM. You can change to:- **`html`**: The cleaned HTML or raw HTML (depending on your crawler config) goes into the LLM.-- **`fit_markdown`**: If you used, for instance, `PruningContentFilter`, the “fit” version of the markdown is used. This can drastically reduce tokens if you trust the filter.+- **`fit_markdown`**: If you used, for instance, `PruningContentFilterLXML`, the “fit” version of the markdown is used. This can drastically reduce tokens if you trust the filter.- **`markdown`**: Standard markdown output from the crawler’s `markdown_generator`.This setting is crucial: if the LLM instructions rely on HTML tags, pick `"html"`. If you prefer a text-based approach, pick `"markdown"`.```pythondiff --git a/docs/md_v2/core/crawler-result.md b/docs/md_v2/core/crawler-result.mdindex a531e043..79f72318 100644--- a/docs/md_v2/core/crawler-result.md+++ b/docs/md_v2/core/crawler-result.md@@ -141,7 +141,7 @@ print(md_res.markdown_with_citations)print(md_res.references_markdown)```-**Note**: If you use a filter like `PruningContentFilter`, you’ll get `fit_markdown` and `fit_html` as well.+**Note**: If you use a filter like `PruningContentFilterLXML`, you’ll get `fit_markdown` and `fit_html` as well.---@@ -336,7 +336,7 @@ else:## 7. Next Steps- **Markdown Generation**: Dive deeper into how to configure `DefaultMarkdownGenerator` and various filters.-- **Content Filtering**: Learn how to use `BM25ContentFilter` and `PruningContentFilter`.+- **Content Filtering**: Learn how to use `BM25ContentFilter` and `PruningContentFilterLXML`.- **Session & Hooks**: If you want to manipulate the page or preserve state across multiple `arun()` calls, see the hooking or session docs.- **LLM Extraction**: For complex or unstructured content requiring AI-driven parsing, check the LLM-based strategies doc.diff --git a/docs/md_v2/core/fit-markdown.md b/docs/md_v2/core/fit-markdown.mdindex 1684a779..149496ca 100644--- a/docs/md_v2/core/fit-markdown.md+++ b/docs/md_v2/core/fit-markdown.md@@ -1,5 +1,22 @@# Fit Markdown with Pruning & BM25+!!! warning "Deprecation: `PruningContentFilter` → `PruningContentFilterLXML`"+ The original `PruningContentFilter` (BeautifulSoup-based) is **deprecated** in favor of+ **`PruningContentFilterLXML`**, an lxml reimplementation that is **~10× faster** on large+ pages with **identical output**. It is already the default for the `crwl` CLI and the+ Docker server's fit-markdown endpoint.++ **Migrate** by swapping the class name (every parameter is unchanged):++ ```python+ from crawl4ai import PruningContentFilterLXML+ prune_filter = PruningContentFilterLXML(threshold=0.48, threshold_type="dynamic")+ ```++ Instantiating the old `PruningContentFilter` now emits a `DeprecationWarning`. **Roadmap:**+ in an upcoming release `PruningContentFilter` will become an alias for the lxml engine, and+ `PruningContentFilterLXML` will be kept as a legacy alias.+**Fit Markdown** is a specialized **filtered** version of your page’s markdown, focusing on the most relevant content. By default, Crawl4AI converts the entire HTML into a broad **raw_markdown**. With fit markdown, we apply a **content filter** algorithm (e.g., **Pruning** or **BM25**) to remove or rank low-value sections—such as repetitive sidebars, shallow text blocks, or irrelevancies—leaving a concise textual “core.”---@@ -17,12 +34,12 @@ In **`CrawlerRunConfig`**, you can specify a **`content_filter`** to shape how c### 1.2 Common Filters-1. **PruningContentFilter** – Scores each node by text density, link density, and tag importance, discarding those below a threshold.+1. **PruningContentFilterLXML** – Scores each node by text density, link density, and tag importance, discarding those below a threshold.2. **BM25ContentFilter** – Focuses on textual relevance using BM25 ranking, especially useful if you have a specific user query (e.g., “machine learning” or “food nutrition”).----## 2. PruningContentFilter+## 2. PruningContentFilterLXML**Pruning** discards less relevant nodes based on **text density, link density, and tag importance**. It’s a heuristic-based approach—if certain sections appear too “thin” or too “spammy,” they’re pruned.@@ -31,12 +48,12 @@ In **`CrawlerRunConfig`**, you can specify a **`content_filter`** to shape how c```pythonimport asynciofrom crawl4ai import AsyncWebCrawler, CrawlerRunConfig-from crawl4ai.content_filter_strategy import PruningContentFilter+from crawl4ai.content_filter_strategy import PruningContentFilterLXMLfrom crawl4ai.markdown_generation_strategy import DefaultMarkdownGeneratorasync def main():# Step 1: Create a pruning filter- prune_filter = PruningContentFilter(+ prune_filter = PruningContentFilterLXML(# Lower → more content retained, higher → more content prunedthreshold=0.45,# "fixed" or "dynamic"@@ -159,7 +176,7 @@ If the content filter is **BM25**, you might see additional logic or references### 5.1 Pruning```python-prune_filter = PruningContentFilter(+prune_filter = PruningContentFilterLXML(threshold=0.5,threshold_type="fixed",min_word_threshold=10@@ -191,7 +208,7 @@ config = CrawlerRunConfig(excluded_tags=["nav", "footer", "header"],exclude_external_links=True,markdown_generator=DefaultMarkdownGenerator(- content_filter=PruningContentFilter(threshold=0.5)+ content_filter=PruningContentFilterLXML(threshold=0.5)))```@@ -235,7 +252,7 @@ class MyCustomFilter(RelevantContentFilter):- **AI Pipelines**: Filter out boilerplate so LLM-based extraction or summarization runs on denser text.**Key Points**:-- **PruningContentFilter**: Great if you just want the “meatiest” text without a user query.+- **PruningContentFilterLXML**: Great if you just want the “meatiest” text without a user query.- **BM25ContentFilter**: Perfect for query-based extraction or searching.- Combine with **`excluded_tags`, `exclude_external_links`, `word_count_threshold`** to refine your final “fit” text.- Fit markdown ends up in **`result.markdown.fit_markdown`**; eventually **`result.markdown.fit_markdown`** in future versions.diff --git a/docs/md_v2/core/markdown-generation.md b/docs/md_v2/core/markdown-generation.mdindex af9b35b5..6d231a21 100644--- a/docs/md_v2/core/markdown-generation.md+++ b/docs/md_v2/core/markdown-generation.md@@ -205,14 +205,20 @@ config = CrawlerRunConfig(markdown_generator=md_generator)**No query provided?** BM25 tries to glean a context from page metadata, or you can simply treat it as a scorched-earth approach that discards text with low generic score. Realistically, you want to supply a query for best results.-### 5.2 PruningContentFilter+### 5.2 PruningContentFilterLXML-If you **don’t** have a specific query, or if you just want a robust “junk remover,” use `PruningContentFilter`. It analyzes text density, link density, HTML structure, and known patterns (like “nav,” “footer”) to systematically prune extraneous or repetitive sections.+!!! warning "Use `PruningContentFilterLXML` (the old `PruningContentFilter` is deprecated)"+ `PruningContentFilterLXML` is the lxml-based engine: **~10× faster** on large pages with+ **identical output**. The legacy `PruningContentFilter` still works but emits a+ `DeprecationWarning`; in an upcoming release `PruningContentFilter` will alias the lxml+ engine and `PruningContentFilterLXML` will remain as a legacy alias.++If you **don’t** have a specific query, or if you just want a robust “junk remover,” use `PruningContentFilterLXML`. It analyzes text density, link density, HTML structure, and known patterns (like “nav,” “footer”) to systematically prune extraneous or repetitive sections.```python-from crawl4ai.content_filter_strategy import PruningContentFilter+from crawl4ai.content_filter_strategy import PruningContentFilterLXML-prune_filter = PruningContentFilter(+prune_filter = PruningContentFilterLXML(threshold=0.5,threshold_type="fixed", # or "dynamic"min_word_threshold=50@@ -225,7 +231,7 @@ prune_filter = PruningContentFilter(- `"dynamic"`: The filter adjusts threshold in a data-driven manner.- **`min_word_threshold`**: Discard blocks under N words as likely too short or unhelpful.-**When to Use PruningContentFilter**+**When to Use PruningContentFilterLXML**- You want a broad cleanup without a user query.- The page has lots of repeated sidebars, footers, or disclaimers that hamper text extraction.@@ -320,12 +326,12 @@ When a content filter is active, the library produces two forms of markdown insiimport asynciofrom crawl4ai import AsyncWebCrawler, CrawlerRunConfigfrom crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator-from crawl4ai.content_filter_strategy import PruningContentFilter+from crawl4ai.content_filter_strategy import PruningContentFilterLXMLasync def main():config = CrawlerRunConfig(markdown_generator=DefaultMarkdownGenerator(- content_filter=PruningContentFilter(threshold=0.6),+ content_filter=PruningContentFilterLXML(threshold=0.6),options={"ignore_links": True}))@@ -379,9 +385,9 @@ Below is a **revised section** under “Combining Filters (BM25 + Pruning)” th## 8. Combining Filters (BM25 + Pruning) in Two Passes-You might want to **prune out** noisy boilerplate first (with `PruningContentFilter`), and then **rank what’s left** against a user query (with `BM25ContentFilter`). You don’t have to crawl the page twice. Instead:+You might want to **prune out** noisy boilerplate first (with `PruningContentFilterLXML`), and then **rank what’s left** against a user query (with `BM25ContentFilter`). You don’t have to crawl the page twice. Instead:-1. **First pass**: Apply `PruningContentFilter` directly to the raw HTML from `result.html` (the crawler’s downloaded HTML).+1. **First pass**: Apply `PruningContentFilterLXML` directly to the raw HTML from `result.html` (the crawler’s downloaded HTML).2. **Second pass**: Take the pruned HTML (or text) from step 1, and feed it into `BM25ContentFilter`, focusing on a user query.### Two-Pass Example@@ -389,7 +395,7 @@ You might want to **prune out** noisy boilerplate first (with `PruningContentFil```pythonimport asynciofrom crawl4ai import AsyncWebCrawler, CrawlerRunConfig-from crawl4ai.content_filter_strategy import PruningContentFilter, BM25ContentFilter+from crawl4ai.content_filter_strategy import PruningContentFilterLXML, BM25ContentFilterfrom bs4 import BeautifulSoupasync def main():@@ -408,8 +414,8 @@ async def main():raw_html = result.html- # 2. First pass: PruningContentFilter on raw HTML- pruning_filter = PruningContentFilter(threshold=0.5, min_word_threshold=50)+ # 2. First pass: PruningContentFilterLXML on raw HTML+ pruning_filter = PruningContentFilterLXML(threshold=0.5, min_word_threshold=50)# filter_content returns a list of "text chunks" or cleaned HTML sectionspruned_chunks = pruning_filter.filter_content(raw_html)@@ -449,7 +455,7 @@ if __name__ == "__main__":### What’s Happening?1. **Raw HTML**: We crawl once and store the raw HTML in `result.html`.-2. **PruningContentFilter**: Takes HTML + optional parameters. It extracts blocks of text or partial HTML, removing headings/sections deemed “noise.” It returns a **list of text chunks**.+2. **PruningContentFilterLXML**: Takes HTML + optional parameters. It extracts blocks of text or partial HTML, removing headings/sections deemed “noise.” It returns a **list of text chunks**.3. **Combine or Transform**: We join these pruned chunks back into a single HTML-like string. (Alternatively, you could store them in a list for further logic—whatever suits your pipeline.)4. **BM25ContentFilter**: We feed the pruned string into `BM25ContentFilter` with a user query. This second pass further narrows the content to chunks relevant to “machine learning.”@@ -495,7 +501,7 @@ In this **Markdown Generation Basics** tutorial, you learned to:- Configure the **DefaultMarkdownGenerator** with HTML-to-text options.- Select different HTML sources using the `content_source` parameter.-- Use **BM25ContentFilter** for query-specific extraction or **PruningContentFilter** for general noise removal.+- Use **BM25ContentFilter** for query-specific extraction or **PruningContentFilterLXML** for general noise removal.- Distinguish between raw and filtered markdown (`fit_markdown`).- Leverage the `MarkdownGenerationResult` object to handle different forms of output (citations, references, etc.).diff --git a/docs/md_v2/core/quickstart.md b/docs/md_v2/core/quickstart.mdindex 78209efd..063c93c6 100644--- a/docs/md_v2/core/quickstart.md+++ b/docs/md_v2/core/quickstart.md@@ -92,17 +92,17 @@ By default, Crawl4AI automatically generates Markdown from each crawled page. Ho- **`result.markdown`**:The direct HTML-to-Markdown conversion.- **`result.markdown.fit_markdown`**:- The same content after applying any configured **content filter** (e.g., `PruningContentFilter`).+ The same content after applying any configured **content filter** (e.g., `PruningContentFilterLXML`).### Example: Using a Filter with `DefaultMarkdownGenerator````pythonfrom crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode-from crawl4ai.content_filter_strategy import PruningContentFilter+from crawl4ai.content_filter_strategy import PruningContentFilterLXMLfrom crawl4ai.markdown_generation_strategy import DefaultMarkdownGeneratormd_generator = DefaultMarkdownGenerator(- content_filter=PruningContentFilter(threshold=0.4, threshold_type="fixed")+ content_filter=PruningContentFilterLXML(threshold=0.4, threshold_type="fixed"))config = CrawlerRunConfig(@@ -116,7 +116,7 @@ async with AsyncWebCrawler() as crawler:print("Fit Markdown length:", len(result.markdown.fit_markdown))```-**Note**: If you do **not** specify a content filter or markdown generator, you’ll typically see only the raw Markdown. `PruningContentFilter` may adds around `50ms` in processing time. We’ll dive deeper into these strategies in a dedicated **Markdown Generation** tutorial.+**Note**: If you do **not** specify a content filter or markdown generator, you’ll typically see only the raw Markdown. `PruningContentFilterLXML` may adds around `50ms` in processing time. We’ll dive deeper into these strategies in a dedicated **Markdown Generation** tutorial.---diff --git a/docs/md_v2/core/simple-crawling.md b/docs/md_v2/core/simple-crawling.mdindex dd31bc97..c9b78533 100644--- a/docs/md_v2/core/simple-crawling.md+++ b/docs/md_v2/core/simple-crawling.md@@ -33,7 +33,7 @@ The `arun()` method returns a `CrawlResult` object with several useful propertie```pythonconfig = CrawlerRunConfig(markdown_generator=DefaultMarkdownGenerator(- content_filter=PruningContentFilter(threshold=0.6),+ content_filter=PruningContentFilterLXML(threshold=0.6),options={"ignore_links": True}))diff --git a/docs/md_v2/extraction/llm-strategies.md b/docs/md_v2/extraction/llm-strategies.mdindex cba4d6e4..197d2ba2 100644--- a/docs/md_v2/extraction/llm-strategies.md+++ b/docs/md_v2/extraction/llm-strategies.md@@ -175,7 +175,7 @@ By chunking, you can potentially process multiple chunks in parallel (dependingBy default, **LLMExtractionStrategy** uses `input_format="markdown"`, meaning the **crawler’s final markdown** is fed to the LLM. You can change to:- **`html`**: The cleaned HTML or raw HTML (depending on your crawler config) goes into the LLM.-- **`fit_markdown`**: If you used, for instance, `PruningContentFilter`, the “fit” version of the markdown is used. This can drastically reduce tokens if you trust the filter.+- **`fit_markdown`**: If you used, for instance, `PruningContentFilterLXML`, the “fit” version of the markdown is used. This can drastically reduce tokens if you trust the filter.- **`markdown`**: Standard markdown output from the crawler’s `markdown_generator`.This setting is crucial: if the LLM instructions rely on HTML tags, pick `"html"`. If you prefer a text-based approach, pick `"markdown"`.diff --git a/tests/async/test_content_filter_prune_lxml.py b/tests/async/test_content_filter_prune_lxml.pyindex afc1226f..889387eb 100644--- a/tests/async/test_content_filter_prune_lxml.py+++ b/tests/async/test_content_filter_prune_lxml.py@@ -153,6 +153,7 @@ class TestPruningContentFilterLXMLBehavior:# --------------------------------------------------------------------------- ## Fidelity: new output must match the original bs4 implementation# --------------------------------------------------------------------------- #[email protected]("ignore::DeprecationWarning") # deliberately exercises the deprecated bs4 filterclass TestPruningEquivalence:@pytest.mark.parametrize("name", list(EQUIVALENCE_CASES.keys()))@pytest.mark.parametrize("config", CONFIGS)@@ -182,6 +183,7 @@ class TestPruningEquivalence:# --------------------------------------------------------------------------- ## Performance: new must be substantially faster on a non-trivial page# --------------------------------------------------------------------------- #[email protected]("ignore::DeprecationWarning") # compares against the deprecated bs4 filterdef test_performance_faster_than_original():# ~600 repeated cards -> a few hundred KB of HTMLhtml = "<body>" + "".join(