Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

deprecate PruningContentFilter; make PruningContentFilterLXML the documented default

unclecode42ad7ecd49df
diff --git a/crawl4ai/content_filter_strategy.py b/crawl4ai/content_filter_strategy.py
index 0909be33..04e521db 100644
--- a/crawl4ai/content_filter_strategy.py
+++ b/crawl4ai/content_filter_strategy.py
@@ -1,6 +1,7 @@
import inspect
import re
import time
+import warnings
from bs4 import BeautifulSoup, Tag
from typing import List, Tuple, Dict, Optional
from rank_bm25 import BM25Okapi
@@ -580,6 +581,19 @@ class PruningContentFilter(RelevantContentFilter):
threshold (float): Fixed threshold value (default: 0.48).
"""
super().__init__(None)
+ # Deprecation: the BeautifulSoup engine is superseded by the lxml one.
+ # Only warn for direct use of this class, not when PruningContentFilterLXML
+ # (a subclass) calls super().__init__.
+ if type(self) is PruningContentFilter:
+ warnings.warn(
+ "PruningContentFilter (BeautifulSoup-based) is deprecated in favor of "
+ "PruningContentFilterLXML, an lxml reimplementation that is ~10x faster "
+ "with identical output. Switch to PruningContentFilterLXML now. In an "
+ "upcoming release PruningContentFilter will become an alias for the lxml "
+ "implementation, and PruningContentFilterLXML will be kept as a legacy alias.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
self.min_word_threshold = min_word_threshold
self.threshold_type = threshold_type
self.threshold = threshold
@@ -1088,3 +1102,17 @@ class LLMContentFilter(RelevantContentFilter):
f"{i:<10} {usage.completion_tokens:>12,} "
f"{usage.prompt_tokens:>12,} {usage.total_tokens:>12,}"
)
+
+
+def __getattr__(name):
+ """Lazily re-export PruningContentFilterLXML from this module.
+
+ The lxml engine lives in ``content_filter_strategy_lxml`` (which imports
+ from this module). Exposing it here via PEP 562 lazy attribute access lets
+ ``from crawl4ai.content_filter_strategy import PruningContentFilterLXML``
+ work without creating an import-time circular dependency.
+ """
+ if name == "PruningContentFilterLXML":
+ from .content_filter_strategy_lxml import PruningContentFilterLXML
+ return PruningContentFilterLXML
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
diff --git a/docs/examples/browser_optimization_example.py b/docs/examples/browser_optimization_example.py
index cf5e1307..25538803 100644
--- a/docs/examples/browser_optimization_example.py
+++ b/docs/examples/browser_optimization_example.py
@@ -34,7 +34,7 @@ async def crawl_sequential(urls: List[str]):
# Configure crawl settings
crawl_config = CrawlerRunConfig(
markdown_generator=DefaultMarkdownGenerator(
- # content_filter=PruningContentFilter(), In case you need fit_markdown
+ # content_filter=PruningContentFilterLXML(), In case you need fit_markdown
),
)
@@ -71,7 +71,7 @@ async def crawl_parallel(urls: List[str], max_concurrent: int = 3):
crawl_config = CrawlerRunConfig(
markdown_generator=DefaultMarkdownGenerator(
- # content_filter=PruningContentFilter(), In case you need fit_markdown
+ # content_filter=PruningContentFilterLXML(), In case you need fit_markdown
),
)
diff --git a/docs/examples/demo_multi_config_clean.py b/docs/examples/demo_multi_config_clean.py
index 09df71ce..085ad33c 100644
--- a/docs/examples/demo_multi_config_clean.py
+++ b/docs/examples/demo_multi_config_clean.py
@@ -16,7 +16,7 @@ from crawl4ai import (
)
from crawl4ai.processors.pdf import PDFContentScrapingStrategy
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
@@ -171,7 +171,7 @@ async def demo_part2_practical_crawling():
CrawlerRunConfig(
url_matcher=["*/blog/*", "*/article/*", "*python.org*"],
markdown_generator=DefaultMarkdownGenerator(
- content_filter=PruningContentFilter(threshold=0.48)
+ content_filter=PruningContentFilterLXML(threshold=0.48)
)
),
diff --git a/docs/examples/docker/demo_docker_api.py b/docs/examples/docker/demo_docker_api.py
index 0a3d51af..e6c3b58d 100644
--- a/docs/examples/docker/demo_docker_api.py
+++ b/docs/examples/docker/demo_docker_api.py
@@ -362,7 +362,7 @@ async def demo_markdown_pruning(client: httpx.AsyncClient):
"type": "DefaultMarkdownGenerator",
"params": {
"content_filter": {
- "type": "PruningContentFilter",
+ "type": "PruningContentFilterLXML",
"params": {
"threshold": 0.6,
"threshold_type": "relative"
diff --git a/docs/examples/docker_config_obj.py b/docs/examples/docker_config_obj.py
index 6ddf157a..2cff64fc 100644
--- a/docs/examples/docker_config_obj.py
+++ b/docs/examples/docker_config_obj.py
@@ -1,4 +1,4 @@
-from crawl4ai import BrowserConfig, CrawlerRunConfig, PruningContentFilter, DefaultMarkdownGenerator
+from crawl4ai import BrowserConfig, CrawlerRunConfig, PruningContentFilterLXML, DefaultMarkdownGenerator
from crawl4ai.deep_crawling.filters import ContentTypeFilter, DomainFilter
from crawl4ai.deep_crawling.scorers import KeywordRelevanceScorer, PathDepthScorer
from crawl4ai.cache_context import CacheMode
@@ -42,7 +42,7 @@ async def part1_basic_config():
crawler_config = CrawlerRunConfig(
word_count_threshold=200,
markdown_generator=DefaultMarkdownGenerator(
- content_filter=PruningContentFilter(threshold=0.5)
+ content_filter=PruningContentFilterLXML(threshold=0.5)
)
)
@@ -90,7 +90,7 @@ async def part3_complex_structures():
config = CrawlerRunConfig(
cache_mode=CacheMode.BYPASS,
markdown_generator=DefaultMarkdownGenerator(
- content_filter=PruningContentFilter()
+ content_filter=PruningContentFilterLXML()
),
deep_crawl_strategy=BFSDeepCrawlStrategy(
max_depth=5,
@@ -172,7 +172,7 @@ async def part5_direct_api():
"type": "DefaultMarkdownGenerator",
"params": {
"content_filter": {
- "type": "PruningContentFilter",
+ "type": "PruningContentFilterLXML",
"params": {
"threshold": 0.48,
"threshold_type": "fixed"
diff --git a/docs/examples/extraction_strategies_examples.py b/docs/examples/extraction_strategies_examples.py
index 5f072f88..6717d89c 100644
--- a/docs/examples/extraction_strategies_examples.py
+++ b/docs/examples/extraction_strategies_examples.py
@@ -17,7 +17,7 @@ from crawl4ai import (
JsonCssExtractionStrategy,
JsonXPathExtractionStrategy,
)
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
@@ -29,7 +29,7 @@ async def run_extraction(crawler: AsyncWebCrawler, url: str, strategy, name: str
cache_mode=CacheMode.BYPASS,
extraction_strategy=strategy,
markdown_generator=DefaultMarkdownGenerator(
- content_filter=PruningContentFilter() # For fit_markdown support
+ content_filter=PruningContentFilterLXML() # For fit_markdown support
),
)
diff --git a/docs/examples/hello_world.py b/docs/examples/hello_world.py
index 2ba2e852..605c8885 100644
--- a/docs/examples/hello_world.py
+++ b/docs/examples/hello_world.py
@@ -4,7 +4,7 @@ from crawl4ai import (
BrowserConfig,
CrawlerRunConfig,
DefaultMarkdownGenerator,
- PruningContentFilter,
+ PruningContentFilterLXML,
CrawlResult
)
@@ -17,7 +17,7 @@ async def main():
async with AsyncWebCrawler(config=browser_config) as crawler:
crawler_config = CrawlerRunConfig(
markdown_generator=DefaultMarkdownGenerator(
- content_filter=PruningContentFilter()
+ content_filter=PruningContentFilterLXML()
),
)
result: CrawlResult = await crawler.arun(
diff --git a/docs/examples/hello_world_undetected.py b/docs/examples/hello_world_undetected.py
index 83ce51ef..ae9d4a34 100644
--- a/docs/examples/hello_world_undetected.py
+++ b/docs/examples/hello_world_undetected.py
@@ -4,7 +4,7 @@ from crawl4ai import (
BrowserConfig,
CrawlerRunConfig,
DefaultMarkdownGenerator,
- PruningContentFilter,
+ PruningContentFilterLXML,
CrawlResult,
UndetectedAdapter
)
@@ -35,7 +35,7 @@ async def main():
# Configure the crawl
crawler_config = CrawlerRunConfig(
markdown_generator=DefaultMarkdownGenerator(
- content_filter=PruningContentFilter()
+ content_filter=PruningContentFilterLXML()
),
capture_console_messages=True, # Enable console capture to test adapter
)
diff --git a/docs/examples/quickstart.py b/docs/examples/quickstart.py
index 9992a6bf..fbedcecc 100644
--- a/docs/examples/quickstart.py
+++ b/docs/examples/quickstart.py
@@ -15,7 +15,7 @@ from bs4 import BeautifulSoup
from pydantic import BaseModel, Field
from crawl4ai import AsyncWebCrawler, CacheMode, BrowserConfig, CrawlerRunConfig
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
from crawl4ai import (
JsonCssExtractionStrategy,
LLMExtractionStrategy,
@@ -48,7 +48,7 @@ async def clean_content():
excluded_tags=["nav", "footer", "aside"],
remove_overlay_elements=True,
markdown_generator=DefaultMarkdownGenerator(
- content_filter=PruningContentFilter(
+ content_filter=PruningContentFilterLXML(
threshold=0.48, threshold_type="fixed", min_word_threshold=0
),
options={"ignore_links": True},
diff --git a/docs/examples/quickstart_examples_set_1.py b/docs/examples/quickstart_examples_set_1.py
index 078d1c4a..d9dabc2b 100644
--- a/docs/examples/quickstart_examples_set_1.py
+++ b/docs/examples/quickstart_examples_set_1.py
@@ -10,7 +10,7 @@ from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode, CrawlResult
from crawl4ai import RoundRobinProxyStrategy
from crawl4ai import JsonCssExtractionStrategy, LLMExtractionStrategy
from crawl4ai import LLMConfig
-from crawl4ai import PruningContentFilter, BM25ContentFilter
+from crawl4ai import PruningContentFilterLXML, BM25ContentFilter
from crawl4ai import DefaultMarkdownGenerator
from crawl4ai import BFSDeepCrawlStrategy, DomainFilter, FilterChain
from crawl4ai import BrowserConfig
@@ -69,7 +69,7 @@ async def demo_fit_markdown():
url = "https://en.wikipedia.org/wiki/Python_(programming_language)",
config=CrawlerRunConfig(
markdown_generator=DefaultMarkdownGenerator(
- content_filter=PruningContentFilter()
+ content_filter=PruningContentFilterLXML()
)
),
)
diff --git a/docs/examples/quickstart_examples_set_2.py b/docs/examples/quickstart_examples_set_2.py
index b12b084c..e78d5701 100644
--- a/docs/examples/quickstart_examples_set_2.py
+++ b/docs/examples/quickstart_examples_set_2.py
@@ -15,7 +15,7 @@ from bs4 import BeautifulSoup
from pydantic import BaseModel, Field
from crawl4ai import AsyncWebCrawler, CacheMode, BrowserConfig, CrawlerRunConfig
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
from crawl4ai import (
JsonCssExtractionStrategy,
LLMExtractionStrategy,
@@ -48,7 +48,7 @@ async def clean_content():
excluded_tags=["nav", "footer", "aside"],
remove_overlay_elements=True,
markdown_generator=DefaultMarkdownGenerator(
- content_filter=PruningContentFilter(
+ content_filter=PruningContentFilterLXML(
threshold=0.48, threshold_type="fixed", min_word_threshold=0
),
options={"ignore_links": True},
diff --git a/docs/examples/serp_api_project_11_feb.py b/docs/examples/serp_api_project_11_feb.py
index 8f9312f7..2241bde0 100644
--- a/docs/examples/serp_api_project_11_feb.py
+++ b/docs/examples/serp_api_project_11_feb.py
@@ -14,7 +14,7 @@ from crawl4ai import (
CrawlerHub,
CrawlResult,
DefaultMarkdownGenerator,
- PruningContentFilter,
+ PruningContentFilterLXML,
)
from pathlib import Path
from pydantic import BaseModel
@@ -35,7 +35,7 @@ async def hello_web():
crawler_config = CrawlerRunConfig(
cache_mode=CacheMode.BYPASS,
markdown_generator=DefaultMarkdownGenerator(
- content_filter=PruningContentFilter(
+ content_filter=PruningContentFilterLXML(
threshold=0.48, threshold_type="fixed", min_word_threshold=0
)
),
diff --git a/docs/examples/session_id_example.py b/docs/examples/session_id_example.py
index e49b7819..4781b705 100644
--- a/docs/examples/session_id_example.py
+++ b/docs/examples/session_id_example.py
@@ -4,7 +4,7 @@ from crawl4ai import (
BrowserConfig,
CrawlerRunConfig,
DefaultMarkdownGenerator,
- PruningContentFilter,
+ PruningContentFilterLXML,
CrawlResult
)
diff --git a/docs/examples/url_seeder/bbc_sport_research_assistant.py b/docs/examples/url_seeder/bbc_sport_research_assistant.py
index 51bcf179..8e797d1d 100644
--- a/docs/examples/url_seeder/bbc_sport_research_assistant.py
+++ b/docs/examples/url_seeder/bbc_sport_research_assistant.py
@@ -53,7 +53,7 @@ from crawl4ai import (
SeedingConfig,
AsyncLogger
)
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
# LiteLLM for AI communication
@@ -343,7 +343,7 @@ async def crawl_selected_urls(urls: List[str], query: str, config: ResearchConfi
# Configure markdown generator with content filter
md_generator = DefaultMarkdownGenerator(
- content_filter=PruningContentFilter(
+ content_filter=PruningContentFilterLXML(
threshold=0.48,
threshold_type="dynamic",
min_word_threshold=10
diff --git a/docs/examples/url_seeder/tutorial_url_seeder.md b/docs/examples/url_seeder/tutorial_url_seeder.md
index 4b9a2201..0cda65bb 100644
--- a/docs/examples/url_seeder/tutorial_url_seeder.md
+++ b/docs/examples/url_seeder/tutorial_url_seeder.md
@@ -73,7 +73,7 @@ from crawl4ai import (
SeedingConfig,
AsyncLogger
)
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
# LiteLLM for AI capabilities
@@ -357,7 +357,7 @@ async def crawl_selected_urls(
# Configure intelligent content extraction
# This removes ads, navigation, and other noise
md_generator = DefaultMarkdownGenerator(
- content_filter=PruningContentFilter(
+ content_filter=PruningContentFilterLXML(
threshold=0.48, # Content relevance threshold
threshold_type="dynamic", # Adapts to page structure
min_word_threshold=10 # Ignore tiny text blocks
diff --git a/docs/examples/url_seeder/url_seeder_quick_demo.py b/docs/examples/url_seeder/url_seeder_quick_demo.py
index f48c5eec..acc73ef0 100644
--- a/docs/examples/url_seeder/url_seeder_quick_demo.py
+++ b/docs/examples/url_seeder/url_seeder_quick_demo.py
@@ -6,7 +6,7 @@ Note: Uses context manager for automatic cleanup of resources.
"""
import asyncio, os
from crawl4ai import AsyncUrlSeeder, AsyncWebCrawler, SeedingConfig, CrawlerRunConfig, AsyncLogger, DefaultMarkdownGenerator
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -39,7 +39,7 @@ async def discover_and_crawl():
async with AsyncWebCrawler() as crawler:
config = CrawlerRunConfig(
markdown_generator=DefaultMarkdownGenerator(
- content_filter=PruningContentFilter( # Smart filtering!
+ content_filter=PruningContentFilterLXML( # Smart filtering!
threshold=0.48, # Remove fluff
threshold_type="fixed",
),
diff --git a/docs/md_v2/advanced/multi-url-crawling.md b/docs/md_v2/advanced/multi-url-crawling.md
index 2c924eff..994fbfcf 100644
--- a/docs/md_v2/advanced/multi-url-crawling.md
+++ b/docs/md_v2/advanced/multi-url-crawling.md
@@ -418,7 +418,7 @@ When crawling diverse content types, you often need different configurations for
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, MatchMode
from crawl4ai.processors.pdf import PDFContentScrapingStrategy
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
async def crawl_mixed_content():
@@ -434,7 +434,7 @@ async def crawl_mixed_content():
CrawlerRunConfig(
url_matcher=["*/blog/*", "*/article/*"],
markdown_generator=DefaultMarkdownGenerator(
- content_filter=PruningContentFilter(threshold=0.48)
+ content_filter=PruningContentFilterLXML(threshold=0.48)
)
),
diff --git a/docs/md_v2/advanced/undetected-browser.md b/docs/md_v2/advanced/undetected-browser.md
index 32b73fc5..c709602f 100644
--- a/docs/md_v2/advanced/undetected-browser.md
+++ b/docs/md_v2/advanced/undetected-browser.md
@@ -216,7 +216,7 @@ async def main():
# Configure the crawl
crawler_config = CrawlerRunConfig(
markdown_generator=DefaultMarkdownGenerator(
- content_filter=PruningContentFilter()
+ content_filter=PruningContentFilterLXML()
),
capture_console_messages=True, # Test adapter console capture
)
diff --git a/docs/md_v2/api/arun_many.md b/docs/md_v2/api/arun_many.md
index 740dcc24..96324ae2 100644
--- a/docs/md_v2/api/arun_many.md
+++ b/docs/md_v2/api/arun_many.md
@@ -105,7 +105,7 @@ Instead of using one config for all URLs, provide a list of configs with `url_ma
from crawl4ai import CrawlerRunConfig, MatchMode
from crawl4ai.processors.pdf import PDFContentScrapingStrategy
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
# PDF files - specialized extraction
@@ -118,7 +118,7 @@ pdf_config = CrawlerRunConfig(
blog_config = CrawlerRunConfig(
url_matcher=["*/blog/*", "*/article/*", "*python.org*"],
markdown_generator=DefaultMarkdownGenerator(
- content_filter=PruningContentFilter(threshold=0.48)
+ content_filter=PruningContentFilterLXML(threshold=0.48)
)
)
diff --git a/docs/md_v2/api/crawl-result.md b/docs/md_v2/api/crawl-result.md
index 8890a704..f061836c 100644
--- a/docs/md_v2/api/crawl-result.md
+++ b/docs/md_v2/api/crawl-result.md
@@ -159,7 +159,7 @@ print(result.markdown.raw_markdown[:200])
print(result.markdown.fit_markdown)
print(result.markdown.fit_html)
```
-**Important**: "Fit" content (in `fit_markdown`/`fit_html`) exists in result.markdown, only if you used a **filter** (like **PruningContentFilter** or **BM25ContentFilter**) within a `MarkdownGenerationStrategy`.
+**Important**: "Fit" content (in `fit_markdown`/`fit_html`) exists in result.markdown, only if you used a **filter** (like **PruningContentFilterLXML** or **BM25ContentFilter**) within a `MarkdownGenerationStrategy`.
---
@@ -414,7 +414,7 @@ async def handle_result(result: CrawlResult):
- `fit_markdown` and `fit_html` - No longer top-level properties in v0.5. Use `result.markdown.fit_markdown` and `result.markdown.fit_html`.
2. **Fit Content**
- - **`fit_markdown`** and **`fit_html`** appear in MarkdownGenerationResult, only if you used a content filter (like **PruningContentFilter** or **BM25ContentFilter**) inside your **MarkdownGenerationStrategy** or set them directly.
+ - **`fit_markdown`** and **`fit_html`** appear in MarkdownGenerationResult, only if you used a content filter (like **PruningContentFilterLXML** or **BM25ContentFilter**) inside your **MarkdownGenerationStrategy** or set them directly.
- If no filter is used, they remain `None`.
3. **References & Citations**
diff --git a/docs/md_v2/complete-sdk-reference.md b/docs/md_v2/complete-sdk-reference.md
index aa0517b2..938111dc 100644
--- a/docs/md_v2/complete-sdk-reference.md
+++ b/docs/md_v2/complete-sdk-reference.md
@@ -151,15 +151,15 @@ if __name__ == "__main__":
## 4. Generating Markdown Output
- **`result.markdown`**:
- **`result.markdown.fit_markdown`**:
- The same content after applying any configured **content filter** (e.g., `PruningContentFilter`).
+ The same content after applying any configured **content filter** (e.g., `PruningContentFilterLXML`).
### Example: Using a Filter with `DefaultMarkdownGenerator`
```python
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
md_generator = DefaultMarkdownGenerator(
- content_filter=PruningContentFilter(threshold=0.4, threshold_type="fixed")
+ content_filter=PruningContentFilterLXML(threshold=0.4, threshold_type="fixed")
)
config = CrawlerRunConfig(
@@ -172,7 +172,7 @@ async with AsyncWebCrawler() as crawler:
print("Raw Markdown length:", len(result.markdown.raw_markdown))
print("Fit Markdown length:", len(result.markdown.fit_markdown))
```
-**Note**: If you do **not** specify a content filter or markdown generator, you’ll typically see only the raw Markdown. `PruningContentFilter` may adds around `50ms` in processing time. We’ll dive deeper into these strategies in a dedicated **Markdown Generation** tutorial.
+**Note**: If you do **not** specify a content filter or markdown generator, you’ll typically see only the raw Markdown. `PruningContentFilterLXML` may adds around `50ms` in processing time. We’ll dive deeper into these strategies in a dedicated **Markdown Generation** tutorial.
## 5. Simple Data Extraction (CSS-based)
```python
from crawl4ai import JsonCssExtractionStrategy
@@ -1006,7 +1006,7 @@ Instead of using one config for all URLs, provide a list of configs with `url_ma
from crawl4ai import CrawlerRunConfig, MatchMode
from crawl4ai.processors.pdf import PDFContentScrapingStrategy
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
# PDF files - specialized extraction
@@ -1019,7 +1019,7 @@ pdf_config = CrawlerRunConfig(
blog_config = CrawlerRunConfig(
url_matcher=["*/blog/*", "*/article/*", "*python.org*"],
markdown_generator=DefaultMarkdownGenerator(
- content_filter=PruningContentFilter(threshold=0.48)
+ content_filter=PruningContentFilterLXML(threshold=0.48)
)
)
@@ -1171,7 +1171,7 @@ print(result.markdown.raw_markdown[:200])
print(result.markdown.fit_markdown)
print(result.markdown.fit_html)
```
-**Important**: "Fit" content (in `fit_markdown`/`fit_html`) exists in result.markdown, only if you used a **filter** (like **PruningContentFilter** or **BM25ContentFilter**) within a `MarkdownGenerationStrategy`.
+**Important**: "Fit" content (in `fit_markdown`/`fit_html`) exists in result.markdown, only if you used a **filter** (like **PruningContentFilterLXML** or **BM25ContentFilter**) within a `MarkdownGenerationStrategy`.
## 4. Media & Links
### 4.1 **`media`** *(Dict[str, List[Dict]])*
**What**: Contains info about discovered images, videos, or audio. Typically keys: `"images"`, `"videos"`, `"audios"`.
@@ -1363,7 +1363,7 @@ async def handle_result(result: CrawlResult):
- `markdown_v2` - Removed in v0.5. Accessing it raises `AttributeError`. Use `markdown`.
- `fit_markdown` and `fit_html` - Removed as top-level `CrawlResult` properties in v0.5. Use `result.markdown.fit_markdown` and `result.markdown.fit_html`.
2. **Fit Content**
- - **`fit_markdown`** and **`fit_html`** appear in MarkdownGenerationResult, only if you used a content filter (like **PruningContentFilter** or **BM25ContentFilter**) inside your **MarkdownGenerationStrategy** or set them directly.
+ - **`fit_markdown`** and **`fit_html`** appear in MarkdownGenerationResult, only if you used a content filter (like **PruningContentFilterLXML** or **BM25ContentFilter**) inside your **MarkdownGenerationStrategy** or set them directly.
- If no filter is used, they remain `None`.
3. **References & Citations**
- If you enable link citations in your `DefaultMarkdownGenerator` (`options={"citations": True}`), you’ll see `markdown_with_citations` plus a **`references_markdown`** block. This helps large language models or academic-like referencing.
@@ -2038,7 +2038,7 @@ The `arun()` method returns a `CrawlResult` object with several useful propertie
```python
config = CrawlerRunConfig(
markdown_generator=DefaultMarkdownGenerator(
- content_filter=PruningContentFilter(threshold=0.6),
+ content_filter=PruningContentFilterLXML(threshold=0.6),
options={"ignore_links": True}
)
)
@@ -2293,12 +2293,12 @@ config = CrawlerRunConfig(markdown_generator=md_generator)
- **`bm25_threshold`**: Raise it to keep fewer blocks; lower it to keep more.
- **`use_stemming`** *(default `True`)*: Whether to apply stemming to the query and content.
- **`language (str)`**: Language for stemming (default: 'english').
-### 5.2 PruningContentFilter
-If you **don’t** have a specific query, or if you just want a robust “junk remover,” use `PruningContentFilter`. It analyzes text density, link density, HTML structure, and known patterns (like “nav,” “footer”) to systematically prune extraneous or repetitive sections.
+### 5.2 PruningContentFilterLXML
+If you **don’t** have a specific query, or if you just want a robust “junk remover,” use `PruningContentFilterLXML`. It analyzes text density, link density, HTML structure, and known patterns (like “nav,” “footer”) to systematically prune extraneous or repetitive sections.
```python
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
-prune_filter = PruningContentFilter(
+prune_filter = PruningContentFilterLXML(
threshold=0.5,
threshold_type="fixed", # or "dynamic"
min_word_threshold=50
@@ -2383,12 +2383,12 @@ When a content filter is active, the library produces two forms of markdown insi
import asyncio
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
async def main():
config = CrawlerRunConfig(
markdown_generator=DefaultMarkdownGenerator(
- content_filter=PruningContentFilter(threshold=0.6),
+ content_filter=PruningContentFilterLXML(threshold=0.6),
options={"ignore_links": True}
)
)
@@ -2424,14 +2424,14 @@ print("FIT:\n", md_obj.fit_markdown)
- Or feed `fit_markdown` into a vector database to reduce token usage.
- `references_markdown` can help you keep track of link provenance.
## 8. Combining Filters (BM25 + Pruning) in Two Passes
-You might want to **prune out** noisy boilerplate first (with `PruningContentFilter`), and then **rank what’s left** against a user query (with `BM25ContentFilter`). You don’t have to crawl the page twice. Instead:
-1. **First pass**: Apply `PruningContentFilter` directly to the raw HTML from `result.html` (the crawler’s downloaded HTML).
+You might want to **prune out** noisy boilerplate first (with `PruningContentFilterLXML`), and then **rank what’s left** against a user query (with `BM25ContentFilter`). You don’t have to crawl the page twice. Instead:
+1. **First pass**: Apply `PruningContentFilterLXML` directly to the raw HTML from `result.html` (the crawler’s downloaded HTML).
2. **Second pass**: Take the pruned HTML (or text) from step 1, and feed it into `BM25ContentFilter`, focusing on a user query.
### Two-Pass Example
```python
import asyncio
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
-from crawl4ai.content_filter_strategy import PruningContentFilter, BM25ContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML, BM25ContentFilter
from bs4 import BeautifulSoup
async def main():
@@ -2450,8 +2450,8 @@ async def main():
raw_html = result.html
- # 2. First pass: PruningContentFilter on raw HTML
- pruning_filter = PruningContentFilter(threshold=0.5, min_word_threshold=50)
+ # 2. First pass: PruningContentFilterLXML on raw HTML
+ pruning_filter = PruningContentFilterLXML(threshold=0.5, min_word_threshold=50)
# filter_content returns a list of "text chunks" or cleaned HTML sections
pruned_chunks = pruning_filter.filter_content(raw_html)
@@ -2518,17 +2518,24 @@ In **`CrawlerRunConfig`**, you can specify a **`content_filter`** to shape how c
- **`result.markdown.fit_markdown`** (filtered or “fit” version)
- **`result.markdown.fit_html`** (the corresponding HTML snippet that produced `fit_markdown`)
### 1.2 Common Filters
-## 2. PruningContentFilter
+## 2. PruningContentFilterLXML
+
+!!! warning "Deprecation"
+ Use `PruningContentFilterLXML` (lxml engine, **~10× faster**, identical output). The legacy
+ `PruningContentFilter` (BeautifulSoup) still works but emits a `DeprecationWarning`. In an
+ upcoming release `PruningContentFilter` will become an alias for the lxml engine, with
+ `PruningContentFilterLXML` kept as a legacy alias.
+
### 2.1 Usage Example
```python
import asyncio
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
async def main():
# Step 1: Create a pruning filter
- prune_filter = PruningContentFilter(
+ prune_filter = PruningContentFilterLXML(
# Lower → more content retained, higher → more content pruned
threshold=0.45,
# "fixed" or "dynamic"
@@ -2623,7 +2630,7 @@ If the content filter is **BM25**, you might see additional logic or references
## 5. Code Patterns Recap
### 5.1 Pruning
```python
-prune_filter = PruningContentFilter(
+prune_filter = PruningContentFilterLXML(
threshold=0.5,
threshold_type="fixed",
min_word_threshold=10
@@ -2647,7 +2654,7 @@ config = CrawlerRunConfig(
excluded_tags=["nav", "footer", "header"],
exclude_external_links=True,
markdown_generator=DefaultMarkdownGenerator(
- content_filter=PruningContentFilter(threshold=0.5)
+ content_filter=PruningContentFilterLXML(threshold=0.5)
)
)
```
@@ -4698,7 +4705,7 @@ To keep context continuous across chunks, we can overlap them. E.g., `overlap_ra
## 7. Input Format
By default, **LLMExtractionStrategy** uses `input_format="markdown"`, meaning the **crawler’s final markdown** is fed to the LLM. You can change to:
- **`html`**: The cleaned HTML or raw HTML (depending on your crawler config) goes into the LLM.
-- **`fit_markdown`**: If you used, for instance, `PruningContentFilter`, the “fit” version of the markdown is used. This can drastically reduce tokens if you trust the filter.
+- **`fit_markdown`**: If you used, for instance, `PruningContentFilterLXML`, the “fit” version of the markdown is used. This can drastically reduce tokens if you trust the filter.
- **`markdown`**: Standard markdown output from the crawler’s `markdown_generator`.
This setting is crucial: if the LLM instructions rely on HTML tags, pick `"html"`. If you prefer a text-based approach, pick `"markdown"`.
```python
diff --git a/docs/md_v2/core/crawler-result.md b/docs/md_v2/core/crawler-result.md
index a531e043..79f72318 100644
--- a/docs/md_v2/core/crawler-result.md
+++ b/docs/md_v2/core/crawler-result.md
@@ -141,7 +141,7 @@ print(md_res.markdown_with_citations)
print(md_res.references_markdown)
```
-**Note**: If you use a filter like `PruningContentFilter`, you’ll get `fit_markdown` and `fit_html` as well.
+**Note**: If you use a filter like `PruningContentFilterLXML`, you’ll get `fit_markdown` and `fit_html` as well.
---
@@ -336,7 +336,7 @@ else:
## 7. Next Steps
- **Markdown Generation**: Dive deeper into how to configure `DefaultMarkdownGenerator` and various filters.
-- **Content Filtering**: Learn how to use `BM25ContentFilter` and `PruningContentFilter`.
+- **Content Filtering**: Learn how to use `BM25ContentFilter` and `PruningContentFilterLXML`.
- **Session & Hooks**: If you want to manipulate the page or preserve state across multiple `arun()` calls, see the hooking or session docs.
- **LLM Extraction**: For complex or unstructured content requiring AI-driven parsing, check the LLM-based strategies doc.
diff --git a/docs/md_v2/core/fit-markdown.md b/docs/md_v2/core/fit-markdown.md
index 1684a779..149496ca 100644
--- a/docs/md_v2/core/fit-markdown.md
+++ b/docs/md_v2/core/fit-markdown.md
@@ -1,5 +1,22 @@
# Fit Markdown with Pruning & BM25
+!!! warning "Deprecation: `PruningContentFilter` → `PruningContentFilterLXML`"
+ The original `PruningContentFilter` (BeautifulSoup-based) is **deprecated** in favor of
+ **`PruningContentFilterLXML`**, an lxml reimplementation that is **~10× faster** on large
+ pages with **identical output**. It is already the default for the `crwl` CLI and the
+ Docker server's fit-markdown endpoint.
+
+ **Migrate** by swapping the class name (every parameter is unchanged):
+
+ ```python
+ from crawl4ai import PruningContentFilterLXML
+ prune_filter = PruningContentFilterLXML(threshold=0.48, threshold_type="dynamic")
+ ```
+
+ Instantiating the old `PruningContentFilter` now emits a `DeprecationWarning`. **Roadmap:**
+ in an upcoming release `PruningContentFilter` will become an alias for the lxml engine, and
+ `PruningContentFilterLXML` will be kept as a legacy alias.
+
**Fit Markdown** is a specialized **filtered** version of your page’s markdown, focusing on the most relevant content. By default, Crawl4AI converts the entire HTML into a broad **raw_markdown**. With fit markdown, we apply a **content filter** algorithm (e.g., **Pruning** or **BM25**) to remove or rank low-value sections—such as repetitive sidebars, shallow text blocks, or irrelevancies—leaving a concise textual “core.”
---
@@ -17,12 +34,12 @@ In **`CrawlerRunConfig`**, you can specify a **`content_filter`** to shape how c
### 1.2 Common Filters
-1. **PruningContentFilter** – Scores each node by text density, link density, and tag importance, discarding those below a threshold.
+1. **PruningContentFilterLXML** – Scores each node by text density, link density, and tag importance, discarding those below a threshold.
2. **BM25ContentFilter** – Focuses on textual relevance using BM25 ranking, especially useful if you have a specific user query (e.g., “machine learning” or “food nutrition”).
---
-## 2. PruningContentFilter
+## 2. PruningContentFilterLXML
**Pruning** discards less relevant nodes based on **text density, link density, and tag importance**. It’s a heuristic-based approach—if certain sections appear too “thin” or too “spammy,” they’re pruned.
@@ -31,12 +48,12 @@ In **`CrawlerRunConfig`**, you can specify a **`content_filter`** to shape how c
```python
import asyncio
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
async def main():
# Step 1: Create a pruning filter
- prune_filter = PruningContentFilter(
+ prune_filter = PruningContentFilterLXML(
# Lower → more content retained, higher → more content pruned
threshold=0.45,
# "fixed" or "dynamic"
@@ -159,7 +176,7 @@ If the content filter is **BM25**, you might see additional logic or references
### 5.1 Pruning
```python
-prune_filter = PruningContentFilter(
+prune_filter = PruningContentFilterLXML(
threshold=0.5,
threshold_type="fixed",
min_word_threshold=10
@@ -191,7 +208,7 @@ config = CrawlerRunConfig(
excluded_tags=["nav", "footer", "header"],
exclude_external_links=True,
markdown_generator=DefaultMarkdownGenerator(
- content_filter=PruningContentFilter(threshold=0.5)
+ content_filter=PruningContentFilterLXML(threshold=0.5)
)
)
```
@@ -235,7 +252,7 @@ class MyCustomFilter(RelevantContentFilter):
- **AI Pipelines**: Filter out boilerplate so LLM-based extraction or summarization runs on denser text.
**Key Points**:
-- **PruningContentFilter**: Great if you just want the “meatiest” text without a user query.
+- **PruningContentFilterLXML**: Great if you just want the “meatiest” text without a user query.
- **BM25ContentFilter**: Perfect for query-based extraction or searching.
- Combine with **`excluded_tags`, `exclude_external_links`, `word_count_threshold`** to refine your final “fit” text.
- Fit markdown ends up in **`result.markdown.fit_markdown`**; eventually **`result.markdown.fit_markdown`** in future versions.
diff --git a/docs/md_v2/core/markdown-generation.md b/docs/md_v2/core/markdown-generation.md
index af9b35b5..6d231a21 100644
--- a/docs/md_v2/core/markdown-generation.md
+++ b/docs/md_v2/core/markdown-generation.md
@@ -205,14 +205,20 @@ config = CrawlerRunConfig(markdown_generator=md_generator)
**No query provided?** BM25 tries to glean a context from page metadata, or you can simply treat it as a scorched-earth approach that discards text with low generic score. Realistically, you want to supply a query for best results.
-### 5.2 PruningContentFilter
+### 5.2 PruningContentFilterLXML
-If you **don’t** have a specific query, or if you just want a robust “junk remover,” use `PruningContentFilter`. It analyzes text density, link density, HTML structure, and known patterns (like “nav,” “footer”) to systematically prune extraneous or repetitive sections.
+!!! warning "Use `PruningContentFilterLXML` (the old `PruningContentFilter` is deprecated)"
+ `PruningContentFilterLXML` is the lxml-based engine: **~10× faster** on large pages with
+ **identical output**. The legacy `PruningContentFilter` still works but emits a
+ `DeprecationWarning`; in an upcoming release `PruningContentFilter` will alias the lxml
+ engine and `PruningContentFilterLXML` will remain as a legacy alias.
+
+If you **don’t** have a specific query, or if you just want a robust “junk remover,” use `PruningContentFilterLXML`. It analyzes text density, link density, HTML structure, and known patterns (like “nav,” “footer”) to systematically prune extraneous or repetitive sections.
```python
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
-prune_filter = PruningContentFilter(
+prune_filter = PruningContentFilterLXML(
threshold=0.5,
threshold_type="fixed", # or "dynamic"
min_word_threshold=50
@@ -225,7 +231,7 @@ prune_filter = PruningContentFilter(
- `"dynamic"`: The filter adjusts threshold in a data-driven manner.
- **`min_word_threshold`**: Discard blocks under N words as likely too short or unhelpful.
-**When to Use PruningContentFilter**
+**When to Use PruningContentFilterLXML**
- You want a broad cleanup without a user query.
- The page has lots of repeated sidebars, footers, or disclaimers that hamper text extraction.
@@ -320,12 +326,12 @@ When a content filter is active, the library produces two forms of markdown insi
import asyncio
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
async def main():
config = CrawlerRunConfig(
markdown_generator=DefaultMarkdownGenerator(
- content_filter=PruningContentFilter(threshold=0.6),
+ content_filter=PruningContentFilterLXML(threshold=0.6),
options={"ignore_links": True}
)
)
@@ -379,9 +385,9 @@ Below is a **revised section** under “Combining Filters (BM25 + Pruning)” th
## 8. Combining Filters (BM25 + Pruning) in Two Passes
-You might want to **prune out** noisy boilerplate first (with `PruningContentFilter`), and then **rank what’s left** against a user query (with `BM25ContentFilter`). You don’t have to crawl the page twice. Instead:
+You might want to **prune out** noisy boilerplate first (with `PruningContentFilterLXML`), and then **rank what’s left** against a user query (with `BM25ContentFilter`). You don’t have to crawl the page twice. Instead:
-1. **First pass**: Apply `PruningContentFilter` directly to the raw HTML from `result.html` (the crawler’s downloaded HTML).
+1. **First pass**: Apply `PruningContentFilterLXML` directly to the raw HTML from `result.html` (the crawler’s downloaded HTML).
2. **Second pass**: Take the pruned HTML (or text) from step 1, and feed it into `BM25ContentFilter`, focusing on a user query.
### Two-Pass Example
@@ -389,7 +395,7 @@ You might want to **prune out** noisy boilerplate first (with `PruningContentFil
```python
import asyncio
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
-from crawl4ai.content_filter_strategy import PruningContentFilter, BM25ContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML, BM25ContentFilter
from bs4 import BeautifulSoup
async def main():
@@ -408,8 +414,8 @@ async def main():
raw_html = result.html
- # 2. First pass: PruningContentFilter on raw HTML
- pruning_filter = PruningContentFilter(threshold=0.5, min_word_threshold=50)
+ # 2. First pass: PruningContentFilterLXML on raw HTML
+ pruning_filter = PruningContentFilterLXML(threshold=0.5, min_word_threshold=50)
# filter_content returns a list of "text chunks" or cleaned HTML sections
pruned_chunks = pruning_filter.filter_content(raw_html)
@@ -449,7 +455,7 @@ if __name__ == "__main__":
### What’s Happening?
1. **Raw HTML**: We crawl once and store the raw HTML in `result.html`.
-2. **PruningContentFilter**: Takes HTML + optional parameters. It extracts blocks of text or partial HTML, removing headings/sections deemed “noise.” It returns a **list of text chunks**.
+2. **PruningContentFilterLXML**: Takes HTML + optional parameters. It extracts blocks of text or partial HTML, removing headings/sections deemed “noise.” It returns a **list of text chunks**.
3. **Combine or Transform**: We join these pruned chunks back into a single HTML-like string. (Alternatively, you could store them in a list for further logic—whatever suits your pipeline.)
4. **BM25ContentFilter**: We feed the pruned string into `BM25ContentFilter` with a user query. This second pass further narrows the content to chunks relevant to “machine learning.”
@@ -495,7 +501,7 @@ In this **Markdown Generation Basics** tutorial, you learned to:
- Configure the **DefaultMarkdownGenerator** with HTML-to-text options.
- Select different HTML sources using the `content_source` parameter.
-- Use **BM25ContentFilter** for query-specific extraction or **PruningContentFilter** for general noise removal.
+- Use **BM25ContentFilter** for query-specific extraction or **PruningContentFilterLXML** for general noise removal.
- Distinguish between raw and filtered markdown (`fit_markdown`).
- Leverage the `MarkdownGenerationResult` object to handle different forms of output (citations, references, etc.).
diff --git a/docs/md_v2/core/quickstart.md b/docs/md_v2/core/quickstart.md
index 78209efd..063c93c6 100644
--- a/docs/md_v2/core/quickstart.md
+++ b/docs/md_v2/core/quickstart.md
@@ -92,17 +92,17 @@ By default, Crawl4AI automatically generates Markdown from each crawled page. Ho
- **`result.markdown`**:
The direct HTML-to-Markdown conversion.
- **`result.markdown.fit_markdown`**:
- The same content after applying any configured **content filter** (e.g., `PruningContentFilter`).
+ The same content after applying any configured **content filter** (e.g., `PruningContentFilterLXML`).
### Example: Using a Filter with `DefaultMarkdownGenerator`
```python
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
md_generator = DefaultMarkdownGenerator(
- content_filter=PruningContentFilter(threshold=0.4, threshold_type="fixed")
+ content_filter=PruningContentFilterLXML(threshold=0.4, threshold_type="fixed")
)
config = CrawlerRunConfig(
@@ -116,7 +116,7 @@ async with AsyncWebCrawler() as crawler:
print("Fit Markdown length:", len(result.markdown.fit_markdown))
```
-**Note**: If you do **not** specify a content filter or markdown generator, you’ll typically see only the raw Markdown. `PruningContentFilter` may adds around `50ms` in processing time. We’ll dive deeper into these strategies in a dedicated **Markdown Generation** tutorial.
+**Note**: If you do **not** specify a content filter or markdown generator, you’ll typically see only the raw Markdown. `PruningContentFilterLXML` may adds around `50ms` in processing time. We’ll dive deeper into these strategies in a dedicated **Markdown Generation** tutorial.
---
diff --git a/docs/md_v2/core/simple-crawling.md b/docs/md_v2/core/simple-crawling.md
index dd31bc97..c9b78533 100644
--- a/docs/md_v2/core/simple-crawling.md
+++ b/docs/md_v2/core/simple-crawling.md
@@ -33,7 +33,7 @@ The `arun()` method returns a `CrawlResult` object with several useful propertie
```python
config = CrawlerRunConfig(
markdown_generator=DefaultMarkdownGenerator(
- content_filter=PruningContentFilter(threshold=0.6),
+ content_filter=PruningContentFilterLXML(threshold=0.6),
options={"ignore_links": True}
)
)
diff --git a/docs/md_v2/extraction/llm-strategies.md b/docs/md_v2/extraction/llm-strategies.md
index cba4d6e4..197d2ba2 100644
--- a/docs/md_v2/extraction/llm-strategies.md
+++ b/docs/md_v2/extraction/llm-strategies.md
@@ -175,7 +175,7 @@ By chunking, you can potentially process multiple chunks in parallel (depending
By default, **LLMExtractionStrategy** uses `input_format="markdown"`, meaning the **crawler’s final markdown** is fed to the LLM. You can change to:
- **`html`**: The cleaned HTML or raw HTML (depending on your crawler config) goes into the LLM.
-- **`fit_markdown`**: If you used, for instance, `PruningContentFilter`, the “fit” version of the markdown is used. This can drastically reduce tokens if you trust the filter.
+- **`fit_markdown`**: If you used, for instance, `PruningContentFilterLXML`, the “fit” version of the markdown is used. This can drastically reduce tokens if you trust the filter.
- **`markdown`**: Standard markdown output from the crawler’s `markdown_generator`.
This setting is crucial: if the LLM instructions rely on HTML tags, pick `"html"`. If you prefer a text-based approach, pick `"markdown"`.
diff --git a/tests/async/test_content_filter_prune_lxml.py b/tests/async/test_content_filter_prune_lxml.py
index afc1226f..889387eb 100644
--- a/tests/async/test_content_filter_prune_lxml.py
+++ b/tests/async/test_content_filter_prune_lxml.py
@@ -153,6 +153,7 @@ class TestPruningContentFilterLXMLBehavior:
# --------------------------------------------------------------------------- #
# Fidelity: new output must match the original bs4 implementation
# --------------------------------------------------------------------------- #
[email protected]("ignore::DeprecationWarning") # deliberately exercises the deprecated bs4 filter
class TestPruningEquivalence:
@pytest.mark.parametrize("name", list(EQUIVALENCE_CASES.keys()))
@pytest.mark.parametrize("config", CONFIGS)
@@ -182,6 +183,7 @@ class TestPruningEquivalence:
# --------------------------------------------------------------------------- #
# Performance: new must be substantially faster on a non-trivial page
# --------------------------------------------------------------------------- #
[email protected]("ignore::DeprecationWarning") # compares against the deprecated bs4 filter
def test_performance_faster_than_original():
# ~600 repeated cards -> a few hundred KB of HTML
html = "<body>" + "".join(