Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

perf(content-filter): lxml-native PruningContentFilterLXML, ~10x faster pruning, made default

unclecodec2e4cd652f7a
diff --git a/crawl4ai/__init__.py b/crawl4ai/__init__.py
index 0d1071ae..c33e8b13 100644
--- a/crawl4ai/__init__.py
+++ b/crawl4ai/__init__.py
@@ -42,6 +42,7 @@ from .content_filter_strategy import (
LLMContentFilter,
RelevantContentFilter,
)
+from .content_filter_strategy_lxml import PruningContentFilterLXML
from .models import CrawlResult, MarkdownGenerationResult, DisplayMode
from .components.crawler_monitor import CrawlerMonitor
from .link_preview import LinkPreview
@@ -176,6 +177,7 @@ __all__ = [
"LLMTableExtraction",
"RelevantContentFilter",
"PruningContentFilter",
+ "PruningContentFilterLXML",
"BM25ContentFilter",
"LLMContentFilter",
"BaseDispatcher",
diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py
index 27320fd4..76445c02 100644
--- a/crawl4ai/async_configs.py
+++ b/crawl4ai/async_configs.py
@@ -133,7 +133,7 @@ ALLOWED_DESERIALIZE_TYPES = {
"CosineStrategy", "RegexExtractionStrategy",
# Markdown / content
"DefaultMarkdownGenerator",
- "PruningContentFilter", "BM25ContentFilter", "LLMContentFilter",
+ "PruningContentFilter", "PruningContentFilterLXML", "BM25ContentFilter", "LLMContentFilter",
# Scraping
"LXMLWebScrapingStrategy", "PDFContentScrapingStrategy",
# Chunking
@@ -190,7 +190,7 @@ UNTRUSTED_ALLOWED_TYPES = {
# non-LLM extraction / markdown / scraping / chunking strategies
"JsonCssExtractionStrategy", "JsonXPathExtractionStrategy",
"JsonLxmlExtractionStrategy", "RegexExtractionStrategy", "CosineStrategy",
- "DefaultMarkdownGenerator", "PruningContentFilter", "BM25ContentFilter",
+ "DefaultMarkdownGenerator", "PruningContentFilter", "PruningContentFilterLXML", "BM25ContentFilter",
"LXMLWebScrapingStrategy", "PDFContentScrapingStrategy",
"RegexChunking",
"DefaultTableExtraction", "NoTableExtraction",
diff --git a/crawl4ai/cli.py b/crawl4ai/cli.py
index 02b67155..ec826f63 100644
--- a/crawl4ai/cli.py
+++ b/crawl4ai/cli.py
@@ -25,6 +25,7 @@ from crawl4ai import (
JsonXPathExtractionStrategy,
BM25ContentFilter,
PruningContentFilter,
+ PruningContentFilterLXML,
BrowserProfiler,
DefaultMarkdownGenerator,
LLMConfig,
@@ -1094,7 +1095,7 @@ def crawl_cmd(url: str, browser_config: str, crawler_config: str, filter_config:
)
elif filter_conf["type"] == "pruning":
crawler_cfg.markdown_generator = DefaultMarkdownGenerator(
- content_filter = PruningContentFilter(
+ content_filter = PruningContentFilterLXML(
user_query=filter_conf.get("query"),
threshold=filter_conf.get("threshold", 0.48)
)
diff --git a/crawl4ai/content_filter_strategy_lxml.py b/crawl4ai/content_filter_strategy_lxml.py
new file mode 100644
index 00000000..24671945
--- /dev/null
+++ b/crawl4ai/content_filter_strategy_lxml.py
@@ -0,0 +1,391 @@
+"""
+content_filter_strategy_lxml.py
+===============================
+
+An lxml-native, single-pass reimplementation of :class:`PruningContentFilter`.
+
+Why
+---
+The original (BeautifulSoup) ``PruningContentFilter`` recomputes
+``node.get_text(strip=True)`` and ``node.encode_contents()`` for *every* node it
+visits while pruning. Both calls walk the node's entire subtree, and in
+BeautifulSoup they are pure-Python and slow. Because pruning visits most nodes
+in the document, the total work is effectively ``O(N * average_subtree_size)``
+-- it degrades super-linearly on wide/deep pages. A 6,000-card page spent
+~2.1 s purely inside ``filter_content``, dominated by repeated subtree
+serialization (``str(node)`` / ``encode_contents``).
+
+How this version is different
+-----------------------------
+Every per-node metric the score needs -- visible-text length, inner-HTML
+length, direct-link text length and word count -- is computed **exactly once**
+in a single *bottom-up* pass and cached on a slotted metrics object. Each text
+fragment in the document is therefore touched a constant number of times
+total, not once per ancestor. Scoring + pruning then run in a single
+*top-down* pass over the cached metrics. Total work is ``O(N)`` plus
+``O(total_text_size)``, entirely on libxml2's C-backed tree.
+
+Fidelity
+--------
+BeautifulSoup's ``lxml`` tree-builder and ``lxml.html`` both parse through
+libxml2, so the parsed trees are structurally identical node-for-node. The
+scoring math, thresholds, tag weights and the original's edge-case quirks
+(the always-zero ``class_id`` contribution, the separator-less word count) are
+reproduced 1:1.
+
+Two details are needed to match BeautifulSoup exactly:
+
+* **Per-fragment stripping.** ``get_text(strip=True)`` strips each text
+ fragment independently. BeautifulSoup's ``decompose()`` leaves the text that
+ surrounded a removed tag as *separate* fragments, so each is stripped on its
+ own. We therefore compute metrics *before* physically removing comments and
+ excluded tags: a removed child contributes 0, but its trailing text (tail)
+ is still measured as its own stripped fragment -- identical to what
+ BeautifulSoup sees after ``decompose``.
+* **No depth cap.** libxml2's HTML parser caps tree depth at 256 by default;
+ ``huge_tree=True`` lifts that so deeply nested pages parse like they do under
+ BeautifulSoup.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Dict, List
+
+from lxml import etree
+from lxml import html as lhtml
+
+from .content_filter_strategy import PruningContentFilter
+
+# HTML void elements, taken verbatim from BeautifulSoup's LXMLTreeBuilder
+# ``empty_element_tags``. These serialize self-closed (``<br/>``) with no end
+# tag, which we must account for when reproducing inner-HTML length.
+VOID_ELEMENTS = frozenset(
+ {
+ "area", "base", "basefont", "bgsound", "br", "col", "command", "embed",
+ "frame", "hr", "image", "img", "input", "isindex", "keygen", "link",
+ "menuitem", "meta", "nextid", "param", "source", "spacer", "track", "wbr",
+ }
+)
+
+# A single reused parser. ``huge_tree=True`` lifts libxml2's 256-level depth
+# cap (and other limits) so output matches BeautifulSoup on pathological pages.
+_HTML_PARSER = lhtml.HTMLParser(huge_tree=True)
+
+# libxml2 treats only these as "blank" characters (IS_BLANK_CH); a text node
+# made up entirely of them is collapsed to a single character by BeautifulSoup's
+# tree builder. Note this is ASCII-only -- U+00A0 (&nbsp;) is NOT collapsed,
+# even though Python's str.strip() would remove it.
+_ASCII_WS = " \t\n\r"
+
+
+def _esc_len(s: str) -> int:
+ """Serialized length of ``s`` after HTML entity escaping (``&``->``&amp;``,
+ ``<``->``&lt;``, ``>``->``&gt;``), matching BeautifulSoup's minimal formatter."""
+ return len(s) + (s.count("&") << 2) + 3 * (s.count("<") + s.count(">"))
+
+
+class _NodeMetrics:
+ """Cached, slotted metrics for a single element (one instance per node).
+
+ Attributes
+ ----------
+ text_len:
+ Length of the element's visible text, computed the way
+ BeautifulSoup's ``get_text(strip=True)`` does it: every text fragment
+ (``.text`` of the node + ``.tail`` of each child) is stripped
+ independently and concatenated with no separator.
+ inner_len:
+ Length of the element's serialized inner HTML, matching
+ ``encode_contents()`` (entity-escaped text, child tags including
+ attributes, void elements self-closed).
+ space_count:
+ Number of ASCII spaces in the stripped, concatenated visible text.
+ ``word_count`` is ``space_count + 1`` -- the original counts words by
+ ``text.count(" ") + 1`` on the separator-less concatenation, and we
+ reproduce that exactly.
+ string_len:
+ Length of the stripped BeautifulSoup ``.string`` for this node, or
+ ``-1`` when ``.string`` would be ``None``. Used only to evaluate
+ ``link_text_len`` over direct-child ``<a>`` elements.
+ """
+
+ __slots__ = ("text_len", "inner_len", "space_count", "string_len")
+
+ def __init__(self, text_len: int, inner_len: int, space_count: int, string_len: int):
+ self.text_len = text_len
+ self.inner_len = inner_len
+ self.space_count = space_count
+ self.string_len = string_len
+
+
+class PruningContentFilterLXML(PruningContentFilter):
+ """Drop-in, lxml-native replacement for :class:`PruningContentFilter`.
+
+ Same constructor and same ``filter_content`` contract (returns a list of
+ HTML block strings). Subclasses :class:`PruningContentFilter` so it shares
+ the exact tag weights, metric weights, thresholds and negative-class
+ patterns -- only the engine changes.
+ """
+
+ def __init__(
+ self,
+ user_query: str = None,
+ min_word_threshold: int = None,
+ threshold_type: str = "fixed",
+ threshold: float = 0.48,
+ ):
+ # Mirror PruningContentFilter's explicit signature (rather than
+ # *args/**kwargs) so config (de)serialization can introspect and
+ # round-trip these fields.
+ super().__init__(
+ user_query=user_query,
+ min_word_threshold=min_word_threshold,
+ threshold_type=threshold_type,
+ threshold=threshold,
+ )
+ # word count is only needed when a min-word threshold is active; skip
+ # the per-fragment space counting otherwise.
+ self._need_words = self.min_word_threshold is not None
+
+ # ------------------------------------------------------------------ #
+ # Public API
+ # ------------------------------------------------------------------ #
+ def filter_content(self, html: str, min_word_threshold: int = None) -> List[str]:
+ if not html or not isinstance(html, str):
+ return []
+
+ try:
+ root = lhtml.document_fromstring(html, parser=_HTML_PARSER)
+ except (etree.ParserError, etree.LxmlError, ValueError):
+ return []
+
+ body = root.find("body")
+ if body is None:
+ body = root
+
+ # Metrics are computed on the *un-stripped* tree so that text which
+ # surrounds soon-to-be-removed nodes is measured as separate fragments
+ # (matching BeautifulSoup's decompose semantics).
+ metrics = self._compute_metrics(body)
+
+ # Now physically drop comments and unwanted tags. ``with_tail=False``
+ # keeps the text that followed them (merged into the previous sibling /
+ # parent) so the serialized output still contains it, exactly as a
+ # BeautifulSoup decompose would leave it.
+ etree.strip_elements(body, etree.Comment, with_tail=False)
+ if self.excluded_tags:
+ etree.strip_elements(body, *self.excluded_tags, with_tail=False)
+
+ body_metrics = metrics.get(body)
+ if body_metrics is None:
+ return []
+
+ # The original scores the body itself first; if it fails, the whole
+ # body is decomposed and nothing is returned.
+ if self._should_remove(body, body_metrics, metrics):
+ return []
+
+ # Top-down pass: a node survives only if it and every ancestor up to
+ # body pass the threshold. Iterative (explicit stack) to avoid Python
+ # recursion limits on deeply nested documents.
+ stack = [body]
+ while stack:
+ node = stack.pop()
+ for child in list(node):
+ if not isinstance(child.tag, str):
+ continue
+ child_metrics = metrics.get(child)
+ if child_metrics is None or self._should_remove(child, child_metrics, metrics):
+ child.drop_tree() # removes subtree, preserves following text
+ else:
+ stack.append(child)
+
+ # Collect surviving direct children of body, in document order,
+ # skipping any that no longer carry visible text after pruning.
+ blocks: List[str] = []
+ for el in body:
+ if not isinstance(el.tag, str):
+ continue
+ if any(t.strip() for t in el.itertext()):
+ blocks.append(lhtml.tostring(el, encoding="unicode", with_tail=False))
+ return blocks
+
+ # ------------------------------------------------------------------ #
+ # Metrics (single bottom-up pass)
+ # ------------------------------------------------------------------ #
+ def _compute_metrics(self, body) -> Dict[object, _NodeMetrics]:
+ """Compute and cache metrics for ``body`` and every descendant element,
+ skipping the interior of excluded subtrees (they will be removed).
+
+ Nodes are visited in reverse document order so every parent sees its
+ children's metrics already cached. A parent is always appended to the
+ work list before its children, so ``reversed`` guarantees children are
+ processed first.
+ """
+ metrics: Dict[object, _NodeMetrics] = {}
+ need_words = self._need_words
+ excluded = self.excluded_tags
+
+ # Pre-order collection that does not descend into excluded subtrees.
+ nodes = []
+ stack = [body]
+ while stack:
+ el = stack.pop()
+ nodes.append(el)
+ for child in el:
+ ct = child.tag
+ if isinstance(ct, str) and ct not in excluded:
+ stack.append(child)
+
+ for el in reversed(nodes):
+ txt = el.text
+ if txt:
+ stripped = txt.strip()
+ text_len = len(stripped)
+ # BeautifulSoup's tree builder collapses an ASCII-whitespace-only
+ # text node to a single character; anything else is kept verbatim.
+ if stripped or txt.strip(_ASCII_WS):
+ inner_len = _esc_len(txt)
+ else:
+ inner_len = 1
+ space_count = stripped.count(" ") if need_words else 0
+ frag_count = 1
+ string_len = text_len # candidate, valid only if frag_count == 1
+ else:
+ text_len = inner_len = space_count = 0
+ frag_count = 0
+ string_len = -1
+
+ for child in el:
+ ct = child.tag
+ kept_element = isinstance(ct, str) and ct not in excluded
+
+ if kept_element:
+ cm = metrics[child]
+ text_len += cm.text_len
+ inner_len += self._outer_len(child, cm.inner_len)
+ if need_words:
+ space_count += cm.space_count
+ frag_count += 1
+ string_len = cm.string_len # recurse if this is the sole content
+ # else: comment / excluded element -> removed, contributes 0,
+ # but its trailing text is still a separate fragment below.
+
+ tail = child.tail
+ if tail:
+ st = tail.strip()
+ text_len += len(st)
+ inner_len += _esc_len(tail) if (st or tail.strip(_ASCII_WS)) else 1
+ if need_words:
+ space_count += st.count(" ")
+ frag_count += 1
+ string_len = len(st) # candidate, valid only if frag_count == 1
+
+ # BeautifulSoup ``Tag.string`` is non-None only when the element has
+ # exactly one content fragment.
+ if frag_count != 1:
+ string_len = -1
+
+ metrics[el] = _NodeMetrics(text_len, inner_len, space_count, string_len)
+
+ return metrics
+
+ def _outer_len(self, el, inner_len: int) -> int:
+ """Serialized outer-HTML length of ``el`` given its cached inner length.
+
+ Mirrors BeautifulSoup's default (minimal) serialization:
+ ``<tag attr="val">inner</tag>``, with void elements self-closed as
+ ``<tag/>`` and HTML entity escaping inside attribute values.
+ """
+ tag = el.tag
+ attrs_len = 0
+ for k, v in el.attrib.items():
+ # ' key="value"' -> 1 space + len(key) + '="' + escaped value + '"'
+ attrs_len += 4 + len(k) + _esc_len(v)
+ tag_len = len(tag)
+ if tag in VOID_ELEMENTS:
+ return 1 + tag_len + attrs_len + 2 # <tag ... />
+ # start "<tag ...>" = 2 + tag_len + attrs_len ; end "</tag>" = 3 + tag_len
+ return (2 + tag_len + attrs_len) + inner_len + (3 + tag_len)
+
+ # ------------------------------------------------------------------ #
+ # Scoring (top-down pass)
+ # ------------------------------------------------------------------ #
+ def _should_remove(self, node, m: _NodeMetrics, metrics: Dict[object, _NodeMetrics]) -> bool:
+ text_len = m.text_len
+ tag_len = m.inner_len
+
+ link_text_len = 0
+ for child in node:
+ if child.tag == "a":
+ cm = metrics.get(child)
+ if cm is not None and cm.string_len > 0:
+ link_text_len += cm.string_len
+
+ score = self._composite_score(node, m, text_len, tag_len, link_text_len)
+
+ if self.threshold_type == "fixed":
+ return score < self.threshold
+
+ # dynamic threshold
+ tag_importance = self.tag_importance.get(node.tag, 0.7)
+ text_ratio = text_len / tag_len if tag_len > 0 else 0
+ link_ratio = link_text_len / text_len if text_len > 0 else 1
+
+ threshold = self.threshold
+ if tag_importance > 1:
+ threshold *= 0.8
+ if text_ratio > 0.4:
+ threshold *= 0.9
+ if link_ratio > 0.6:
+ threshold *= 1.2
+
+ return score < threshold
+
+ def _composite_score(self, node, m: _NodeMetrics, text_len: int, tag_len: int, link_text_len: int) -> float:
+ if self.min_word_threshold:
+ word_count = m.space_count + 1
+ if word_count < self.min_word_threshold:
+ return -1.0
+
+ cfg = self.metric_config
+ w = self.metric_weights
+ score = 0.0
+ total = 0.0
+
+ if cfg["text_density"]:
+ density = text_len / tag_len if tag_len > 0 else 0
+ score += w["text_density"] * density
+ total += w["text_density"]
+
+ if cfg["link_density"]:
+ link_density = 1 - (link_text_len / text_len if text_len > 0 else 0)
+ score += w["link_density"] * link_density
+ total += w["link_density"]
+
+ if cfg["tag_weight"]:
+ score += w["tag_weight"] * self.tag_weights.get(node.tag, 0.5)
+ total += w["tag_weight"]
+
+ if cfg["class_id_weight"]:
+ # Faithful to the original: _class_id_score is <= 0, so max(0, .)
+ # contributes nothing -- it only dilutes the denominator.
+ score += w["class_id_weight"] * max(0, self._class_id_score(node))
+ total += w["class_id_weight"]
+
+ if cfg["text_length"]:
+ score += w["text_length"] * math.log(text_len + 1)
+ total += w["text_length"]
+
+ return score / total if total > 0 else 0
+
+ def _class_id_score(self, node) -> float:
+ score = 0.0
+ cls = node.get("class")
+ if cls is not None and self.negative_patterns.match(cls):
+ score -= 0.5
+ node_id = node.get("id")
+ if node_id is not None and self.negative_patterns.match(node_id):
+ score -= 0.5
+ return score
diff --git a/crawl4ai/types.py b/crawl4ai/types.py
index 72a0828e..b0cd4ff5 100644
--- a/crawl4ai/types.py
+++ b/crawl4ai/types.py
@@ -49,6 +49,7 @@ MarkdownGenerationResult = Union['MarkdownGenerationResultType']
# Content filter types
RelevantContentFilter = Union['RelevantContentFilterType']
PruningContentFilter = Union['PruningContentFilterType']
+PruningContentFilterLXML = Union['PruningContentFilterLXMLType']
BM25ContentFilter = Union['BM25ContentFilterType']
LLMContentFilter = Union['LLMContentFilterType']
@@ -152,6 +153,9 @@ if TYPE_CHECKING:
BM25ContentFilter as BM25ContentFilterType,
LLMContentFilter as LLMContentFilterType,
)
+ from .content_filter_strategy_lxml import (
+ PruningContentFilterLXML as PruningContentFilterLXMLType,
+ )
# Dispatcher imports
from .async_dispatcher import (
diff --git a/deploy/docker/api.py b/deploy/docker/api.py
index 1756b925..7fcf02e0 100644
--- a/deploy/docker/api.py
+++ b/deploy/docker/api.py
@@ -69,6 +69,7 @@ from crawl4ai.content_filter_strategy import (
BM25ContentFilter,
LLMContentFilter
)
+from crawl4ai.content_filter_strategy_lxml import PruningContentFilterLXML
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy
@@ -354,7 +355,7 @@ async def handle_markdown_request(
from llm_broker import resolve_llm
_llm = resolve_llm(config, provider)
content_filter = {
- FilterType.FIT: PruningContentFilter(),
+ FilterType.FIT: PruningContentFilterLXML(),
FilterType.BM25: BM25ContentFilter(user_query=query or ""),
FilterType.LLM: LLMContentFilter(
llm_config=LLMConfig(
diff --git a/tests/async/test_content_filter_prune_lxml.py b/tests/async/test_content_filter_prune_lxml.py
new file mode 100644
index 00000000..afc1226f
--- /dev/null
+++ b/tests/async/test_content_filter_prune_lxml.py
@@ -0,0 +1,209 @@
+"""Tests for the lxml-native PruningContentFilterLXML.
+
+Two concerns:
+ 1. The new filter satisfies the same behavioral contract as the original
+ (ported from test_content_filter_prune.py).
+ 2. The new filter is output-equivalent to the original (bs4) implementation
+ across a battery of HTML shapes and configs -- this is the fidelity guard
+ for the lxml rewrite.
+"""
+import os
+import sys
+import time
+
+import pytest
+from bs4 import BeautifulSoup
+
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy_lxml import PruningContentFilterLXML
+
+
+# --------------------------------------------------------------------------- #
+# Fixtures
+# --------------------------------------------------------------------------- #
+def basic_html():
+ return """
+ <html><body><article>
+ <h1>Main Article</h1>
+ <p>This is a high-quality paragraph with substantial text content. It contains enough words to pass the threshold and has good text density without too many links. This kind of content should survive the pruning process.</p>
+ <div class="sidebar">Low quality sidebar content</div>
+ <div class="social-share">Share buttons</div>
+ </article></body></html>
+ """
+
+
+def link_heavy_html():
+ return """
+ <html><body><div class="content">
+ <p>Good content paragraph that should remain.</p>
+ <div class="links">
+ <a href="#">Link 1</a><a href="#">Link 2</a><a href="#">Link 3</a><a href="#">Link 4</a>
+ </div>
+ </div></body></html>
+ """
+
+
+def mixed_content_html():
+ return """
+ <html><body><article>
+ <h1>Article Title</h1>
+ <p class="summary">Short summary.</p>
+ <div class="content">
+ <p>Long high-quality paragraph with substantial content that should definitely survive the pruning process. This content has good text density and proper formatting which makes it valuable for retention.</p>
+ </div>
+ <div class="comments"><p>Short comment 1</p><p>Short comment 2</p></div>
+ </article></body></html>
+ """
+
+
+# Battery of shapes that stress the metric reproduction (void elements,
+# entities, nested/!direct anchors, text around removed tags, comments,
+# unicode, tables, deep nesting, attributes).
+EQUIVALENCE_CASES = {
+ "void_img": '<body><div><p>text with <img src="a.png?x=1&y=2" alt="hi"> image content here now</p></div></body>',
+ "void_mix": '<body><div>alpha<br>beta<hr><input name="q" value="z"> gamma content text</div></body>',
+ "entities_text": '<body><div><p>Tom &amp; Jerry &lt;tag&gt; &copy; 2024 &nbsp; end of paragraph text</p></div></body>',
+ "entities_attr": '<body><div><a href="/s?q=a&b=c&d=e">a very long anchor text node here</a> body text follows</div></body>',
+ "nested_a_span": '<body><div class="x"><a href="#"><span>nested link text inside span</span></a> tail text after anchor</div></body>',
+ "nested_a_multi": '<body><div><a href="#">part one <b>bold</b> part two</a> and more body text content</div></body>',
+ "a_not_direct": '<body><div><p>wrapper text <a href="#">deep link</a> more wrapper text content here</p></div></body>',
+ "tail_after_nav": '<body><div>before text content <nav>NAVIGATION MENU</nav> after text continues here now</div></body>',
+ "tail_after_script": '<body><div>start text <script>var x=1;</script> middle part <style>.a{}</style> end content text</div></body>',
+ "comment_interspersed": '<body><div>alpha text <!-- a comment --> beta text <!--another--> gamma delta content</div></body>',
+ "pre_whitespace": '<body><pre> line one\n line two\n line three with content </pre></body>',
+ "table": '<body><table><tr><td>cell one content</td><td>cell two content</td></tr><tr><td>three</td><td>four</td></tr></table></body>',
+ "unicode_emoji": '<body><div><p>Héllo wörld 你好 مرحبا 🚀🔥 emoji test paragraph content</p></div></body>',
+ "nbsp": '<body><div><p>&nbsp;&nbsp;&nbsp;</p><p>actual real content paragraph text goes here now today</p></div></body>',
+ "many_attrs": '<body><div id="main" class="a b c" data-x="1" data-y="2&3" role="main"><p>paragraph text content here for the win today</p></div></body>',
+ "header_footer_aside": '<body><header>HEAD</header><main><p>Main content paragraph that is substantial enough to keep around here.</p></main><aside>ASIDE JUNK</aside><footer>FOOT</footer></body>',
+ "links_vs_content": '<body><div class="links">' + ''.join(f'<a href="#">L{i}</a>' for i in range(20)) + '</div><div class="content"><p>Good paragraph with real content that should clearly survive pruning here today.</p></div></body>',
+ "deep_nested": '<body>' + '<div>' * 300 + 'deep content text that is long enough to matter here' + '</div>' * 300 + '</body>',
+ "malformed": "<div>Unclosed div<p>Nested<span>content that keeps going on here</div>",
+}
+
+CONFIGS = [
+ {},
+ {"threshold_type": "dynamic"},
+ {"threshold_type": "dynamic", "threshold": 0.45},
+ {"threshold": 0.3},
+ {"threshold": 0.7},
+ {"min_word_threshold": 3},
+ {"min_word_threshold": 10},
+]
+
+
+def _norm_block_texts(blocks):
+ """Visible text of each block, whitespace-normalized (serialization-agnostic)."""
+ return [" ".join(BeautifulSoup(b, "lxml").get_text(" ").split()) for b in blocks]
+
+
+# --------------------------------------------------------------------------- #
+# Behavioral contract (ported, run against the NEW class)
+# --------------------------------------------------------------------------- #
+class TestPruningContentFilterLXMLBehavior:
+ def test_basic_pruning(self, basic_html):
+ contents = PruningContentFilterLXML(min_word_threshold=5).filter_content(basic_html)
+ combined = " ".join(contents).lower()
+ assert "high-quality paragraph" in combined
+ assert "sidebar content" not in combined
+ assert "share buttons" not in combined
+
+ def test_min_word_threshold(self, mixed_content_html):
+ contents = PruningContentFilterLXML(min_word_threshold=10).filter_content(mixed_content_html)
+ combined = " ".join(contents).lower()
+ assert "short summary" not in combined
+ assert "long high-quality paragraph" in combined
+ assert "short comment" not in combined
+
+ def test_link_density_impact(self, link_heavy_html):
+ contents = PruningContentFilterLXML(threshold_type="dynamic").filter_content(link_heavy_html)
+ combined = " ".join(contents).lower()
+ assert "good content paragraph" in combined
+ assert len([c for c in contents if "href" in c]) < 2
+
+ def test_tag_importance(self, mixed_content_html):
+ contents = PruningContentFilterLXML(threshold_type="dynamic").filter_content(mixed_content_html)
+ assert any("article" in c.lower() for c in contents) or any("h1" in c.lower() for c in contents)
+
+ def test_empty_input(self):
+ f = PruningContentFilterLXML()
+ assert f.filter_content("") == []
+ assert f.filter_content(None) == []
+
+ def test_malformed_html(self):
+ contents = PruningContentFilterLXML().filter_content("<div>Unclosed div<p>Nested<span>content</div>")
+ assert isinstance(contents, list)
+
+ @pytest.mark.parametrize("threshold,expected_count", [(0.3, 4), (0.48, 2), (0.7, 1)])
+ def test_threshold_levels(self, mixed_content_html, threshold, expected_count):
+ contents = PruningContentFilterLXML(threshold_type="fixed", threshold=threshold).filter_content(mixed_content_html)
+ assert len(contents) <= expected_count
+
+ def test_consistent_output(self, basic_html):
+ f = PruningContentFilterLXML()
+ assert f.filter_content(basic_html) == f.filter_content(basic_html)
+
+
+# --------------------------------------------------------------------------- #
+# Fidelity: new output must match the original bs4 implementation
+# --------------------------------------------------------------------------- #
+class TestPruningEquivalence:
+ @pytest.mark.parametrize("name", list(EQUIVALENCE_CASES.keys()))
+ @pytest.mark.parametrize("config", CONFIGS)
+ def test_blocks_match_original(self, name, config):
+ html = EQUIVALENCE_CASES[name]
+ old = PruningContentFilter(**config).filter_content(html)
+ new = PruningContentFilterLXML(**config).filter_content(html)
+ assert _norm_block_texts(old) == _norm_block_texts(new), (
+ f"{name} {config}\n old={_norm_block_texts(old)}\n new={_norm_block_texts(new)}"
+ )
+
+ def test_fixture_files_match_original(self):
+ """If saved server fixtures exist, diff old vs new on them too (large)."""
+ fixtures = os.environ.get("PRUNE_FIXTURES_DIR")
+ if not fixtures or not os.path.isdir(fixtures):
+ pytest.skip("no PRUNE_FIXTURES_DIR set")
+ for fn in sorted(os.listdir(fixtures)):
+ if not fn.endswith(".html"):
+ continue
+ html = open(os.path.join(fixtures, fn)).read()
+ for config in CONFIGS:
+ old = PruningContentFilter(**config).filter_content(html)
+ new = PruningContentFilterLXML(**config).filter_content(html)
+ assert _norm_block_texts(old) == _norm_block_texts(new), f"{fn} {config}"
+
+
+# --------------------------------------------------------------------------- #
+# Performance: new must be substantially faster on a non-trivial page
+# --------------------------------------------------------------------------- #
+def test_performance_faster_than_original():
+ # ~600 repeated cards -> a few hundred KB of HTML
+ html = "<body>" + "".join(
+ f'<div class="card"><h3>Card {i}</h3>'
+ f'<p>Some descriptive paragraph text for card number {i} with enough words to score.</p>'
+ f'<div class="meta"><a href="#">tag-a</a><a href="#">tag-b</a></div></div>'
+ for i in range(600)
+ ) + "</body>"
+
+ def timed(cls):
+ f = cls()
+ t0 = time.perf_counter()
+ out = f.filter_content(html)
+ return time.perf_counter() - t0, out
+
+ old_t, old_out = timed(PruningContentFilter)
+ new_t, new_out = timed(PruningContentFilterLXML)
+
+ assert _norm_block_texts(old_out) == _norm_block_texts(new_out)
+ # Expect a large margin; assert a conservative 2x to avoid flakiness.
+ assert new_t < old_t / 2, f"new={new_t*1000:.1f}ms not <2x faster than old={old_t*1000:.1f}ms"
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-q"])