Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(tables): expand rowspan/colspan into a grid and keep <th> row headers
ntohididd542e7550f7
diff --git a/crawl4ai/table_extraction.py b/crawl4ai/table_extraction.pyindex 7edb3b76..2379872f 100644--- a/crawl4ai/table_extraction.py+++ b/crawl4ai/table_extraction.py@@ -210,6 +210,39 @@ class DefaultTableExtraction(TableExtractionStrategy):threshold = kwargs.get("table_score_threshold", self.table_score_threshold)return score >= threshold+ @staticmethod+ def _build_grid(rows: List[etree.Element]) -> List[List[str]]:+ """Lay <tr> cells into a rectangular grid, honouring colspan/rowspan.++ Each cell writes its text into every slot of the rectangle it spans,+ including slots in later rows, so a row only has to fill the slots the+ rows above it left free. Nothing is carried between iterations, so a+ short row cannot leave a stale span behind (see issue #2258).+ """+ grid: List[List[str]] = [[] for _ in rows]++ def put(r: int, c: int, text: str) -> None:+ if r >= len(grid): # rowspan reaching past the last row+ return+ row = grid[r]+ row.extend([None] * (c + 1 - len(row)))+ row[c] = text++ for r, tr in enumerate(rows):+ c = 0+ for cell in tr.xpath("./th|./td"):+ while c < len(grid[r]) and grid[r][c] is not None:+ c += 1 # slot already taken by a rowspan from above+ text = cell.text_content().strip()+ colspan = max(int(cell.get("colspan") or 1), 1)+ rowspan = max(int(cell.get("rowspan") or 1), 1)+ for dr in range(rowspan):+ for dc in range(colspan):+ put(r + dr, c + dc, text)+ c += colspan++ return [[text or "" for text in row] for row in grid if row]+def extract_table_data(self, table: etree.Element) -> Dict[str, Any]:"""Extract structured data from a table element.@@ -232,6 +265,7 @@ class DefaultTableExtraction(TableExtractionStrategy):# Extract headers with colspan handlingheaders = []+ implicit_header_row = Nonethead_rows = table.xpath(".//thead/tr")if thead_rows:header_cells = thead_rows[0].xpath(".//th")@@ -243,21 +277,21 @@ class DefaultTableExtraction(TableExtractionStrategy):# Check first row for headersfirst_row = table.xpath(".//tr[1]")if first_row:- for cell in first_row[0].xpath(".//th|.//td"):+ implicit_header_row = first_row[0]+ for cell in implicit_header_row.xpath(".//th|.//td"):text = cell.text_content().strip()colspan = int(cell.get("colspan", 1))headers.extend([text] * colspan)- # Extract rows with colspan handling- rows = []- for row in table.xpath(".//tr[not(ancestor::thead)]"):- row_data = []- for cell in row.xpath(".//td"):- text = cell.text_content().strip()- colspan = int(cell.get("colspan", 1))- row_data.extend([text] * colspan)- if row_data:- rows.append(row_data)+ # Extract rows, laying each cell into a grid so colspan/rowspan land in+ # the columns they actually cover and <th> row headers are kept (#2258).+ body_rows = table.xpath(".//tr[not(ancestor::thead)]")+ # A first row made only of <th> is the header, so don't repeat it as+ # data. A first row mixing <th scope="row"> with <td> is a key/value+ # data row and must stay.+ if implicit_header_row is not None and not implicit_header_row.xpath("./td"):+ body_rows = body_rows[1:]+ rows = self._build_grid(body_rows)# Align rows with headersmax_columns = len(headers) if headers else (diff --git a/tests/test_table_rowspan_grid.py b/tests/test_table_rowspan_grid.pynew file mode 100644index 00000000..a54724da--- /dev/null+++ b/tests/test_table_rowspan_grid.py@@ -0,0 +1,123 @@+"""Table grid expansion: <th> row headers, colspan and rowspan.++Issue #2258: body rows were collected with `.//td`, so a `<th scope="row">`+key column was dropped and every following cell shifted left; `rowspan` was+ignored entirely, so a spanning value never reached the rows it covers.++https://github.com/unclecode/crawl4ai/issues/2258+"""++from lxml import html as lhtml++from crawl4ai import DefaultTableExtraction+++def _extract(html: str):+ # Wrap the snippet so fromstring always returns a container element.+ return DefaultTableExtraction().extract_tables(lhtml.fromstring(f"<div>{html}</div>"))+++def test_th_row_headers_in_body_are_kept():+ """A <th scope="row"> key column must stay the first column."""+ tables = _extract("""+ <table>+ <tr><th></th><th>Feature A</th><th>Feature B</th></tr>+ <tr><th scope="row">Item 1</th><td>yes</td><td>yes</td></tr>+ <tr><th scope="row">Item 2</th><td>no</td><td>yes</td></tr>+ </table>""")+ assert len(tables) == 1+ assert tables[0]["headers"] == ["", "Feature A", "Feature B"]+ assert tables[0]["rows"] == [["Item 1", "yes", "yes"], ["Item 2", "no", "yes"]]+++def test_rowspan_value_repeats_into_covered_rows():+ """A rowspan cell's value appears in every row it covers."""+ tables = _extract("""+ <table>+ <thead><tr><th>Group</th><th>Option X</th><th>Option Y</th></tr></thead>+ <tbody>+ <tr><td rowspan="2">Group 1</td><td>value x</td><td>value y</td></tr>+ <tr><td colspan="2">note that applies to X and Y</td></tr>+ </tbody></table>""")+ assert len(tables) == 1+ assert tables[0]["headers"] == ["Group", "Option X", "Option Y"]+ assert tables[0]["rows"] == [+ ["Group 1", "value x", "value y"],+ ["Group 1", "note that applies to X and Y", "note that applies to X and Y"],+ ]+++def test_rowspan_does_not_leak_past_a_short_row():+ """A span in the last column must land in the short row, not the next one.++ Regression guard: carrying rowspan state between rows and advancing it+ cell-by-cell stops early on a short row, leaving the span live so it+ overwrites a real value one row further down.+ """+ tables = _extract("""+ <table>+ <thead><tr><th>H1</th><th>H2</th><th>H3</th></tr></thead>+ <tbody>+ <tr><td>a</td><td>b</td><td rowspan="2">SPAN</td></tr>+ <tr><td>c</td></tr>+ <tr><td>x</td><td>y</td><td>z</td></tr>+ </tbody></table>""")+ assert tables[0]["rows"] == [+ ["a", "b", "SPAN"],+ ["c", "", "SPAN"],+ ["x", "y", "z"],+ ]+++def test_rowspan_and_colspan_on_the_same_cell():+ """A cell spanning both ways fills the whole rectangle it covers."""+ tables = _extract("""+ <table>+ <thead><tr><th>A</th><th>B</th><th>C</th><th>D</th></tr></thead>+ <tbody>+ <tr><td rowspan="2" colspan="2">BIG</td><td>c1</td><td>d1</td></tr>+ <tr><td>c2</td><td>d2</td></tr>+ </tbody></table>""")+ assert tables[0]["rows"] == [+ ["BIG", "BIG", "c1", "d1"],+ ["BIG", "BIG", "c2", "d2"],+ ]+++def test_rowspan_reaching_past_the_last_row_is_clamped():+ """An over-long rowspan must not invent rows or raise."""+ tables = _extract("""+ <table>+ <thead><tr><th>A</th><th>B</th></tr></thead>+ <tbody>+ <tr><td rowspan="9">X</td><td>b1</td></tr>+ <tr><td>b2</td></tr>+ </tbody></table>""")+ assert tables[0]["rows"] == [["X", "b1"], ["X", "b2"]]+++def test_plain_table_without_spans_is_unchanged():+ """Regression guard for the common case."""+ tables = _extract("""+ <table>+ <thead><tr><th>Name</th><th>Qty</th><th>Price</th></tr></thead>+ <tbody>+ <tr><td>Widget</td><td>2</td><td>10</td></tr>+ <tr><td>Gadget</td><td>5</td><td>25</td></tr>+ </tbody></table>""")+ assert tables[0]["headers"] == ["Name", "Qty", "Price"]+ assert tables[0]["rows"] == [["Widget", "2", "10"], ["Gadget", "5", "25"]]+++def test_key_value_first_row_is_not_mistaken_for_a_header():+ """A first row of <th scope="row"> + <td> is data, not a header row.++ Real pages (Wikipedia infoboxes) open a two-column key/value table with+ such a row. Treating it as the header would drop it from `rows`.+ """+ tables = _extract("""+ <table>+ <tr><th scope="row">Food</th><td>Apple pie</td></tr>+ <tr><th scope="row">Drink</th><td>Apple cider</td></tr>+ </table>""")+ assert tables[0]["rows"] == [["Food", "Apple pie"], ["Drink", "Apple cider"]]