Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): end HTML paragraphs with a blank line so sections break between them

Tom Villani9c311659439a
diff --git a/engine/crates/xerj-autoindex/src/extract/html.rs b/engine/crates/xerj-autoindex/src/extract/html.rs
index 7985a3a81..6d59b2030 100644
--- a/engine/crates/xerj-autoindex/src/extract/html.rs
+++ b/engine/crates/xerj-autoindex/src/extract/html.rs
@@ -279,9 +279,7 @@ fn parse(html: &str) -> Doc {
// text never reaches the body any other way, and a
// table that is not the dominant one is not emitted
// as rows. An empty cell keeps its column.
- if !doc.body.is_empty() && !doc.body.ends_with('\n') {
- doc.body.push('\n');
- }
+ line_break(&mut doc.body, 2);
for row in &cur_table {
if row.iter().any(|c| !c.is_empty()) {
doc.body.push_str(&row.join(" | "));
@@ -312,16 +310,16 @@ fn parse(html: &str) -> Doc {
cur_row_th.push(name == "th");
cur_cell = Some(String::new());
}
- (false, "br")
- | (false, "p")
- | (true, "p")
- | (false, "div")
- | (true, "div")
- | (false, "li")
- | (true, "tr")
- if !doc.body.ends_with('\n') && !doc.body.is_empty() =>
- {
- doc.body.push('\n');
+ // Block boundaries are blank lines: `split_sections` packs
+ // paragraphs at `\n\n`, so a single newline here made a page
+ // of paragraphs one paragraph, hard-cut mid-word. Line-level
+ // breaks (`br`, list items, table rows) stay single newlines,
+ // so a list or a table stays one block.
+ (_, "p") | (_, "div") | (true, "ul") | (true, "ol") => {
+ line_break(&mut doc.body, 2);
+ }
+ (false, "br") | (false, "li") | (true, "tr") => {
+ line_break(&mut doc.body, 1);
}
_ => {}
}
@@ -432,6 +430,20 @@ fn raw_text_end(bytes: &[u8], from: usize, name: &[u8]) -> usize {
bytes.len()
}
+/// End the body's current line (`n` = 1) or paragraph (`n` = 2): drop the
+/// space `flush_text` leaves after a run of text, then make the body end in
+/// at least `n` newlines. Nothing is added to an empty body.
+fn line_break(body: &mut String, n: usize) {
+ body.truncate(body.trim_end_matches(' ').len());
+ if body.is_empty() {
+ return;
+ }
+ let have = body.len() - body.trim_end_matches('\n').len();
+ for _ in have..n {
+ body.push('\n');
+ }
+}
+
fn normalize_ws(s: &str) -> String {
s.split_whitespace().collect::<Vec<_>>().join(" ")
}
@@ -767,6 +779,38 @@ mod tests {
);
}
+ /// `split_sections` packs paragraphs at blank lines. A `<p>` used to end
+ /// with a single newline, so a page of paragraphs was ONE paragraph to it
+ /// and was hard-cut every `SECTION_CHARS` bytes, mid-word; the heading
+ /// before such a run was emitted as a section on its own. Paragraph and
+ /// div ends are now blank lines, so sections end where paragraphs do.
+ #[test]
+ fn sections_of_a_long_page_end_at_paragraph_ends() {
+ let para = |i: usize| format!("Paragraph {i} {} end{i}.", "lorem ipsum dolor ".repeat(24));
+ let html: String = std::iter::once("<html><body><h1>Title</h1>".to_string())
+ .chain((0..10).map(|i| format!("<p>{}</p>", para(i))))
+ .chain(std::iter::once(
+ "<ul><li>one</li><li>two</li></ul><p>tail</p></body></html>".into(),
+ ))
+ .collect();
+ let (stats, recs) = run("long.html", &html);
+ assert!(stats.records > 1, "{} bytes split", html.len());
+ for r in &recs[..recs.len() - 1] {
+ let body = body_of(r).trim_end();
+ assert!(
+ (0..10).any(|i| body.ends_with(&format!("end{i}."))),
+ "section ends mid-paragraph: ...{:?}",
+ &body[body.len().saturating_sub(40)..]
+ );
+ }
+ assert!(
+ body_of(&recs[0]).len() > 100,
+ "the heading is not a section of its own"
+ );
+ let last = body_of(recs.last().unwrap());
+ assert!(last.contains("one\ntwo"), "list items stay lines: {last:?}");
+ }
+
#[test]
fn a_page_without_a_title_falls_back_to_a_heading_then_to_the_file_stem() {
let (_, recs) = run(