Skip to content

A fast, helpful, and open-source document parser

improve searchItems detection

Logan Markewichd341371eae7c
diff --git a/.changeset/whole-owls-poke.md b/.changeset/whole-owls-poke.md
new file mode 100644
index 0000000..9932961
--- /dev/null
+++ b/.changeset/whole-owls-poke.md
@@ -0,0 +1,5 @@
+---
+"@llamaindex/liteparse": patch
+---
+
+Improve searchItems output on complex text
diff --git a/src/processing/searchItems.test.ts b/src/processing/searchItems.test.ts
index 7e43b1c..2326a57 100644
--- a/src/processing/searchItems.test.ts
+++ b/src/processing/searchItems.test.ts
@@ -2,8 +2,8 @@ import { describe, expect, it } from "vitest";
import { searchItems } from "./searchItems";
import { JsonTextItem } from "../core/types";
-function item(text: string, x: number, y: number, width: number, height = 12): JsonTextItem {
- return { text, x, y, width, height };
+function item(text: string, x: number, y: number, width: number, height = 12, fontSize = 12): JsonTextItem {
+ return { text, x, y, width, height, fontSize };
}
describe("searchItems", () => {
@@ -53,6 +53,38 @@ describe("searchItems", () => {
expect(results).toHaveLength(0);
});
+ it("matches spatially adjacent items without inserting a space", () => {
+ // "29-CA-" and "261755" are adjacent (gap=0), so joined as "29-CA-261755"
+ // "Case No." and "29-CA-" have a word gap (gap=5 > tolerance), so space inserted
+ const items = [
+ item("Case No.", 10, 50, 60),
+ item("29-CA-", 75, 50, 50),
+ item("261755", 125, 50, 50),
+ ];
+ const results = searchItems(items, { phrase: "Case No. 29-CA-261755" });
+ expect(results).toHaveLength(1);
+ expect(results[0].text).toBe("Case No. 29-CA-261755");
+ });
+
+ it("matches adjacent items with en-dash", () => {
+ // "pages 10–" and "20" are adjacent (gap=0)
+ const items = [item("pages 10\u2013", 10, 50, 60), item("20", 70, 50, 20)];
+ const results = searchItems(items, { phrase: "pages 10\u201320" });
+ expect(results).toHaveLength(1);
+ });
+
+ it("narrows correctly past adjacent items", () => {
+ // "prefix" has word gap to "29-CA-", "29-CA-" is adjacent to "261755"
+ const items = [
+ item("prefix", 10, 50, 40),
+ item("29-CA-", 55, 50, 50),
+ item("261755", 105, 50, 50),
+ ];
+ const results = searchItems(items, { phrase: "29-CA-261755" });
+ expect(results).toHaveLength(1);
+ expect(results[0].x).toBe(55);
+ });
+
it("merges bounding boxes vertically for wrapped phrases", () => {
const items = [item("temperature", 10, 50, 80, 12), item("range", 10, 65, 40, 12)];
const results = searchItems(items, { phrase: "temperature range" });
diff --git a/src/processing/searchItems.ts b/src/processing/searchItems.ts
index dd2801e..3d654f4 100644
--- a/src/processing/searchItems.ts
+++ b/src/processing/searchItems.ts
@@ -29,18 +29,33 @@ export function searchItems(items: JsonTextItem[], options: SearchItemsOptions):
const normalize = caseSensitive ? (s: string) => s : (s: string) => s.toLowerCase();
const q = normalize(options.phrase);
+ // Pre-compute separator between each pair of adjacent items.
+ // If two items are on the same line and the next item starts where the
+ // previous one ends (spatially adjacent), they are joined without a space.
+ // Otherwise a space is inserted.
+ const seps: string[] = new Array(items.length).fill("");
+ for (let i = 1; i < items.length; i++) {
+ const prev = items[i - 1];
+ const cur = items[i];
+ const fontSize = prev.fontSize ?? cur.fontSize ?? 12;
+ const sameLine = Math.abs(cur.y - prev.y) < fontSize * 0.5;
+ const gap = cur.x - (prev.x + prev.width);
+ seps[i] = sameLine && gap <= fontSize * 0.3 ? "" : " ";
+ }
+
let start = 0;
while (start < items.length) {
let combined = "";
let found = false;
for (let end = start; end < items.length; end++) {
- combined += (end > start ? " " : "") + items[end].text;
+ if (end > start) combined += seps[end];
+ combined += items[end].text;
if (normalize(combined).includes(q)) {
// Narrow from the left: drop leading items that aren't part of the match
let narrowed = combined;
let s = start;
while (s < end) {
- const without = narrowed.slice(items[s].text.length + 1);
+ const without = narrowed.slice(items[s].text.length + seps[s + 1].length);
if (normalize(without).includes(q)) {
narrowed = without;
s++;