A fast, helpful, and open-source document parser
improve searchItems detection
Logan Markewichd341371eae7c
diff --git a/.changeset/whole-owls-poke.md b/.changeset/whole-owls-poke.mdnew file mode 100644index 0000000..9932961--- /dev/null+++ b/.changeset/whole-owls-poke.md@@ -0,0 +1,5 @@+---+"@llamaindex/liteparse": patch+---++Improve searchItems output on complex textdiff --git a/src/processing/searchItems.test.ts b/src/processing/searchItems.test.tsindex 7e43b1c..2326a57 100644--- a/src/processing/searchItems.test.ts+++ b/src/processing/searchItems.test.ts@@ -2,8 +2,8 @@ import { describe, expect, it } from "vitest";import { searchItems } from "./searchItems";import { JsonTextItem } from "../core/types";-function item(text: string, x: number, y: number, width: number, height = 12): JsonTextItem {- return { text, x, y, width, height };+function item(text: string, x: number, y: number, width: number, height = 12, fontSize = 12): JsonTextItem {+ return { text, x, y, width, height, fontSize };}describe("searchItems", () => {@@ -53,6 +53,38 @@ describe("searchItems", () => {expect(results).toHaveLength(0);});+ it("matches spatially adjacent items without inserting a space", () => {+ // "29-CA-" and "261755" are adjacent (gap=0), so joined as "29-CA-261755"+ // "Case No." and "29-CA-" have a word gap (gap=5 > tolerance), so space inserted+ const items = [+ item("Case No.", 10, 50, 60),+ item("29-CA-", 75, 50, 50),+ item("261755", 125, 50, 50),+ ];+ const results = searchItems(items, { phrase: "Case No. 29-CA-261755" });+ expect(results).toHaveLength(1);+ expect(results[0].text).toBe("Case No. 29-CA-261755");+ });++ it("matches adjacent items with en-dash", () => {+ // "pages 10–" and "20" are adjacent (gap=0)+ const items = [item("pages 10\u2013", 10, 50, 60), item("20", 70, 50, 20)];+ const results = searchItems(items, { phrase: "pages 10\u201320" });+ expect(results).toHaveLength(1);+ });++ it("narrows correctly past adjacent items", () => {+ // "prefix" has word gap to "29-CA-", "29-CA-" is adjacent to "261755"+ const items = [+ item("prefix", 10, 50, 40),+ item("29-CA-", 55, 50, 50),+ item("261755", 105, 50, 50),+ ];+ const results = searchItems(items, { phrase: "29-CA-261755" });+ expect(results).toHaveLength(1);+ expect(results[0].x).toBe(55);+ });+it("merges bounding boxes vertically for wrapped phrases", () => {const items = [item("temperature", 10, 50, 80, 12), item("range", 10, 65, 40, 12)];const results = searchItems(items, { phrase: "temperature range" });diff --git a/src/processing/searchItems.ts b/src/processing/searchItems.tsindex dd2801e..3d654f4 100644--- a/src/processing/searchItems.ts+++ b/src/processing/searchItems.ts@@ -29,18 +29,33 @@ export function searchItems(items: JsonTextItem[], options: SearchItemsOptions):const normalize = caseSensitive ? (s: string) => s : (s: string) => s.toLowerCase();const q = normalize(options.phrase);+ // Pre-compute separator between each pair of adjacent items.+ // If two items are on the same line and the next item starts where the+ // previous one ends (spatially adjacent), they are joined without a space.+ // Otherwise a space is inserted.+ const seps: string[] = new Array(items.length).fill("");+ for (let i = 1; i < items.length; i++) {+ const prev = items[i - 1];+ const cur = items[i];+ const fontSize = prev.fontSize ?? cur.fontSize ?? 12;+ const sameLine = Math.abs(cur.y - prev.y) < fontSize * 0.5;+ const gap = cur.x - (prev.x + prev.width);+ seps[i] = sameLine && gap <= fontSize * 0.3 ? "" : " ";+ }+let start = 0;while (start < items.length) {let combined = "";let found = false;for (let end = start; end < items.length; end++) {- combined += (end > start ? " " : "") + items[end].text;+ if (end > start) combined += seps[end];+ combined += items[end].text;if (normalize(combined).includes(q)) {// Narrow from the left: drop leading items that aren't part of the matchlet narrowed = combined;let s = start;while (s < end) {- const without = narrowed.slice(items[s].text.length + 1);+ const without = narrowed.slice(items[s].text.length + seps[s + 1].length);if (normalize(without).includes(q)) {narrowed = without;s++;