Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

feat(chat): open cited documents at the cited passage (#3700)

lyingbug1bf717a54d44
diff --git a/docreader/main.py b/docreader/main.py
index 51f2cd0c4..5098c4f23 100644
--- a/docreader/main.py
+++ b/docreader/main.py
@@ -26,6 +26,7 @@ from docreader.proto.docreader_pb2 import (
ListEnginesResponse,
ParserEngineInfo,
)
+from docreader.source_wire import source_blocks_to_proto
from docreader.utils.request import init_logging_request_id, request_id_context
_SURROGATE_RE = re.compile(r"[\ud800-\udfff]")
@@ -209,6 +210,7 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
metadata={k: _c(str(v)) for k, v in result.metadata.items()}
if result.metadata
else {},
+ source_blocks=source_blocks_to_proto(result),
)
logger.info(
"Read response: content_len=%d, images=%d",
@@ -258,6 +260,7 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
if result.metadata
else {},
image_count=image_count,
+ source_blocks=source_blocks_to_proto(result),
)
)
diff --git a/docreader/models/document.py b/docreader/models/document.py
index 1ab8c46e9..a165e1c77 100644
--- a/docreader/models/document.py
+++ b/docreader/models/document.py
@@ -70,6 +70,16 @@ class Document(BaseModel):
)
chunks: List[Chunk] = Field(default_factory=list, description="document chunks")
+ source_blocks: List[Dict[str, Any]] = Field(
+ default_factory=list,
+ description=(
+ "Ranges of content mapped to positions in the original file: "
+ '{"start": int, "end": int, "locator": {"type": "pdf", "page": 1, '
+ '"bbox": [x0, y0, x1, y1]}}. Offsets are code point indices into '
+ "content, end exclusive; ordinals are 1-based; bbox is fractions of "
+ "the page with the origin at the top-left corner."
+ ),
+ )
metadata: Dict[str, Any] = Field(
default_factory=dict,
description="metadata fields",
diff --git a/docreader/parser/excel_parser.py b/docreader/parser/excel_parser.py
index e302a7fcf..48e37819f 100644
--- a/docreader/parser/excel_parser.py
+++ b/docreader/parser/excel_parser.py
@@ -93,6 +93,7 @@ class ExcelParser(BaseParser):
"""
chunks: List[Chunk] = []
text: List[str] = []
+ source_blocks: List[dict] = []
start, end = 0, 0
excel_file = _open_excel_file(content, file_type=self.file_type)
@@ -107,8 +108,10 @@ class ExcelParser(BaseParser):
# Remove rows where all values are NaN (completely empty rows)
df.dropna(how="all", inplace=True)
- # Process each row in the DataFrame
- for _, row in df.iterrows():
+ # Process each row in the DataFrame. The index is the 0-based
+ # sheet row (rows are read from row 1 and dropped rows keep
+ # their labels), so index + 1 is the row number users see.
+ for row_index, row in df.iterrows():
page_content = []
# Build key-value pairs for non-null values
for k, v in row.items():
@@ -128,10 +131,29 @@ class ExcelParser(BaseParser):
chunks.append(
Chunk(content=content_row, seq=len(chunks), start=start, end=end)
)
+ try:
+ row_number = int(row_index) + 1
+ except (TypeError, ValueError):
+ row_number = 0
+ if row_number > 0:
+ source_blocks.append(
+ {
+ "start": start,
+ "end": end,
+ "locator": {
+ "type": "sheet",
+ "sheet": str(excel_sheet_name),
+ "row_start": row_number,
+ "row_end": row_number,
+ },
+ }
+ )
start = end
# Combine all text and return as Document
- return Document(content="".join(text), chunks=chunks)
+ return Document(
+ content="".join(text), chunks=chunks, source_blocks=source_blocks
+ )
def _read_sheet_dataframe(
diff --git a/docreader/parser/pdf_parser.py b/docreader/parser/pdf_parser.py
index 4a5419652..1b31db726 100644
--- a/docreader/parser/pdf_parser.py
+++ b/docreader/parser/pdf_parser.py
@@ -28,6 +28,7 @@ from docreader.config import CONFIG
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.parser.concurrency import parser_worker_limit
+from docreader.parser.source_locator import PageGeometry, page_blocks
logger = logging.getLogger(__name__)
@@ -134,6 +135,12 @@ MAX_FIGURE_HEIGHT_RATIO = _env_float("DOCREADER_PDF_MAX_FIGURE_HEIGHT_RATIO", 0.
# Can be overridden per-upload via parser_engine_overrides.pdf_force_scanned.
FORCE_SCANNED_PDF = _env_bool("DOCREADER_PDF_FORCE_SCANNED", False)
+# --- Source locators ---------------------------------------------------------
+# Record where each paragraph of a native text page sits on the page, so a
+# citation can open the PDF there and outline the paragraph. Costs one glyph
+# pass per text page; disable to only record page numbers.
+SOURCE_BOXES = _env_bool("DOCREADER_PDF_SOURCE_BOXES", True)
+
# pdfium / Adobe text layers often emit U+FFFE for missing hyphenation or ligatures.
_PDF_ARTIFACT_RE = re.compile(r"[\u00ad\u200b-\u200f\ufeff\ufffe\uffff]")
_PDF_ARTIFACT_JOIN_RE = re.compile(r"(\w)[\u00ad\ufffe](\w)")
@@ -629,12 +636,13 @@ def _point_in_boxes(x: float, y: float, boxes: list) -> bool:
return False
-def _page_chars(textpage, page, raw) -> tuple:
+def _page_chars(textpage, page, raw, loose: bool = False) -> tuple:
"""Return ``(chars, page_width)`` with hidden/off-page glyphs filtered.
Working at the glyph level (instead of pdfium rect segments) keeps mixed
CJK + Latin/number lines in their true left-to-right order, which the
- rect-level ``get_text_bounded`` API scrambles.
+ rect-level ``get_text_bounded`` API scrambles. ``loose`` boxes span the
+ font's ascent to descent, so every glyph of a line shares one height.
"""
n = textpage.count_chars()
if n <= 0:
@@ -645,7 +653,7 @@ def _page_chars(textpage, page, raw) -> tuple:
chars: list = []
for i in range(n):
try:
- left, bottom, right, top = textpage.get_charbox(i)
+ left, bottom, right, top = textpage.get_charbox(i, loose=loose)
except Exception:
continue
ch = textpage.get_text_range(i, 1)
@@ -658,7 +666,7 @@ def _page_chars(textpage, page, raw) -> tuple:
continue # off-page glyph
if invisible and _point_in_boxes((x0 + x1) / 2, (y0 + y1) / 2, invisible):
continue # covered by an invisible text object
- chars.append({"x0": x0, "y0": y0, "x1": x1, "y1": y1, "ch": ch})
+ chars.append({"x0": x0, "y0": y0, "x1": x1, "y1": y1, "ch": ch, "i": i})
return chars, width
@@ -988,6 +996,81 @@ def _extract_layout_text(page, raw) -> str:
_close_pdfium_resource(textpage)
+# A visual line is cut where a gap wider than one glyph height coincides with
+# a jump in the text stream: the gutter between two columns, not a word space.
+LINE_SEGMENT_GAP_RATIO = 1.0
+LINE_SEGMENT_STREAM_JUMP = 8
+LINE_SEGMENT_MIN_CHARS = 3
+
+
+def _split_line_at_gaps(line: dict) -> list:
+ """Cut a visual line where it crosses from one column into another.
+
+ Narrow two-column gutters (ACL/IEEE papers leave ~16pt) and anything that
+ spans them (a title, a wide table) defeat the full-height gutter search,
+ so glyphs of both columns land in one line; each side must be placed and
+ boxed on its own. Columns are far apart in the text stream while glyphs of
+ one line are consecutive, which keeps wide justified word spaces intact.
+ """
+ whole = [{"text": line["text"], "bbox": line["bbox"]}]
+ chars = [c for c in line["chars"] if not c["ch"].isspace()]
+ if len(chars) < 2 * LINE_SEGMENT_MIN_CHARS:
+ return whole
+ min_gap = max(max(c["y1"] - c["y0"] for c in chars), 1.0) * LINE_SEGMENT_GAP_RATIO
+ segments: list = [[chars[0]]]
+ for prev, cur in zip(chars, chars[1:]):
+ jump = abs(cur.get("i", 0) - prev.get("i", 0))
+ if cur["x0"] - prev["x1"] > min_gap and jump > LINE_SEGMENT_STREAM_JUMP:
+ segments.append([cur])
+ else:
+ segments[-1].append(cur)
+ if len(segments) == 1 or any(len(seg) < LINE_SEGMENT_MIN_CHARS for seg in segments):
+ return whole
+ out = []
+ for seg in segments:
+ text = _join_line_glyphs(seg)
+ if text.strip():
+ out.append({"text": text, "bbox": _chars_bbox(seg)})
+ return out or whole
+
+
+def _page_line_boxes(page, raw) -> list:
+ """Visual lines of a text page as ``{"text", "bbox"}`` in PDF points.
+
+ Lines are grouped per column so a two-column page does not fuse the
+ columns' glyphs into one line. Any failure yields no lines, which only
+ downgrades the page's locators to page numbers.
+ """
+ textpage = None
+ try:
+ textpage = page.get_textpage()
+ # Tight glyph boxes put "a" and "d" at different heights, which splits
+ # one line of text into fragments by letter shape.
+ chars, width = _page_chars(textpage, page, raw, loose=True)
+ if not chars:
+ return []
+ heights = [c["y1"] - c["y0"] for c in chars if c["y1"] - c["y0"] > 0]
+ scale = (statistics.median(heights) if heights else 1.0) or 1.0
+ lines = []
+ for col in _split_columns(chars, scale, width):
+ for ln in _group_lines_with_chars(col):
+ lines.extend(_split_line_at_gaps(ln))
+ return lines
+ except Exception:
+ logger.debug("line box extraction failed", exc_info=True)
+ return []
+ finally:
+ _close_pdfium_resource(textpage)
+
+
+def _page_geometry(page):
+ try:
+ return PageGeometry(page.get_cropbox(), page.get_rotation())
+ except Exception:
+ logger.debug("page geometry unavailable", exc_info=True)
+ return None
+
+
def _effective_scale(page, scale: float, max_edge: int) -> float:
"""Reduce ``scale`` so the rendered long edge never exceeds ``max_edge`` px.
@@ -1393,10 +1476,23 @@ class PDFScannedParser(BaseParser):
finally:
_close_pdfium_resource(pdf)
+ source_blocks = []
+ offset = 0
for i in range(page_count):
page_filename = f"{base_name}_page_{i+1}.jpg"
ref_path = f"images/{page_filename}"
- markdown_lines.append(f"![{page_filename}]({ref_path})")
+ line = f"![{page_filename}]({ref_path})"
+ if markdown_lines:
+ offset += 2
+ markdown_lines.append(line)
+ source_blocks.append(
+ {
+ "start": offset,
+ "end": offset + len(line),
+ "locator": {"type": "pdf", "page": i + 1},
+ }
+ )
+ offset += len(line)
images[ref_path] = base64.b64encode(rendered[i]).decode("utf-8")
text = "\n\n".join(markdown_lines)
@@ -1407,6 +1503,7 @@ class PDFScannedParser(BaseParser):
"image_source_type": "scanned_pdf",
"page_count": page_count,
},
+ source_blocks=source_blocks,
)
except Exception as e:
logger.exception("PDFScannedParser failed to parse PDF: %s", e)
@@ -1499,9 +1596,12 @@ class PDFParser(BaseParser):
texts: list = []
classes: list = []
vector_clips: dict = {}
+ line_boxes: dict = {}
+ geometries: dict = {}
for i in range(page_count):
page = pdf[i]
try:
+ geometries[i] = _page_geometry(page)
plain = _extract_page_text(page)
ratio = _page_image_area_ratio(page, pdfium_r)
cls = _classify_page(ratio, len(plain.strip()))
@@ -1534,6 +1634,8 @@ class PDFParser(BaseParser):
for ref_path, b64, _y, _cap in clips:
images[ref_path] = b64
text = _postprocess_pdf_text(text)
+ if cls == "text" and SOURCE_BOXES:
+ line_boxes[i] = _page_line_boxes(page, pdfium_r)
if cls == "text" and vector_clips.get(i):
text = _inject_figure_markdown_before_captions(
text, vector_clips[i]
@@ -1574,27 +1676,60 @@ class PDFParser(BaseParser):
finally:
_close_pdfium_resource(pdf)
- # Assemble markdown in reading order.
+ # Assemble markdown in reading order, recording where each page's
+ # text lands so citations can point back at the page (and paragraph).
embedded_count = 0
vector_figure_count = 0
blocks = []
+ source_blocks = []
+ offset = 0
+
+ def place(part: str) -> int:
+ nonlocal offset
+ if blocks:
+ offset += 2 # the "\n\n" separator
+ start = offset
+ blocks.append(part)
+ offset += len(part)
+ return start
+
for i in range(page_count):
+ page_only = {"type": "pdf", "page": i + 1}
if classes[i] == "scanned":
page_filename = f"{base_name}_page_{i+1}.jpg"
- blocks.append(f"![{page_filename}](images/{page_filename})")
+ part = f"![{page_filename}](images/{page_filename})"
+ start = place(part)
+ source_blocks.append(
+ {"start": start, "end": start + len(part), "locator": page_only}
+ )
else:
stripped = texts[i].strip()
if stripped:
- blocks.append(stripped)
+ start = place(stripped)
+ source_blocks.extend(
+ page_blocks(
+ stripped,
+ start,
+ i + 1,
+ line_boxes.get(i) or [],
+ geometries.get(i),
+ )
+ )
vector_figure_count += len(vector_clips.get(i, []))
page_images = list(embedded.get(i, []))
page_images.sort(key=lambda item: item[2], reverse=True)
for ref_path, _b64, _y in page_images:
fname = os.path.basename(ref_path)
- blocks.append(f"![{fname}]({ref_path})")
+ part = f"![{fname}]({ref_path})"
+ start = place(part)
+ source_blocks.append(
+ {"start": start, "end": start + len(part), "locator": dict(page_only)}
+ )
embedded_count += 1
- content_text = "\n\n".join(blocks).strip()
+ # Every part is already stripped, so the join needs no trimming and
+ # the recorded offsets stay exact.
+ content_text = "\n\n".join(blocks)
metadata = {
"page_count": page_count,
@@ -1615,4 +1750,9 @@ class PDFParser(BaseParser):
embedded_count,
len(content_text),
)
- return Document(content=content_text, images=images, metadata=metadata)
+ return Document(
+ content=content_text,
+ images=images,
+ metadata=metadata,
+ source_blocks=source_blocks,
+ )
diff --git a/docreader/parser/source_locator.py b/docreader/parser/source_locator.py
new file mode 100644
index 000000000..29ef3e9b2
--- /dev/null
+++ b/docreader/parser/source_locator.py
@@ -0,0 +1,293 @@
+"""Source locators: map ranges of parsed markdown back to the original file.
+
+A source block is ``{"start": int, "end": int, "locator": {...}}`` where
+``start``/``end`` are code point offsets into the document content (end
+exclusive) and the locator says where that text sits in the original file.
+For PDFs the locator is ``{"type": "pdf", "page": n}`` plus an optional
+``"bbox": [x0, y0, x1, y1]`` given as fractions of the displayed page with
+the origin at its top-left corner, so viewers can draw it at any zoom.
+"""
+
+from __future__ import annotations
+
+import re
+import statistics
+import unicodedata
+from typing import Iterable, List, Optional, Sequence, Tuple
+
+# How far past the cursor a line is searched for, in normalized characters.
+_SEARCH_WINDOW = 6000
+# How much of a line's normalized text is searched for.
+_KEY_CHARS = 24
+# Lines shorter than this (normalized) are too ambiguous to place.
+_MIN_KEY_CHARS = 2
+# Lines shorter than this are placed only when their text is unique on the
+# page: labels inside figures ("LLM", "EBAE") otherwise steal body offsets.
+# Wide (CJK) characters count as four, since each carries about a word.
+_SHORT_KEY_WEIGHT = 16
+
+_LIST_START_RE = re.compile(
+ r"^\s*(?:"
+ r"\d{1,3}(?:\.\d{1,3}){0,4}[\.、.))]?\s" # 1 1. 8.14.11 2)
+ r"|[((]\d{1,3}[))]" # (1) (1)
+ r"|[一二三四五六七八九十百]+[、..]" # 一、
+ r"|[•·●○■□◆◇▪\-–*]\s" # bullets
+ r"|第[一二三四五六七八九十百零\d]+[章节条款部分篇]" # 第三章
+ r")"
+)
+_SENTENCE_END = tuple("。..!?!?::;;」』”))")
+
+
+def _fold(ch: str) -> str:
+ """Letters and digits only, lower-cased, full-width folded."""
+ if not ch.isalnum():
+ return ""
+ return unicodedata.normalize("NFKC", ch).lower()
+
+
+def normalize_with_positions(text: str) -> Tuple[str, List[int]]:
+ """Project text onto letters and digits.
+
+ Returns the projection and, for each of its characters, the offset in
+ ``text`` of the character it came from.
+ """
+ out: List[str] = []
+ pos: List[int] = []
+ for i, ch in enumerate(text):
+ folded = _fold(ch)
+ for f in folded:
+ out.append(f)
+ pos.append(i)
+ return "".join(out), pos
+
+
+def normalize(text: str) -> str:
+ return "".join(_fold(ch) for ch in text)
+
+
+def _key_weight(key: str) -> int:
+ return sum(4 if unicodedata.east_asian_width(ch) in ("W", "F") else 1 for ch in key)
+
+
+def _column_cursor(boxes, idx: int, ends: List[Optional[int]]) -> Optional[int]:
+ """End of the nearest placed line above ``boxes[idx]`` in the same column."""
+ x0, y0, x1, _ = boxes[idx]
+ for j in range(idx - 1, max(-1, idx - 80), -1):
+ if ends[j] is None:
+ continue
+ px0, py0, px1, _ = boxes[j]
+ if px0 < x1 and x0 < px1 and py0 > y0:
+ return ends[j]
+ return None
+
+
+def locate_lines(
+ page_text: str,
+ line_texts: Sequence[str],
+ boxes: Optional[Sequence[Tuple[float, float, float, float]]] = None,
+) -> List[Optional[int]]:
+ """Find where each visual line starts in the page text.
+
+ Lines are searched in order after the previous hit so repeated phrases
+ resolve to the right occurrence. With ``boxes`` (PDF points, bottom-left
+ origin) the search first continues from the line above in the same
+ column, since glyph lines of side-by-side columns arrive interleaved. A
+ line not found after either cursor is searched from the start of the page,
+ which tolerates text extractors that order columns differently from the
+ glyph grouping. Returns one offset (or None) per line.
+ """
+ norm, pos = normalize_with_positions(page_text)
+ hits: List[Optional[int]] = []
+ ends: List[Optional[int]] = []
+ cursor = 0
+ for idx, line in enumerate(line_texts):
+ key = normalize(line)[:_KEY_CHARS]
+ if len(key) < _MIN_KEY_CHARS or (
+ _key_weight(key) < _SHORT_KEY_WEIGHT and norm.count(key) > 1
+ ):
+ hits.append(None)
+ ends.append(None)
+ continue
+ cursors = [cursor]
+ if boxes is not None:
+ column = _column_cursor(boxes, idx, ends)
+ if column is not None:
+ cursors.insert(0, column)
+ at = -1
+ for start_at in cursors:
+ at = norm.find(key, start_at, start_at + _SEARCH_WINDOW + len(key))
+ if at >= 0:
+ break
+ if at < 0:
+ at = norm.find(key)
+ if at < 0:
+ hits.append(None)
+ ends.append(None)
+ continue
+ else:
+ cursor = at + len(key)
+ ends.append(at + len(key))
+ start = pos[at]
+ # Pull in leading punctuation or markup on the same text line
+ # (brackets, "## ", list bullets) that normalization skipped.
+ while start > 0 and page_text[start - 1] != "\n" and not page_text[start - 1].isalnum():
+ start -= 1
+ hits.append(start)
+ return hits
+
+
+class PageGeometry:
+ """Converts PDF user-space points to fractions of the displayed page."""
+
+ def __init__(self, cropbox: Tuple[float, float, float, float], rotation: int = 0):
+ left, bottom, right, top = cropbox
+ self.left, self.bottom = min(left, right), min(bottom, top)
+ self.width = abs(right - left) or 1.0
+ self.height = abs(top - bottom) or 1.0
+ self.rotation = int(rotation or 0) % 360
+
+ def bbox(self, box: Tuple[float, float, float, float]) -> List[float]:
+ x0, y0, x1, y1 = box
+ # Unrotated, top-left origin.
+ ux0 = (x0 - self.left) / self.width
+ ux1 = (x1 - self.left) / self.width
+ uy0 = 1.0 - (y1 - self.bottom) / self.height
+ uy1 = 1.0 - (y0 - self.bottom) / self.height
+ corners = [self._rotate(ux0, uy0), self._rotate(ux1, uy1)]
+ xs = [c[0] for c in corners]
+ ys = [c[1] for c in corners]
+ return [
+ round(_clamp(min(xs)), 4),
+ round(_clamp(min(ys)), 4),
+ round(_clamp(max(xs)), 4),
+ round(_clamp(max(ys)), 4),
+ ]
+
+ def _rotate(self, x: float, y: float) -> Tuple[float, float]:
+ # Viewers display /Rotate clockwise.
+ if self.rotation == 90:
+ return 1.0 - y, x
+ if self.rotation == 180:
+ return 1.0 - x, 1.0 - y
+ if self.rotation == 270:
+ return y, 1.0 - x
+ return x, y
+
+
+def _clamp(v: float) -> float:
+ return 0.0 if v < 0 else 1.0 if v > 1 else v
+
+
+def _union(boxes: Iterable[Tuple[float, float, float, float]]):
+ boxes = list(boxes)
+ return (
+ min(b[0] for b in boxes),
+ min(b[1] for b in boxes),
+ max(b[2] for b in boxes),
+ max(b[3] for b in boxes),
+ )
+
+
+def _line_gap(prev: dict, cur: dict) -> Optional[float]:
+ """Vertical gap from ``prev`` down to ``cur`` when they share a column."""
+ px0, py0, px1, _ = prev["bbox"]
+ cx0, _, cx1, cy1 = cur["bbox"]
+ if cx1 < px0 or cx0 > px1 or cy1 > py0:
+ return None
+ return py0 - cy1
+
+
+def _starts_paragraph(
+ prev: dict, cur: dict, line_h: float, usual_gap: float, col_right: float
+) -> bool:
+ """Whether ``cur`` opens a new visual paragraph after ``prev``.
+
+ Boxes are PDF points with a bottom-left origin, so reading downwards means
+ decreasing y. ``usual_gap`` is the page's typical gap between consecutive
+ lines of a paragraph, which absorbs double-spaced layouts.
+ """
+ px0, py0, px1, _ = prev["bbox"]
+ cx0, _, cx1, cy1 = cur["bbox"]
+ if cx1 < px0 or cx0 > px1:
+ return True # moved to another column
+ if cy1 > py0 + line_h * 0.5:
+ return True # went back up the page: a new column or region
+ if py0 - cy1 > usual_gap * 1.5 + line_h * 0.3:
+ return True # vertical gap wider than the usual line spacing
+ if _LIST_START_RE.match(cur["text"]):
+ return True
+ ends_sentence = prev["text"].rstrip().endswith(_SENTENCE_END)
+ short = col_right - px1 > line_h * 2
+ return ends_sentence and short
+
+
+def page_blocks(
+ page_text: str,
+ offset: int,
+ page_number: int,
+ lines: Sequence[dict],
+ geometry: Optional[PageGeometry],
+) -> List[dict]:
+ """Source blocks for one text page placed at ``offset`` in the document.
+
+ ``lines`` are visual lines ``{"text", "bbox"}`` (bbox in PDF points). Lines
+ are placed in ``page_text``, grouped into paragraphs by geometry, and each
+ paragraph becomes a block spanning from its first line to the next
+ paragraph, boxed by the union of its lines. Text before the first placed
+ line, or a page with no placeable lines, gets a page-only block, so the
+ blocks always tile the page's text.
+ """
+ end = offset + len(page_text)
+ page_only = {"type": "pdf", "page": page_number}
+ if not page_text:
+ return []
+ placed = []
+ if lines and geometry is not None:
+ starts = locate_lines(
+ page_text, [ln["text"] for ln in lines], [ln["bbox"] for ln in lines]
+ )
+ placed = sorted(
+ (
+ (at, ln)
+ for at, ln in zip(starts, lines)
+ if at is not None
+ ),
+ key=lambda item: item[0],
+ )
+ if not placed:
+ return [{"start": offset, "end": end, "locator": dict(page_only)}]
+
+ heights = [ln["bbox"][3] - ln["bbox"][1] for _, ln in placed if ln["bbox"][3] > ln["bbox"][1]]
+ line_h = statistics.median(heights) if heights else 10.0
+ gaps = [
+ g
+ for g in (_line_gap(a[1], b[1]) for a, b in zip(placed, placed[1:]))
+ if g is not None and g >= 0
+ ]
+ usual_gap = statistics.median(gaps) if gaps else line_h * 0.5
+
+ paragraphs: List[Tuple[int, List[dict]]] = []
+ for at, ln in placed:
+ if paragraphs:
+ prev = paragraphs[-1][1][-1]
+ col_right = max(p["bbox"][2] for p in paragraphs[-1][1])
+ col_right = max(col_right, ln["bbox"][2])
+ if not _starts_paragraph(prev, ln, line_h, usual_gap, col_right):
+ paragraphs[-1][1].append(ln)
+ continue
+ paragraphs.append((at, [ln]))
+
+ blocks: List[dict] = []
+ first_at = paragraphs[0][0]
+ if first_at > 0 and page_text[:first_at].strip():
+ blocks.append({"start": offset, "end": offset + first_at, "locator": dict(page_only)})
+ for i, (at, members) in enumerate(paragraphs):
+ stop = paragraphs[i + 1][0] if i + 1 < len(paragraphs) else len(page_text)
+ if i == 0:
+ at = 0 if not blocks else at
+ if stop <= at:
+ continue
+ locator = dict(page_only)
+ locator["bbox"] = geometry.bbox(_union(m["bbox"] for m in members))
+ blocks.append({"start": offset + at, "end": offset + stop, "locator": locator})
+ return blocks
diff --git a/docreader/proto/docreader.pb.go b/docreader/proto/docreader.pb.go
index f071eecb1..507c1bd13 100644
--- a/docreader/proto/docreader.pb.go
+++ b/docreader/proto/docreader.pb.go
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
-// protoc-gen-go v1.36.7
-// protoc v5.29.3
+// protoc-gen-go v1.36.6
+// protoc v6.33.4
// source: docreader.proto
package proto
@@ -242,6 +242,71 @@ func (x *ImageRef) GetImageData() []byte {
return nil
}
+// SourceBlock ties a range of markdown_content to the place in the original
+// file it was parsed from, so a citation can open the file there.
+type SourceBlock struct {
+ state protoimpl.MessageState `protogen:"open.v1"`
+ // Unicode code point offsets into markdown_content, end exclusive.
+ Start uint32 `protobuf:"varint,1,opt,name=start,proto3" json:"start,omitempty"`
+ End uint32 `protobuf:"varint,2,opt,name=end,proto3" json:"end,omitempty"`
+ // JSON-encoded locator: {"type":"pdf","page":3,"bbox":[x0,y0,x1,y1]}
+ // with ordinals 1-based and bbox as fractions of the page, origin top-left.
+ LocatorJson string `protobuf:"bytes,3,opt,name=locator_json,json=locatorJson,proto3" json:"locator_json,omitempty"`
+ unknownFields protoimpl.UnknownFields
+ sizeCache protoimpl.SizeCache
+}
+
+func (x *SourceBlock) Reset() {
+ *x = SourceBlock{}
+ mi := &file_docreader_proto_msgTypes[3]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+}
+
+func (x *SourceBlock) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*SourceBlock) ProtoMessage() {}
+
+func (x *SourceBlock) ProtoReflect() protoreflect.Message {
+ mi := &file_docreader_proto_msgTypes[3]
+ if x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use SourceBlock.ProtoReflect.Descriptor instead.
+func (*SourceBlock) Descriptor() ([]byte, []int) {
+ return file_docreader_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *SourceBlock) GetStart() uint32 {
+ if x != nil {
+ return x.Start
+ }
+ return 0
+}
+
+func (x *SourceBlock) GetEnd() uint32 {
+ if x != nil {
+ return x.End
+ }
+ return 0
+}
+
+func (x *SourceBlock) GetLocatorJson() string {
+ if x != nil {
+ return x.LocatorJson
+ }
+ return ""
+}
+
type ReadResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
MarkdownContent string `protobuf:"bytes,1,opt,name=markdown_content,json=markdownContent,proto3" json:"markdown_content,omitempty"`
@@ -249,13 +314,14 @@ type ReadResponse struct {
ImageDirPath string `protobuf:"bytes,3,opt,name=image_dir_path,json=imageDirPath,proto3" json:"image_dir_path,omitempty"`
Metadata map[string]string `protobuf:"bytes,4,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
Error string `protobuf:"bytes,5,opt,name=error,proto3" json:"error,omitempty"`
+ SourceBlocks []*SourceBlock `protobuf:"bytes,6,rep,name=source_blocks,json=sourceBlocks,proto3" json:"source_blocks,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *ReadResponse) Reset() {
*x = ReadResponse{}
- mi := &file_docreader_proto_msgTypes[3]
+ mi := &file_docreader_proto_msgTypes[4]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -267,7 +333,7 @@ func (x *ReadResponse) String() string {
func (*ReadResponse) ProtoMessage() {}
func (x *ReadResponse) ProtoReflect() protoreflect.Message {
- mi := &file_docreader_proto_msgTypes[3]
+ mi := &file_docreader_proto_msgTypes[4]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -280,7 +346,7 @@ func (x *ReadResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use ReadResponse.ProtoReflect.Descriptor instead.
func (*ReadResponse) Descriptor() ([]byte, []int) {
- return file_docreader_proto_rawDescGZIP(), []int{3}
+ return file_docreader_proto_rawDescGZIP(), []int{4}
}
func (x *ReadResponse) GetMarkdownContent() string {
@@ -318,6 +384,13 @@ func (x *ReadResponse) GetError() string {
return ""
}
+func (x *ReadResponse) GetSourceBlocks() []*SourceBlock {
+ if x != nil {
+ return x.SourceBlocks
+ }
+ return nil
+}
+
// Metadata header for a streamed read. Sent exactly once, before any image.
type ReadStreamMeta struct {
state protoimpl.MessageState `protogen:"open.v1"`
@@ -326,13 +399,14 @@ type ReadStreamMeta struct {
Metadata map[string]string `protobuf:"bytes,3,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
Error string `protobuf:"bytes,4,opt,name=error,proto3" json:"error,omitempty"`
ImageCount uint32 `protobuf:"varint,5,opt,name=image_count,json=imageCount,proto3" json:"image_count,omitempty"` // best-effort total image count (0 if unknown)
+ SourceBlocks []*SourceBlock `protobuf:"bytes,6,rep,name=source_blocks,json=sourceBlocks,proto3" json:"source_blocks,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *ReadStreamMeta) Reset() {
*x = ReadStreamMeta{}
- mi := &file_docreader_proto_msgTypes[4]
+ mi := &file_docreader_proto_msgTypes[5]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -344,7 +418,7 @@ func (x *ReadStreamMeta) String() string {
func (*ReadStreamMeta) ProtoMessage() {}
func (x *ReadStreamMeta) ProtoReflect() protoreflect.Message {
- mi := &file_docreader_proto_msgTypes[4]
+ mi := &file_docreader_proto_msgTypes[5]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -357,7 +431,7 @@ func (x *ReadStreamMeta) ProtoReflect() protoreflect.Message {
// Deprecated: Use ReadStreamMeta.ProtoReflect.Descriptor instead.
func (*ReadStreamMeta) Descriptor() ([]byte, []int) {
- return file_docreader_proto_rawDescGZIP(), []int{4}
+ return file_docreader_proto_rawDescGZIP(), []int{5}
}
func (x *ReadStreamMeta) GetMarkdownContent() string {
@@ -395,6 +469,13 @@ func (x *ReadStreamMeta) GetImageCount() uint32 {
return 0
}
+func (x *ReadStreamMeta) GetSourceBlocks() []*SourceBlock {
+ if x != nil {
+ return x.SourceBlocks
+ }
+ return nil
+}
+
// One frame of a ReadStream. The first frame carries `meta`; every subsequent
// frame carries a single `image`.
type ReadStreamResponse struct {
@@ -410,7 +491,7 @@ type ReadStreamResponse struct {
func (x *ReadStreamResponse) Reset() {
*x = ReadStreamResponse{}
- mi := &file_docreader_proto_msgTypes[5]
+ mi := &file_docreader_proto_msgTypes[6]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -422,7 +503,7 @@ func (x *ReadStreamResponse) String() string {
func (*ReadStreamResponse) ProtoMessage() {}
func (x *ReadStreamResponse) ProtoReflect() protoreflect.Message {
- mi := &file_docreader_proto_msgTypes[5]
+ mi := &file_docreader_proto_msgTypes[6]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -435,7 +516,7 @@ func (x *ReadStreamResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use ReadStreamResponse.ProtoReflect.Descriptor instead.
func (*ReadStreamResponse) Descriptor() ([]byte, []int) {
- return file_docreader_proto_rawDescGZIP(), []int{5}
+ return file_docreader_proto_rawDescGZIP(), []int{6}
}
func (x *ReadStreamResponse) GetPayload() isReadStreamResponse_Payload {
@@ -488,7 +569,7 @@ type ListEnginesRequest struct {
func (x *ListEnginesRequest) Reset() {
*x = ListEnginesRequest{}
- mi := &file_docreader_proto_msgTypes[6]
+ mi := &file_docreader_proto_msgTypes[7]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -500,7 +581,7 @@ func (x *ListEnginesRequest) String() string {
func (*ListEnginesRequest) ProtoMessage() {}
func (x *ListEnginesRequest) ProtoReflect() protoreflect.Message {
- mi := &file_docreader_proto_msgTypes[6]
+ mi := &file_docreader_proto_msgTypes[7]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -513,7 +594,7 @@ func (x *ListEnginesRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use ListEnginesRequest.ProtoReflect.Descriptor instead.
func (*ListEnginesRequest) Descriptor() ([]byte, []int) {
- return file_docreader_proto_rawDescGZIP(), []int{6}
+ return file_docreader_proto_rawDescGZIP(), []int{7}
}
func (x *ListEnginesRequest) GetConfigOverrides() map[string]string {
@@ -536,7 +617,7 @@ type ParserEngineInfo struct {
func (x *ParserEngineInfo) Reset() {
*x = ParserEngineInfo{}
- mi := &file_docreader_proto_msgTypes[7]
+ mi := &file_docreader_proto_msgTypes[8]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -548,7 +629,7 @@ func (x *ParserEngineInfo) String() string {
func (*ParserEngineInfo) ProtoMessage() {}
func (x *ParserEngineInfo) ProtoReflect() protoreflect.Message {
- mi := &file_docreader_proto_msgTypes[7]
+ mi := &file_docreader_proto_msgTypes[8]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -561,7 +642,7 @@ func (x *ParserEngineInfo) ProtoReflect() protoreflect.Message {
// Deprecated: Use ParserEngineInfo.ProtoReflect.Descriptor instead.
func (*ParserEngineInfo) Descriptor() ([]byte, []int) {
- return file_docreader_proto_rawDescGZIP(), []int{7}
+ return file_docreader_proto_rawDescGZIP(), []int{8}
}
func (x *ParserEngineInfo) GetName() string {
@@ -608,7 +689,7 @@ type ListEnginesResponse struct {
func (x *ListEnginesResponse) Reset() {
*x = ListEnginesResponse{}
- mi := &file_docreader_proto_msgTypes[8]
+ mi := &file_docreader_proto_msgTypes[9]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -620,7 +701,7 @@ func (x *ListEnginesResponse) String() string {
func (*ListEnginesResponse) ProtoMessage() {}
func (x *ListEnginesResponse) ProtoReflect() protoreflect.Message {
- mi := &file_docreader_proto_msgTypes[8]
+ mi := &file_docreader_proto_msgTypes[9]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -633,7 +714,7 @@ func (x *ListEnginesResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use ListEnginesResponse.ProtoReflect.Descriptor instead.
func (*ListEnginesResponse) Descriptor() ([]byte, []int) {
- return file_docreader_proto_rawDescGZIP(), []int{8}
+ return file_docreader_proto_rawDescGZIP(), []int{9}
}
func (x *ListEnginesResponse) GetEngines() []*ParserEngineInfo {
@@ -671,24 +752,30 @@ const file_docreader_proto_rawDesc = "" +
"\vstorage_key\x18\x04 \x01(\tR\n" +
"storageKey\x12\x1d\n" +
"\n" +
- "image_data\x18\x05 \x01(\fR\timageData\"\xa9\x02\n" +
+ "image_data\x18\x05 \x01(\fR\timageData\"X\n" +
+ "\vSourceBlock\x12\x14\n" +
+ "\x05start\x18\x01 \x01(\rR\x05start\x12\x10\n" +
+ "\x03end\x18\x02 \x01(\rR\x03end\x12!\n" +
+ "\flocator_json\x18\x03 \x01(\tR\vlocatorJson\"\xe6\x02\n" +
"\fReadResponse\x12)\n" +
"\x10markdown_content\x18\x01 \x01(\tR\x0fmarkdownContent\x122\n" +
"\n" +
"image_refs\x18\x02 \x03(\v2\x13.docreader.ImageRefR\timageRefs\x12$\n" +
"\x0eimage_dir_path\x18\x03 \x01(\tR\fimageDirPath\x12A\n" +
"\bmetadata\x18\x04 \x03(\v2%.docreader.ReadResponse.MetadataEntryR\bmetadata\x12\x14\n" +
- "\x05error\x18\x05 \x01(\tR\x05error\x1a;\n" +
+ "\x05error\x18\x05 \x01(\tR\x05error\x12;\n" +
+ "\rsource_blocks\x18\x06 \x03(\v2\x16.docreader.SourceBlockR\fsourceBlocks\x1a;\n" +
"\rMetadataEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
- "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x9a\x02\n" +
+ "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xd7\x02\n" +
"\x0eReadStreamMeta\x12)\n" +
"\x10markdown_content\x18\x01 \x01(\tR\x0fmarkdownContent\x12$\n" +
"\x0eimage_dir_path\x18\x02 \x01(\tR\fimageDirPath\x12C\n" +
"\bmetadata\x18\x03 \x03(\v2'.docreader.ReadStreamMeta.MetadataEntryR\bmetadata\x12\x14\n" +
"\x05error\x18\x04 \x01(\tR\x05error\x12\x1f\n" +
"\vimage_count\x18\x05 \x01(\rR\n" +
- "imageCount\x1a;\n" +
+ "imageCount\x12;\n" +
+ "\rsource_blocks\x18\x06 \x03(\v2\x16.docreader.SourceBlockR\fsourceBlocks\x1a;\n" +
"\rMetadataEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"}\n" +
@@ -728,43 +815,46 @@ func file_docreader_proto_rawDescGZIP() []byte {
return file_docreader_proto_rawDescData
}
-var file_docreader_proto_msgTypes = make([]protoimpl.MessageInfo, 13)
+var file_docreader_proto_msgTypes = make([]protoimpl.MessageInfo, 14)
var file_docreader_proto_goTypes = []any{
(*ReadConfig)(nil), // 0: docreader.ReadConfig
(*ReadRequest)(nil), // 1: docreader.ReadRequest
(*ImageRef)(nil), // 2: docreader.ImageRef
- (*ReadResponse)(nil), // 3: docreader.ReadResponse
- (*ReadStreamMeta)(nil), // 4: docreader.ReadStreamMeta
- (*ReadStreamResponse)(nil), // 5: docreader.ReadStreamResponse
- (*ListEnginesRequest)(nil), // 6: docreader.ListEnginesRequest
- (*ParserEngineInfo)(nil), // 7: docreader.ParserEngineInfo
- (*ListEnginesResponse)(nil), // 8: docreader.ListEnginesResponse
- nil, // 9: docreader.ReadConfig.ParserEngineOverridesEntry
- nil, // 10: docreader.ReadResponse.MetadataEntry
- nil, // 11: docreader.ReadStreamMeta.MetadataEntry
- nil, // 12: docreader.ListEnginesRequest.ConfigOverridesEntry
+ (*SourceBlock)(nil), // 3: docreader.SourceBlock
+ (*ReadResponse)(nil), // 4: docreader.ReadResponse
+ (*ReadStreamMeta)(nil), // 5: docreader.ReadStreamMeta
+ (*ReadStreamResponse)(nil), // 6: docreader.ReadStreamResponse
+ (*ListEnginesRequest)(nil), // 7: docreader.ListEnginesRequest
+ (*ParserEngineInfo)(nil), // 8: docreader.ParserEngineInfo
+ (*ListEnginesResponse)(nil), // 9: docreader.ListEnginesResponse
+ nil, // 10: docreader.ReadConfig.ParserEngineOverridesEntry
+ nil, // 11: docreader.ReadResponse.MetadataEntry
+ nil, // 12: docreader.ReadStreamMeta.MetadataEntry
+ nil, // 13: docreader.ListEnginesRequest.ConfigOverridesEntry
}
var file_docreader_proto_depIdxs = []int32{
- 9, // 0: docreader.ReadConfig.parser_engine_overrides:type_name -> docreader.ReadConfig.ParserEngineOverridesEntry
+ 10, // 0: docreader.ReadConfig.parser_engine_overrides:type_name -> docreader.ReadConfig.ParserEngineOverridesEntry
0, // 1: docreader.ReadRequest.config:type_name -> docreader.ReadConfig
2, // 2: docreader.ReadResponse.image_refs:type_name -> docreader.ImageRef
- 10, // 3: docreader.ReadResponse.metadata:type_name -> docreader.ReadResponse.MetadataEntry
- 11, // 4: docreader.ReadStreamMeta.metadata:type_name -> docreader.ReadStreamMeta.MetadataEntry
- 4, // 5: docreader.ReadStreamResponse.meta:type_name -> docreader.ReadStreamMeta
- 2, // 6: docreader.ReadStreamResponse.image:type_name -> docreader.ImageRef
- 12, // 7: docreader.ListEnginesRequest.config_overrides:type_name -> docreader.ListEnginesRequest.ConfigOverridesEntry
- 7, // 8: docreader.ListEnginesResponse.engines:type_name -> docreader.ParserEngineInfo
- 1, // 9: docreader.DocReader.Read:input_type -> docreader.ReadRequest
- 1, // 10: docreader.DocReader.ReadStream:input_type -> docreader.ReadRequest
- 6, // 11: docreader.DocReader.ListEngines:input_type -> docreader.ListEnginesRequest
- 3, // 12: docreader.DocReader.Read:output_type -> docreader.ReadResponse
- 5, // 13: docreader.DocReader.ReadStream:output_type -> docreader.ReadStreamResponse
- 8, // 14: docreader.DocReader.ListEngines:output_type -> docreader.ListEnginesResponse
- 12, // [12:15] is the sub-list for method output_type
- 9, // [9:12] is the sub-list for method input_type
- 9, // [9:9] is the sub-list for extension type_name
- 9, // [9:9] is the sub-list for extension extendee
- 0, // [0:9] is the sub-list for field type_name
+ 11, // 3: docreader.ReadResponse.metadata:type_name -> docreader.ReadResponse.MetadataEntry
+ 3, // 4: docreader.ReadResponse.source_blocks:type_name -> docreader.SourceBlock
+ 12, // 5: docreader.ReadStreamMeta.metadata:type_name -> docreader.ReadStreamMeta.MetadataEntry
+ 3, // 6: docreader.ReadStreamMeta.source_blocks:type_name -> docreader.SourceBlock
+ 5, // 7: docreader.ReadStreamResponse.meta:type_name -> docreader.ReadStreamMeta
+ 2, // 8: docreader.ReadStreamResponse.image:type_name -> docreader.ImageRef
+ 13, // 9: docreader.ListEnginesRequest.config_overrides:type_name -> docreader.ListEnginesRequest.ConfigOverridesEntry
+ 8, // 10: docreader.ListEnginesResponse.engines:type_name -> docreader.ParserEngineInfo
+ 1, // 11: docreader.DocReader.Read:input_type -> docreader.ReadRequest
+ 1, // 12: docreader.DocReader.ReadStream:input_type -> docreader.ReadRequest
+ 7, // 13: docreader.DocReader.ListEngines:input_type -> docreader.ListEnginesRequest
+ 4, // 14: docreader.DocReader.Read:output_type -> docreader.ReadResponse
+ 6, // 15: docreader.DocReader.ReadStream:output_type -> docreader.ReadStreamResponse
+ 9, // 16: docreader.DocReader.ListEngines:output_type -> docreader.ListEnginesResponse
+ 14, // [14:17] is the sub-list for method output_type
+ 11, // [11:14] is the sub-list for method input_type
+ 11, // [11:11] is the sub-list for extension type_name
+ 11, // [11:11] is the sub-list for extension extendee
+ 0, // [0:11] is the sub-list for field type_name
}
func init() { file_docreader_proto_init() }
@@ -772,7 +862,7 @@ func file_docreader_proto_init() {
if File_docreader_proto != nil {
return
}
- file_docreader_proto_msgTypes[5].OneofWrappers = []any{
+ file_docreader_proto_msgTypes[6].OneofWrappers = []any{
(*ReadStreamResponse_Meta)(nil),
(*ReadStreamResponse_Image)(nil),
}
@@ -782,7 +872,7 @@ func file_docreader_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_docreader_proto_rawDesc), len(file_docreader_proto_rawDesc)),
NumEnums: 0,
- NumMessages: 13,
+ NumMessages: 14,
NumExtensions: 0,
NumServices: 1,
},
diff --git a/docreader/proto/docreader.proto b/docreader/proto/docreader.proto
index f1f8a4414..be6ab5446 100644
--- a/docreader/proto/docreader.proto
+++ b/docreader/proto/docreader.proto
@@ -43,12 +43,24 @@ message ImageRef {
bytes image_data = 5; // inline bytes fallback
}
+// SourceBlock ties a range of markdown_content to the place in the original
+// file it was parsed from, so a citation can open the file there.
+message SourceBlock {
+ // Unicode code point offsets into markdown_content, end exclusive.
+ uint32 start = 1;
+ uint32 end = 2;
+ // JSON-encoded locator: {"type":"pdf","page":3,"bbox":[x0,y0,x1,y1]}
+ // with ordinals 1-based and bbox as fractions of the page, origin top-left.
+ string locator_json = 3;
+}
+
message ReadResponse {
string markdown_content = 1;
repeated ImageRef image_refs = 2;
string image_dir_path = 3;
map<string, string> metadata = 4;
string error = 5;
+ repeated SourceBlock source_blocks = 6;
}
// Metadata header for a streamed read. Sent exactly once, before any image.
@@ -58,6 +70,7 @@ message ReadStreamMeta {
map<string, string> metadata = 3;
string error = 4;
uint32 image_count = 5; // best-effort total image count (0 if unknown)
+ repeated SourceBlock source_blocks = 6;
}
// One frame of a ReadStream. The first frame carries `meta`; every subsequent
diff --git a/docreader/proto/docreader_grpc.pb.go b/docreader/proto/docreader_grpc.pb.go
index abb5bbc81..6e20ba9d2 100644
--- a/docreader/proto/docreader_grpc.pb.go
+++ b/docreader/proto/docreader_grpc.pb.go
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.5.1
-// - protoc v5.29.3
+// - protoc v6.33.4
// source: docreader.proto
package proto
diff --git a/docreader/proto/docreader_pb2.py b/docreader/proto/docreader_pb2.py
index 8e3d16e00..7a92fad0a 100644
--- a/docreader/proto/docreader_pb2.py
+++ b/docreader/proto/docreader_pb2.py
@@ -24,7 +24,7 @@ _sym_db = _symbol_database.Default()
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x64ocreader.proto\x12\tdocreader\"\xba\x01\n\nReadConfig\x12\x15\n\rparser_engine\x18\x01 \x01(\t\x12Q\n\x17parser_engine_overrides\x18\x02 \x03(\x0b\x32\x30.docreader.ReadConfig.ParserEngineOverridesEntry\x1a<\n\x1aParserEngineOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01J\x04\x08\x03\x10\x04\"\xa0\x01\n\x0bReadRequest\x12\x14\n\x0c\x66ile_content\x18\x01 \x01(\x0c\x12\x11\n\tfile_name\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\x12\x0b\n\x03url\x18\x04 \x01(\t\x12\r\n\x05title\x18\x05 \x01(\t\x12%\n\x06\x63onfig\x18\x06 \x01(\x0b\x32\x15.docreader.ReadConfig\x12\x12\n\nrequest_id\x18\x07 \x01(\t\"n\n\x08ImageRef\x12\x10\n\x08\x66ilename\x18\x01 \x01(\t\x12\x14\n\x0coriginal_ref\x18\x02 \x01(\t\x12\x11\n\tmime_type\x18\x03 \x01(\t\x12\x13\n\x0bstorage_key\x18\x04 \x01(\t\x12\x12\n\nimage_data\x18\x05 \x01(\x0c\"\xe2\x01\n\x0cReadResponse\x12\x18\n\x10markdown_content\x18\x01 \x01(\t\x12\'\n\nimage_refs\x18\x02 \x03(\x0b\x32\x13.docreader.ImageRef\x12\x16\n\x0eimage_dir_path\x18\x03 \x01(\t\x12\x37\n\x08metadata\x18\x04 \x03(\x0b\x32%.docreader.ReadResponse.MetadataEntry\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xd2\x01\n\x0eReadStreamMeta\x12\x18\n\x10markdown_content\x18\x01 \x01(\t\x12\x16\n\x0eimage_dir_path\x18\x02 \x01(\t\x12\x39\n\x08metadata\x18\x03 \x03(\x0b\x32\'.docreader.ReadStreamMeta.MetadataEntry\x12\r\n\x05\x65rror\x18\x04 \x01(\t\x12\x13\n\x0bimage_count\x18\x05 \x01(\r\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"p\n\x12ReadStreamResponse\x12)\n\x04meta\x18\x01 \x01(\x0b\x32\x19.docreader.ReadStreamMetaH\x00\x12$\n\x05image\x18\x02 \x01(\x0b\x32\x13.docreader.ImageRefH\x00\x42\t\n\x07payload\"\x9a\x01\n\x12ListEnginesRequest\x12L\n\x10\x63onfig_overrides\x18\x01 \x03(\x0b\x32\x32.docreader.ListEnginesRequest.ConfigOverridesEntry\x1a\x36\n\x14\x43onfigOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"x\n\x10ParserEngineInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x12\n\nfile_types\x18\x03 \x03(\t\x12\x11\n\tavailable\x18\x04 \x01(\x08\x12\x1a\n\x12unavailable_reason\x18\x05 \x01(\t\"C\n\x13ListEnginesResponse\x12,\n\x07\x65ngines\x18\x01 \x03(\x0b\x32\x1b.docreader.ParserEngineInfo2\xdf\x01\n\tDocReader\x12\x39\n\x04Read\x12\x16.docreader.ReadRequest\x1a\x17.docreader.ReadResponse\"\x00\x12G\n\nReadStream\x12\x16.docreader.ReadRequest\x1a\x1d.docreader.ReadStreamResponse\"\x00\x30\x01\x12N\n\x0bListEngines\x12\x1d.docreader.ListEnginesRequest\x1a\x1e.docreader.ListEnginesResponse\"\x00\x42\x35Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x64ocreader.proto\x12\tdocreader\"\xba\x01\n\nReadConfig\x12\x15\n\rparser_engine\x18\x01 \x01(\t\x12Q\n\x17parser_engine_overrides\x18\x02 \x03(\x0b\x32\x30.docreader.ReadConfig.ParserEngineOverridesEntry\x1a<\n\x1aParserEngineOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01J\x04\x08\x03\x10\x04\"\xa0\x01\n\x0bReadRequest\x12\x14\n\x0c\x66ile_content\x18\x01 \x01(\x0c\x12\x11\n\tfile_name\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\x12\x0b\n\x03url\x18\x04 \x01(\t\x12\r\n\x05title\x18\x05 \x01(\t\x12%\n\x06\x63onfig\x18\x06 \x01(\x0b\x32\x15.docreader.ReadConfig\x12\x12\n\nrequest_id\x18\x07 \x01(\t\"n\n\x08ImageRef\x12\x10\n\x08\x66ilename\x18\x01 \x01(\t\x12\x14\n\x0coriginal_ref\x18\x02 \x01(\t\x12\x11\n\tmime_type\x18\x03 \x01(\t\x12\x13\n\x0bstorage_key\x18\x04 \x01(\t\x12\x12\n\nimage_data\x18\x05 \x01(\x0c\"?\n\x0bSourceBlock\x12\r\n\x05start\x18\x01 \x01(\r\x12\x0b\n\x03\x65nd\x18\x02 \x01(\r\x12\x14\n\x0clocator_json\x18\x03 \x01(\t\"\x91\x02\n\x0cReadResponse\x12\x18\n\x10markdown_content\x18\x01 \x01(\t\x12\'\n\nimage_refs\x18\x02 \x03(\x0b\x32\x13.docreader.ImageRef\x12\x16\n\x0eimage_dir_path\x18\x03 \x01(\t\x12\x37\n\x08metadata\x18\x04 \x03(\x0b\x32%.docreader.ReadResponse.MetadataEntry\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x12-\n\rsource_blocks\x18\x06 \x03(\x0b\x32\x16.docreader.SourceBlock\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x81\x02\n\x0eReadStreamMeta\x12\x18\n\x10markdown_content\x18\x01 \x01(\t\x12\x16\n\x0eimage_dir_path\x18\x02 \x01(\t\x12\x39\n\x08metadata\x18\x03 \x03(\x0b\x32\'.docreader.ReadStreamMeta.MetadataEntry\x12\r\n\x05\x65rror\x18\x04 \x01(\t\x12\x13\n\x0bimage_count\x18\x05 \x01(\r\x12-\n\rsource_blocks\x18\x06 \x03(\x0b\x32\x16.docreader.SourceBlock\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"p\n\x12ReadStreamResponse\x12)\n\x04meta\x18\x01 \x01(\x0b\x32\x19.docreader.ReadStreamMetaH\x00\x12$\n\x05image\x18\x02 \x01(\x0b\x32\x13.docreader.ImageRefH\x00\x42\t\n\x07payload\"\x9a\x01\n\x12ListEnginesRequest\x12L\n\x10\x63onfig_overrides\x18\x01 \x03(\x0b\x32\x32.docreader.ListEnginesRequest.ConfigOverridesEntry\x1a\x36\n\x14\x43onfigOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"x\n\x10ParserEngineInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x12\n\nfile_types\x18\x03 \x03(\t\x12\x11\n\tavailable\x18\x04 \x01(\x08\x12\x1a\n\x12unavailable_reason\x18\x05 \x01(\t\"C\n\x13ListEnginesResponse\x12,\n\x07\x65ngines\x18\x01 \x03(\x0b\x32\x1b.docreader.ParserEngineInfo2\xdf\x01\n\tDocReader\x12\x39\n\x04Read\x12\x16.docreader.ReadRequest\x1a\x17.docreader.ReadResponse\"\x00\x12G\n\nReadStream\x12\x16.docreader.ReadRequest\x1a\x1d.docreader.ReadStreamResponse\"\x00\x30\x01\x12N\n\x0bListEngines\x12\x1d.docreader.ListEnginesRequest\x1a\x1e.docreader.ListEnginesResponse\"\x00\x42\x35Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3')
_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -48,24 +48,26 @@ if not _descriptor._USE_C_DESCRIPTORS:
_globals['_READREQUEST']._serialized_end=380
_globals['_IMAGEREF']._serialized_start=382
_globals['_IMAGEREF']._serialized_end=492
- _globals['_READRESPONSE']._serialized_start=495
- _globals['_READRESPONSE']._serialized_end=721
- _globals['_READRESPONSE_METADATAENTRY']._serialized_start=674
- _globals['_READRESPONSE_METADATAENTRY']._serialized_end=721
- _globals['_READSTREAMMETA']._serialized_start=724
- _globals['_READSTREAMMETA']._serialized_end=934
- _globals['_READSTREAMMETA_METADATAENTRY']._serialized_start=674
- _globals['_READSTREAMMETA_METADATAENTRY']._serialized_end=721
- _globals['_READSTREAMRESPONSE']._serialized_start=936
- _globals['_READSTREAMRESPONSE']._serialized_end=1048
- _globals['_LISTENGINESREQUEST']._serialized_start=1051
- _globals['_LISTENGINESREQUEST']._serialized_end=1205
- _globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_start=1151
- _globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_end=1205
- _globals['_PARSERENGINEINFO']._serialized_start=1207
- _globals['_PARSERENGINEINFO']._serialized_end=1327
- _globals['_LISTENGINESRESPONSE']._serialized_start=1329
- _globals['_LISTENGINESRESPONSE']._serialized_end=1396
- _globals['_DOCREADER']._serialized_start=1399
- _globals['_DOCREADER']._serialized_end=1622
+ _globals['_SOURCEBLOCK']._serialized_start=494
+ _globals['_SOURCEBLOCK']._serialized_end=557
+ _globals['_READRESPONSE']._serialized_start=560
+ _globals['_READRESPONSE']._serialized_end=833
+ _globals['_READRESPONSE_METADATAENTRY']._serialized_start=786
+ _globals['_READRESPONSE_METADATAENTRY']._serialized_end=833
+ _globals['_READSTREAMMETA']._serialized_start=836
+ _globals['_READSTREAMMETA']._serialized_end=1093
+ _globals['_READSTREAMMETA_METADATAENTRY']._serialized_start=786
+ _globals['_READSTREAMMETA_METADATAENTRY']._serialized_end=833
+ _globals['_READSTREAMRESPONSE']._serialized_start=1095
+ _globals['_READSTREAMRESPONSE']._serialized_end=1207
+ _globals['_LISTENGINESREQUEST']._serialized_start=1210
+ _globals['_LISTENGINESREQUEST']._serialized_end=1364
+ _globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_start=1310
+ _globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_end=1364
+ _globals['_PARSERENGINEINFO']._serialized_start=1366
+ _globals['_PARSERENGINEINFO']._serialized_end=1486
+ _globals['_LISTENGINESRESPONSE']._serialized_start=1488
+ _globals['_LISTENGINESRESPONSE']._serialized_end=1555
+ _globals['_DOCREADER']._serialized_start=1558
+ _globals['_DOCREADER']._serialized_end=1781
# @@protoc_insertion_point(module_scope)
diff --git a/docreader/proto/docreader_pb2.pyi b/docreader/proto/docreader_pb2.pyi
index e10b4e53f..0f16dc0d6 100644
--- a/docreader/proto/docreader_pb2.pyi
+++ b/docreader/proto/docreader_pb2.pyi
@@ -53,8 +53,18 @@ class ImageRef(_message.Message):
image_data: bytes
def __init__(self, filename: _Optional[str] = ..., original_ref: _Optional[str] = ..., mime_type: _Optional[str] = ..., storage_key: _Optional[str] = ..., image_data: _Optional[bytes] = ...) -> None: ...
+class SourceBlock(_message.Message):
+ __slots__ = ("start", "end", "locator_json")
+ START_FIELD_NUMBER: _ClassVar[int]
+ END_FIELD_NUMBER: _ClassVar[int]
+ LOCATOR_JSON_FIELD_NUMBER: _ClassVar[int]
+ start: int
+ end: int
+ locator_json: str
+ def __init__(self, start: _Optional[int] = ..., end: _Optional[int] = ..., locator_json: _Optional[str] = ...) -> None: ...
+
class ReadResponse(_message.Message):
- __slots__ = ("markdown_content", "image_refs", "image_dir_path", "metadata", "error")
+ __slots__ = ("markdown_content", "image_refs", "image_dir_path", "metadata", "error", "source_blocks")
class MetadataEntry(_message.Message):
__slots__ = ("key", "value")
KEY_FIELD_NUMBER: _ClassVar[int]
@@ -67,15 +77,17 @@ class ReadResponse(_message.Message):
IMAGE_DIR_PATH_FIELD_NUMBER: _ClassVar[int]
METADATA_FIELD_NUMBER: _ClassVar[int]
ERROR_FIELD_NUMBER: _ClassVar[int]
+ SOURCE_BLOCKS_FIELD_NUMBER: _ClassVar[int]
markdown_content: str
image_refs: _containers.RepeatedCompositeFieldContainer[ImageRef]
image_dir_path: str
metadata: _containers.ScalarMap[str, str]
error: str
- def __init__(self, markdown_content: _Optional[str] = ..., image_refs: _Optional[_Iterable[_Union[ImageRef, _Mapping]]] = ..., image_dir_path: _Optional[str] = ..., metadata: _Optional[_Mapping[str, str]] = ..., error: _Optional[str] = ...) -> None: ...
+ source_blocks: _containers.RepeatedCompositeFieldContainer[SourceBlock]
+ def __init__(self, markdown_content: _Optional[str] = ..., image_refs: _Optional[_Iterable[_Union[ImageRef, _Mapping]]] = ..., image_dir_path: _Optional[str] = ..., metadata: _Optional[_Mapping[str, str]] = ..., error: _Optional[str] = ..., source_blocks: _Optional[_Iterable[_Union[SourceBlock, _Mapping]]] = ...) -> None: ...
class ReadStreamMeta(_message.Message):
- __slots__ = ("markdown_content", "image_dir_path", "metadata", "error", "image_count")
+ __slots__ = ("markdown_content", "image_dir_path", "metadata", "error", "image_count", "source_blocks")
class MetadataEntry(_message.Message):
__slots__ = ("key", "value")
KEY_FIELD_NUMBER: _ClassVar[int]
@@ -88,12 +100,14 @@ class ReadStreamMeta(_message.Message):
METADATA_FIELD_NUMBER: _ClassVar[int]
ERROR_FIELD_NUMBER: _ClassVar[int]
IMAGE_COUNT_FIELD_NUMBER: _ClassVar[int]
+ SOURCE_BLOCKS_FIELD_NUMBER: _ClassVar[int]
markdown_content: str
image_dir_path: str
metadata: _containers.ScalarMap[str, str]
error: str
image_count: int
- def __init__(self, markdown_content: _Optional[str] = ..., image_dir_path: _Optional[str] = ..., metadata: _Optional[_Mapping[str, str]] = ..., error: _Optional[str] = ..., image_count: _Optional[int] = ...) -> None: ...
+ source_blocks: _containers.RepeatedCompositeFieldContainer[SourceBlock]
+ def __init__(self, markdown_content: _Optional[str] = ..., image_dir_path: _Optional[str] = ..., metadata: _Optional[_Mapping[str, str]] = ..., error: _Optional[str] = ..., image_count: _Optional[int] = ..., source_blocks: _Optional[_Iterable[_Union[SourceBlock, _Mapping]]] = ...) -> None: ...
class ReadStreamResponse(_message.Message):
__slots__ = ("meta", "image")
diff --git a/docreader/source_wire.py b/docreader/source_wire.py
new file mode 100644
index 000000000..8166b1b29
--- /dev/null
+++ b/docreader/source_wire.py
@@ -0,0 +1,34 @@
+"""Wire form of source blocks (see docreader.parser.source_locator).
+
+Kept apart from docreader.main so it can be imported without the parser
+registry.
+"""
+
+import json
+
+from docreader.proto.docreader_pb2 import SourceBlock
+
+
+def source_blocks_to_proto(document) -> list:
+ """Convert a document's source blocks to their wire form.
+
+ Malformed entries are skipped: positions are an optional extra and must
+ never fail a parse.
+ """
+ out = []
+ for block in getattr(document, "source_blocks", None) or []:
+ try:
+ start, end = int(block["start"]), int(block["end"])
+ locator = block["locator"]
+ if end <= start or start < 0 or not locator.get("type"):
+ continue
+ out.append(
+ SourceBlock(
+ start=start,
+ end=end,
+ locator_json=json.dumps(locator, ensure_ascii=False),
+ )
+ )
+ except (KeyError, TypeError, ValueError, AttributeError):
+ continue
+ return out
diff --git a/docreader/tests/test_excel_parser.py b/docreader/tests/test_excel_parser.py
index 682613ff2..abc0fcd72 100644
--- a/docreader/tests/test_excel_parser.py
+++ b/docreader/tests/test_excel_parser.py
@@ -511,3 +511,26 @@ class ExcelParserTest(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
+
+
+class ExcelSourceBlocksTest(unittest.TestCase):
+ def test_rows_carry_sheet_and_row_number(self):
+ import io
+
+ import openpyxl
+
+ from docreader.parser.excel_parser import ExcelParser
+
+ wb = openpyxl.Workbook()
+ ws = wb.active
+ ws.title = "明细"
+ ws["B3"] = "螺栓"
+ ws["A5"] = "合计"
+ buf = io.BytesIO()
+ wb.save(buf)
+
+ doc = ExcelParser(file_name="a.xlsx", file_type="xlsx").parse_into_text(buf.getvalue())
+ rows = [(b["locator"]["sheet"], b["locator"]["row_start"]) for b in doc.source_blocks]
+ self.assertEqual(rows, [("明细", 3), ("明细", 5)])
+ first = doc.source_blocks[0]
+ self.assertEqual(doc.content[first["start"]:first["end"]], "B: 螺栓\n")
diff --git a/docreader/tests/test_source_locator.py b/docreader/tests/test_source_locator.py
new file mode 100644
index 000000000..f84cf2f42
--- /dev/null
+++ b/docreader/tests/test_source_locator.py
@@ -0,0 +1,143 @@
+import json
+import unittest
+
+from docreader.models.document import Document
+from docreader.parser.source_locator import (
+ PageGeometry,
+ locate_lines,
+ page_blocks,
+)
+
+
+def _ln(text, x0, y0, x1, y1):
+ return {"text": text, "bbox": (x0, y0, x1, y1)}
+
+
+class LocateLinesTest(unittest.TestCase):
+ def test_finds_lines_in_order_and_pulls_in_leading_markup(self):
+ text = "## 标题\n【版权声明】\n正文 第一行\n正文 第一行"
+ hits = locate_lines(text, ["标题", "版权声明】", "正文第一行", "正文 第一行", "无此行"])
+ self.assertEqual(hits[0], 0)
+ self.assertEqual(text[hits[1]:hits[1] + 2], "【版")
+ self.assertEqual(hits[2], text.index("正文"))
+ self.assertEqual(hits[3], text.rindex("正文"))
+ self.assertIsNone(hits[4])
+
+
+class PageGeometryTest(unittest.TestCase):
+ def test_flips_to_top_left_origin(self):
+ g = PageGeometry((0, 0, 200, 100))
+ self.assertEqual(g.bbox((20, 80, 100, 90)), [0.1, 0.1, 0.5, 0.2])
+
+ def test_applies_cropbox_offset_and_rotation(self):
+ g = PageGeometry((100, 100, 300, 200), rotation=90)
+ # Unrotated [0.1, 0.1, 0.5, 0.2] turns clockwise.
+ self.assertEqual(g.bbox((120, 180, 200, 190)), [0.8, 0.1, 0.9, 0.5])
+
+
+class PageBlocksTest(unittest.TestCase):
+ def test_groups_lines_into_boxed_paragraphs(self):
+ page = (
+ "8.14.11 索夹与吊索施工应符合下列规定:\n"
+ "1 在满足施工需要的前提下,应减小猫道面层开孔面积,并应在开孔位置四周绑\n"
+ "扎防滑木条,设立警示标志。\n"
+ "2 索夹在主缆上定位后,应紧固螺栓。"
+ )
+ lines = [
+ _ln("8.14.11 索夹与吊索施工应符合下列规定:", 50, 700, 300, 712),
+ _ln("1 在满足施工需要的前提下,应减小猫道面层开孔面积,并应在开孔位置四周绑", 60, 684, 540, 696),
+ _ln("扎防滑木条,设立警示标志。", 50, 668, 200, 680),
+ _ln("2 索夹在主缆上定位后,应紧固螺栓。", 60, 652, 400, 664),
+ ]
+ blocks = page_blocks(page, 10, 57, lines, PageGeometry((0, 0, 600, 800)))
+ self.assertEqual(len(blocks), 3)
+ self.assertEqual(blocks[0]["start"], 10)
+ item1 = blocks[1]
+ self.assertEqual(page[item1["start"] - 10:item1["end"] - 10].split("\n")[0][:1], "1")
+ self.assertIn("设立警示标志。", page[item1["start"] - 10:item1["end"] - 10])
+ self.assertEqual(item1["locator"]["page"], 57)
+ x0, y0, x1, y1 = item1["locator"]["bbox"]
+ self.assertAlmostEqual(y0, 1 - 696 / 800, places=3)
+ self.assertAlmostEqual(y1, 1 - 668 / 800, places=3)
+ self.assertEqual(blocks[-1]["end"], 10 + len(page))
+
+ def test_page_without_lines_gets_page_block(self):
+ blocks = page_blocks("文字", 5, 2, [], None)
+ self.assertEqual(blocks, [{"start": 5, "end": 7, "locator": {"type": "pdf", "page": 2}}])
+
+
+class WireTest(unittest.TestCase):
+ def test_source_blocks_serialize_and_skip_malformed(self):
+ from docreader.source_wire import source_blocks_to_proto
+
+ doc = Document(
+ content="abc",
+ source_blocks=[
+ {"start": 0, "end": 3, "locator": {"type": "pdf", "page": 1}},
+ {"start": 2, "end": 2, "locator": {"type": "pdf", "page": 1}},
+ {"start": 0, "end": 1},
+ ],
+ )
+ wire = source_blocks_to_proto(doc)
+ self.assertEqual(len(wire), 1)
+ self.assertEqual(json.loads(wire[0].locator_json), {"type": "pdf", "page": 1})
+
+
+class LocateColumnsTest(unittest.TestCase):
+ def test_repeated_line_follows_its_own_column(self):
+ text = (
+ "alpha left first line\nsame tail phrase here\n"
+ "bravo left third line\nsame tail phrase here\n"
+ "omega right first line\nsigma right second line"
+ )
+ left, right = (0, 100), (120, 220)
+ lines = [
+ ("alpha left first line", left, 700),
+ ("omega right first line", right, 700),
+ ("same tail phrase here", left, 680),
+ ("sigma right second line", right, 680),
+ ("bravo left third line", left, 660),
+ ("same tail phrase here", left, 640),
+ ]
+ hits = locate_lines(
+ text,
+ [t for t, _, _ in lines],
+ [(x[0], y, x[1], y + 10) for _, x, y in lines],
+ )
+ self.assertEqual(hits[2], text.index("same tail"))
+ self.assertEqual(hits[5], text.rindex("same tail"))
+
+ def test_short_repeated_label_is_not_placed(self):
+ text = "Figure shows the LLM pipeline\nThe LLM is prompted twice"
+ hits = locate_lines(text, ["LLM", "The LLM is prompted twice"])
+ self.assertIsNone(hits[0])
+ self.assertEqual(hits[1], text.index("The LLM"))
+
+
+class SplitLineAtGapsTest(unittest.TestCase):
+ @staticmethod
+ def _line(chars):
+ glyphs = [
+ {"ch": ch, "x0": x, "x1": x + 5, "y0": 0.0, "y1": 10.0, "i": i}
+ for ch, x, i in chars
+ ]
+ return {"text": "".join(c for c, _, _ in chars), "bbox": (0, 0, 0, 0), "chars": glyphs}
+
+ def test_cuts_fused_columns_at_a_stream_jump(self):
+ from docreader.parser.pdf_parser import _split_line_at_gaps
+
+ left = [(ch, 10 + 5 * k, 100 + k) for k, ch in enumerate("left side")]
+ right = [(ch, 80 + 5 * k, 900 + k) for k, ch in enumerate("right side")]
+ parts = _split_line_at_gaps(self._line(left + right))
+ self.assertEqual([p["bbox"][0] for p in parts], [10, 80])
+
+ def test_keeps_a_wide_word_space_within_one_column(self):
+ from docreader.parser.pdf_parser import _split_line_at_gaps
+
+ chars = [(ch, 10 + 5 * k, k) for k, ch in enumerate("wide")]
+ chars += [(ch, 60 + 5 * k, 5 + k) for k, ch in enumerate("space")]
+ self.assertEqual(len(_split_line_at_gaps(self._line(chars))), 1)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/frontend/package-lock.json b/frontend/package-lock.json
index bdbe9e176..0f70a6ed2 100644
--- a/frontend/package-lock.json
+++ b/frontend/package-lock.json
@@ -26,6 +26,7 @@
"mermaid": "^11.15.0",
"pagefind": "^1.5.2",
"papaparse": "^5.5.3",
+ "pdfjs-dist": "^6.3.289",
"pinia": "^3.0.4",
"swiper": "^12.1.4",
"tdesign-icons-vue-next": "0.4.4",
@@ -1061,6 +1062,256 @@
"resolved": "https://mirrors.tencent.com/npm/@microsoft/fetch-event-source/-/fetch-event-source-2.0.1.tgz",
"integrity": "sha512-W6CLUJ2eBMw3Rec70qrsEW0jOm/3twwJv21mrmj2yORiaVmVYGS4sSS5yUwvQc1ZlDLYGPnClVWmUUMagKNsfA=="
},
+ "node_modules/@napi-rs/canvas": {
+ "version": "1.0.9",
+ "resolved": "https://mirrors.tencent.com/npm/@napi-rs/canvas/-/canvas-1.0.9.tgz",
+ "integrity": "sha512-QviPdJImDi/jMAvBfqaw+19BndMd/sizXVW3NnpMd3VJGz++QXkOHcP9kWR/smHG0hNjHeyuHFyrx/5lD0oNcQ==",
+ "license": "MIT",
+ "optional": true,
+ "workspaces": [
+ "e2e/*"
+ ],
+ "engines": {
+ "node": ">= 10"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/sponsors/Brooooooklyn"
+ },
+ "optionalDependencies": {
+ "@napi-rs/canvas-android-arm64": "1.0.9",
+ "@napi-rs/canvas-darwin-arm64": "1.0.9",
+ "@napi-rs/canvas-darwin-x64": "1.0.9",
+ "@napi-rs/canvas-linux-arm-gnueabihf": "1.0.9",
+ "@napi-rs/canvas-linux-arm64-gnu": "1.0.9",
+ "@napi-rs/canvas-linux-arm64-musl": "1.0.9",
+ "@napi-rs/canvas-linux-riscv64-gnu": "1.0.9",
+ "@napi-rs/canvas-linux-x64-gnu": "1.0.9",
+ "@napi-rs/canvas-linux-x64-musl": "1.0.9",
+ "@napi-rs/canvas-win32-arm64-msvc": "1.0.9",
+ "@napi-rs/canvas-win32-x64-msvc": "1.0.9"
+ }
+ },
+ "node_modules/@napi-rs/canvas-android-arm64": {
+ "version": "1.0.9",
+ "resolved": "https://mirrors.tencent.com/npm/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-1.0.9.tgz",
+ "integrity": "sha512-4LGXk2/0HVzE29K8SzML5WubgCp++B1FH3qgl35XmSZE+lLdr6P9VRQEnZ0MCLZMTSuJP41yyhtoiVNEuvrTIA==",
+ "cpu": [
+ "arm64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "android"
+ ],
+ "engines": {
+ "node": ">= 10"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/sponsors/Brooooooklyn"
+ }
+ },
+ "node_modules/@napi-rs/canvas-darwin-arm64": {
+ "version": "1.0.9",
+ "resolved": "https://mirrors.tencent.com/npm/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-1.0.9.tgz",
+ "integrity": "sha512-YNdfLBzY0W/Pep9fo2L6RmoNlNksnn05LRnX66W63R3ij58S25QOTcjdtEt2v8+PnCESzqZsYzUo+QPeIR44NA==",
+ "cpu": [
+ "arm64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "darwin"
+ ],
+ "engines": {
+ "node": ">= 10"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/sponsors/Brooooooklyn"
+ }
+ },
+ "node_modules/@napi-rs/canvas-darwin-x64": {
+ "version": "1.0.9",
+ "resolved": "https://mirrors.tencent.com/npm/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-1.0.9.tgz",
+ "integrity": "sha512-ceZQSknTEcy3dOXoekv59LTCkXjvnLsq+VW5PeNNDHEPQbRS5Ervkm1EaDa7WLAjiYWMLuSQTRTHao1dEX4prg==",
+ "cpu": [
+ "x64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "darwin"
+ ],
+ "engines": {
+ "node": ">= 10"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/sponsors/Brooooooklyn"
+ }
+ },
+ "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": {
+ "version": "1.0.9",
+ "resolved": "https://mirrors.tencent.com/npm/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-1.0.9.tgz",
+ "integrity": "sha512-XhfI0Wwv4llhd6nnWDtY3kQKjq0r+y1i91PlJlJI24ag2U9WrnwbG1qS3+fDLEyouwEVFchiKkTEHozK+5iUNA==",
+ "cpu": [
+ "arm"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">= 10"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/sponsors/Brooooooklyn"
+ }
+ },
+ "node_modules/@napi-rs/canvas-linux-arm64-gnu": {
+ "version": "1.0.9",
+ "resolved": "https://mirrors.tencent.com/npm/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-1.0.9.tgz",
+ "integrity": "sha512-012oiYtKaE7i9oxc8q7nraT7kDOpLcaCmFLzVe9Ty34RHDdoDzbWLrVh827CNxYh/EADX1eSikA3ymLjo/nNuw==",
+ "cpu": [
+ "arm64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">= 10"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/sponsors/Brooooooklyn"
+ }
+ },
+ "node_modules/@napi-rs/canvas-linux-arm64-musl": {
+ "version": "1.0.9",
+ "resolved": "https://mirrors.tencent.com/npm/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-1.0.9.tgz",
+ "integrity": "sha512-Ls5UWYFFn63casTZEczbeyEg3vRDRkv9lscuGwfchtY5yLLQhgOB8SN4YGxmfJ5vTaBwZ2YUxBS3NtmjJmFXdA==",
+ "cpu": [
+ "arm64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">= 10"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/sponsors/Brooooooklyn"
+ }
+ },
+ "node_modules/@napi-rs/canvas-linux-riscv64-gnu": {
+ "version": "1.0.9",
+ "resolved": "https://mirrors.tencent.com/npm/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-1.0.9.tgz",
+ "integrity": "sha512-hLKEGxV7ZiRHqndePTokgDMdBlo/rDfzg7P4p4QIv9pUhuYobnu3R2NIFLCRghG0nwfo+s2sw+c1xZFeCmEAsw==",
+ "cpu": [
+ "riscv64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">= 10"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/sponsors/Brooooooklyn"
+ }
+ },
+ "node_modules/@napi-rs/canvas-linux-x64-gnu": {
+ "version": "1.0.9",
+ "resolved": "https://mirrors.tencent.com/npm/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-1.0.9.tgz",
+ "integrity": "sha512-6kaz3w0QMy77PDWk6rJ1ksIihdad3qzEyX2o2oGT8GwCaypfT5mhjr8buOO5hstyLxcWXDScuz56RsINLtBPIQ==",
+ "cpu": [
+ "x64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">= 10"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/sponsors/Brooooooklyn"
+ }
+ },
+ "node_modules/@napi-rs/canvas-linux-x64-musl": {
+ "version": "1.0.9",
+ "resolved": "https://mirrors.tencent.com/npm/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-1.0.9.tgz",
+ "integrity": "sha512-xrGvmS3v55hmZ86ls/kBLVNMUTYio3f6Ik0DireemG994VfPAwiA3ZXA0Uf1bByctkB3NQ1Sfb+H5bkdUnnzfQ==",
+ "cpu": [
+ "x64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">= 10"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/sponsors/Brooooooklyn"
+ }
+ },
+ "node_modules/@napi-rs/canvas-win32-arm64-msvc": {
+ "version": "1.0.9",
+ "resolved": "https://mirrors.tencent.com/npm/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-1.0.9.tgz",
+ "integrity": "sha512-yjmVS3ArZeRVCP7jqbPq4rpZa/BhTeI7ELE2XqJg3snICQBDevLZyArxswHkiTnT34KRic33/4fLirrHI+SY8A==",
+ "cpu": [
+ "arm64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "win32"
+ ],
+ "engines": {
+ "node": ">= 10"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/sponsors/Brooooooklyn"
+ }
+ },
+ "node_modules/@napi-rs/canvas-win32-x64-msvc": {
+ "version": "1.0.9",
+ "resolved": "https://mirrors.tencent.com/npm/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-1.0.9.tgz",
+ "integrity": "sha512-QlSYQdMQslB81nlABo9wNfQ6npFhE7/O+saCZdqVGueGanyRk4jCogD5EwQenfP3kIq9e+mm6GreQBjX5MrA8g==",
+ "cpu": [
+ "x64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "win32"
+ ],
+ "engines": {
+ "node": ">= 10"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/sponsors/Brooooooklyn"
+ }
+ },
"node_modules/@novnc/novnc": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/@novnc/novnc/-/novnc-1.7.0.tgz",
@@ -4302,6 +4553,18 @@
"integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==",
"license": "MIT"
},
+ "node_modules/pdfjs-dist": {
+ "version": "6.3.289",
+ "resolved": "https://mirrors.tencent.com/npm/pdfjs-dist/-/pdfjs-dist-6.3.289.tgz",
+ "integrity": "sha512-ZHjSVpDa3D6izMq8/04lvkhkATUmL9px6ChPaXc1k6nU2Mrhlg1/7F0bdUqCwUjw3NsPTfPZsMDUU6ZIcRaeQw==",
+ "license": "Apache-2.0",
+ "engines": {
+ "node": ">=22.13.0 || >=24"
+ },
+ "optionalDependencies": {
+ "@napi-rs/canvas": "^1.0.0"
+ }
+ },
"node_modules/perfect-debounce": {
"version": "1.0.0",
"resolved": "https://mirrors.tencent.com/npm/perfect-debounce/-/perfect-debounce-1.0.0.tgz",
diff --git a/frontend/package.json b/frontend/package.json
index 1b1b5354b..ee38c4824 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -34,6 +34,7 @@
"mermaid": "^11.15.0",
"pagefind": "^1.5.2",
"papaparse": "^5.5.3",
+ "pdfjs-dist": "^6.3.289",
"pinia": "^3.0.4",
"swiper": "^12.1.4",
"tdesign-icons-vue-next": "0.4.4",
diff --git a/frontend/src/assets/theme/theme.css b/frontend/src/assets/theme/theme.css
index 51aa65412..aef4d07cf 100644
--- a/frontend/src/assets/theme/theme.css
+++ b/frontend/src/assets/theme/theme.css
@@ -217,6 +217,14 @@ select:-webkit-autofill {
--app-accent-purple: #9d7cff;
}
+/* 引用原文定位的高亮:荧光笔黄,和品牌绿的界面元素(表头、选中态)区分开 */
+:root {
+ --app-source-highlight: #ffd84d;
+}
+:root[theme-mode="dark"] {
+ --app-source-highlight: #b58f1c;
+}
+
/* 全局旋转动画:spinner / 刷新图标统一用 `animation: wk-spin 0.8s linear infinite` */
@keyframes wk-spin {
to {
diff --git a/frontend/src/components/ChatReferenceSourceView.vue b/frontend/src/components/ChatReferenceSourceView.vue
new file mode 100644
index 000000000..38c2931b3
--- /dev/null
+++ b/frontend/src/components/ChatReferenceSourceView.vue
@@ -0,0 +1,349 @@
+<template>
+ <div class="reference-source">
+ <div class="reference-source__meta">
+ <ArtifactFileIcon :file-name="displayName" />
+ <div class="reference-source__meta-main">
+ <h4 class="reference-source__title" :title="displayTitle">{{ displayTitle }}</h4>
+ <p v-if="statusText" class="reference-source__status" :class="`is-${status}`">{{ statusText }}</p>
+ </div>
+ <div ref="toolbarSlot" class="reference-source__toolbar" />
+ <t-button
+ v-if="mode === 'file' && locate"
+ theme="default" variant="text" size="small" shape="square"
+ :title="t('chat.referenceSourceRelocate')" :aria-label="t('chat.referenceSourceRelocate')"
+ @click="relocate"
+ >
+ <template #icon><t-icon name="location" /></template>
+ </t-button>
+ <a
+ v-if="documentHref"
+ class="reference-source__open"
+ :href="documentHref"
+ target="_blank"
+ rel="noopener noreferrer"
+ :title="t('chat.navigateToDocument')"
+ :aria-label="t('chat.navigateToDocument')"
+ >
+ <t-icon name="jump" size="16px" />
+ </a>
+ </div>
+
+ <div v-if="mode === 'loading'" class="reference-source__placeholder">
+ <t-loading size="small" />
+ </div>
+
+ <DocumentPreview
+ v-else-if="mode === 'file'"
+ class="reference-source__preview"
+ :knowledge-id="target.knowledgeId"
+ :file-type="meta.fileType"
+ :file-name="meta.fileName"
+ :active="active"
+ source-mode
+ fill-height
+ :toolbar-target="toolbarSlot"
+ :locate="locate"
+ @located="onLocated"
+ />
+
+ <div v-else-if="mode === 'web'" class="reference-source__web">
+ <p class="reference-source__quote">{{ webQuote }}</p>
+ <t-button theme="primary" variant="outline" size="small" @click="openWebSource">
+ <template #icon><t-icon name="jump" /></template>
+ {{ t('chat.referenceSourceOpenWeb') }}
+ </t-button>
+ <p class="reference-source__url">{{ meta.knowledgeSource }}</p>
+ </div>
+ </div>
+</template>
+
+<script setup lang="ts">
+import { computed, reactive, ref, watch } from 'vue'
+import { useI18n } from 'vue-i18n'
+import { useRouter } from 'vue-router'
+import DocumentPreview from '@/components/document-preview.vue'
+import ArtifactFileIcon from '@/views/chat/components/ArtifactFileIcon.vue'
+import { getChunkByIdOnly, getKnowledgeDetails } from '@/api/knowledge-base/index'
+import { isKnownPreviewableFile } from '@/utils/filePreview'
+import type { ReferenceSourceTarget } from '@/utils/referenceSources'
+import {
+ locatorPages,
+ parseSourceLocators,
+ selectLocatorsForSentence,
+ selectQuoteForSentence,
+ textFragmentUrl,
+ type SourceLocateRequest,
+ type SourceLocator,
+} from '@/utils/sourceLocator'
+
+const props = defineProps<{
+ target: ReferenceSourceTarget
+ active: boolean
+}>()
+
+const emit = defineEmits<{
+ /** The cited document has no original file this view can show. */
+ unavailable: []
+}>()
+
+const { t } = useI18n()
+const router = useRouter()
+
+type Mode = 'loading' | 'file' | 'web' | 'none'
+type Status = 'idle' | 'locating' | 'found' | 'page' | 'missing'
+
+const mode = ref<Mode>('loading')
+const toolbarSlot = ref<HTMLElement | null>(null)
+const status = ref<Status>('idle')
+const locatedPage = ref(0)
+const locate = ref<SourceLocateRequest | null>(null)
+const locators = ref<SourceLocator[]>([])
+const chunkContent = ref('')
+const chunkType = ref('')
+const meta = reactive({ fileName: '', fileType: '', knowledgeSource: '', knowledgeType: '', title: '' })
+let loadVersion = 0
+/** The knowledge whose file the preview currently shows. */
+let shownKnowledgeId = ''
+
+const displayName = computed(() => meta.fileName || props.target.fileName || props.target.title || '')
+const displayTitle = computed(() => props.target.title || meta.title || displayName.value)
+
+const documentHref = computed(() => {
+ if (!props.target.knowledgeBaseId) return ''
+ return router.resolve({
+ path: `/platform/knowledge-bases/${props.target.knowledgeBaseId}`,
+ query: { knowledge_id: props.target.knowledgeId },
+ }).href
+})
+
+const statusText = computed(() => {
+ switch (status.value) {
+ case 'locating':
+ return t('chat.referenceSourceLocating')
+ case 'found':
+ return locatedPage.value ? t('chat.referenceSourceFoundPage', { page: locatedPage.value }) : ''
+ case 'page':
+ return t('chat.referenceSourceFoundPage', { page: locatedPage.value })
+ case 'missing':
+ return t('chat.referenceSourceNotFound')
+ default:
+ return ''
+ }
+})
+
+const webQuote = computed(() => {
+ const quote = locators.value.find((l) => l.quote)?.quote || chunkContent.value || props.target.content || ''
+ return quote.replace(/\s+/g, ' ').trim().slice(0, 280)
+})
+
+function isWebUrl(value: string) {
+ return /^https?:\/\//i.test(value || '')
+}
+
+async function loadMeta(version: number) {
+ meta.fileName = props.target.fileName || ''
+ meta.fileType = ''
+ meta.knowledgeSource = props.target.knowledgeSource || ''
+ meta.knowledgeType = ''
+ meta.title = props.target.title || ''
+ if (meta.fileName && meta.knowledgeSource) return
+ try {
+ const res: any = await getKnowledgeDetails(props.target.knowledgeId)
+ if (version !== loadVersion) return
+ const data = res?.data || {}
+ meta.fileName = meta.fileName || data.file_name || ''
+ meta.fileType = data.file_type || ''
+ meta.knowledgeSource = meta.knowledgeSource || data.source || ''
+ meta.knowledgeType = data.type || ''
+ meta.title = meta.title || data.title || ''
+ } catch {
+ // Without details the file name from the reference decides below.
+ }
+}
+
+async function loadChunk(version: number) {
+ locators.value = parseSourceLocators(props.target.locators)
+ chunkContent.value = ''
+ chunkType.value = props.target.chunkType || ''
+ if (locators.value.length) return
+ // Old conversations and agent tool results carry no locators; the chunk
+ // itself has them (and its own, unexpanded text for matching).
+ try {
+ const res: any = await getChunkByIdOnly(props.target.chunkId)
+ if (version !== loadVersion) return
+ locators.value = parseSourceLocators(res?.data?.source_locators)
+ chunkContent.value = String(res?.data?.content || '')
+ chunkType.value = chunkType.value || String(res?.data?.chunk_type || '')
+ } catch {
+ chunkContent.value = props.target.content || ''
+ }
+}
+
+function resolveMode(): Mode {
+ const type = (meta.knowledgeType || meta.fileType || '').toLowerCase()
+ if (meta.knowledgeSource === 'manual' || type === 'manual' || type === 'faq' || props.target.chunkType === 'faq') return 'none'
+ if (meta.fileName && isKnownPreviewableFile(meta.fileName, meta.fileType)) return 'file'
+ if (isWebUrl(meta.knowledgeSource)) return 'web'
+ return 'none'
+}
+
+function buildLocate(): SourceLocateRequest {
+ const anchor = props.target.anchorText || ''
+ const selected = selectLocatorsForSentence(locators.value, anchor)
+ const quotes = chunkContent.value ? selectQuoteForSentence(chunkContent.value, anchor) : []
+ const scope = chunkContent.value || props.target.content || ''
+ return { locators: selected, quotes, token: Date.now(), sentence: anchor, scope }
+}
+
+async function load() {
+ const version = ++loadVersion
+ status.value = 'idle'
+ locatedPage.value = 0
+ locate.value = null
+ // Another citation in the open file: keep the preview mounted and only
+ // move the highlight, instead of downloading and rendering the file again.
+ const sameFile = mode.value === 'file' && shownKnowledgeId === props.target.knowledgeId
+ if (!sameFile) mode.value = 'loading'
+ await Promise.all([sameFile ? Promise.resolve() : loadMeta(version), loadChunk(version)])
+ if (version !== loadVersion) return
+ shownKnowledgeId = props.target.knowledgeId
+ const next = resolveMode()
+ mode.value = next
+ if (next === 'none') {
+ emit('unavailable')
+ return
+ }
+ // A summary speaks for the whole document: open it without hunting for a passage.
+ if (next === 'file' && chunkType.value !== 'summary') {
+ status.value = 'locating'
+ locate.value = buildLocate()
+ }
+}
+
+function relocate() {
+ if (!locate.value) return
+ status.value = 'locating'
+ locate.value = { ...locate.value, token: Date.now() }
+}
+
+function onLocated(result: { found: boolean; precise: boolean; page?: number }) {
+ locatedPage.value = result.page || locatorPages(locate.value?.locators || [])[0] || 0
+ if (!result.found) status.value = 'missing'
+ else if (!result.precise && locatedPage.value) status.value = 'page'
+ else status.value = 'found'
+}
+
+function openWebSource() {
+ const url = textFragmentUrl(meta.knowledgeSource, webQuote.value.slice(0, 120))
+ window.open(url, '_blank', 'noopener,noreferrer')
+}
+
+watch(
+ () => [props.target.chunkId, props.target.knowledgeId, props.target.anchorText],
+ () => void load(),
+ { immediate: true },
+)
+</script>
+
+<style scoped lang="less">
+.reference-source {
+ display: flex;
+ flex-direction: column;
+ height: 100%;
+ min-height: 0;
+}
+
+.reference-source__meta {
+ display: flex;
+ align-items: center;
+ gap: var(--app-space-2);
+ padding: 8px 12px;
+ border-bottom: 1px solid var(--td-component-stroke);
+ flex-shrink: 0;
+}
+
+.reference-source__meta-main {
+ flex: 1;
+ min-width: 0;
+}
+
+.reference-source__title {
+ margin: 0;
+ font-size: var(--app-text-base);
+ font-weight: 500;
+ line-height: 20px;
+ color: var(--td-text-color-primary);
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+}
+
+.reference-source__status {
+ margin: 2px 0 0;
+ font-size: var(--app-text-sm);
+ line-height: 1.4;
+ color: var(--td-text-color-placeholder);
+
+ &.is-missing {
+ color: var(--td-warning-color);
+ }
+}
+
+.reference-source__toolbar {
+ display: flex;
+ align-items: center;
+}
+
+.reference-source__open {
+ display: inline-flex;
+ align-items: center;
+ justify-content: center;
+ width: 28px;
+ height: 28px;
+ border-radius: var(--app-radius-md);
+ color: var(--td-text-color-placeholder);
+
+ &:hover {
+ color: var(--td-text-color-primary);
+ background: var(--td-bg-color-container-hover);
+ }
+}
+
+.reference-source__placeholder {
+ display: flex;
+ justify-content: center;
+ padding: 32px 0;
+}
+
+.reference-source__preview {
+ flex: 1;
+ min-height: 0;
+}
+
+.reference-source__web {
+ display: flex;
+ flex-direction: column;
+ align-items: flex-start;
+ gap: 12px;
+ padding: 16px 12px;
+}
+
+.reference-source__quote {
+ margin: 0;
+ padding: 10px 12px;
+ border-left: 3px solid var(--app-source-highlight);
+ background: color-mix(in srgb, var(--app-source-highlight) 18%, transparent);
+ font-size: var(--app-text-md);
+ line-height: 1.6;
+ color: var(--td-text-color-primary);
+ white-space: pre-wrap;
+ word-break: break-word;
+}
+
+.reference-source__url {
+ margin: 0;
+ font-size: var(--app-text-sm);
+ color: var(--td-text-color-placeholder);
+ word-break: break-all;
+}
+</style>
diff --git a/frontend/src/components/ChatReferencesDrawer.vue b/frontend/src/components/ChatReferencesDrawer.vue
index 39b982792..b886d5ec7 100644
--- a/frontend/src/components/ChatReferencesDrawer.vue
+++ b/frontend/src/components/ChatReferencesDrawer.vue
@@ -4,13 +4,25 @@
<aside
v-if="visible"
class="chat-references-panel"
- :class="{ 'is-overlay': useOverlay, 'is-embedded': embeddedMode }"
+ :class="{ 'is-overlay': useOverlay, 'is-embedded': embeddedMode, 'is-source': !!sourceTarget }"
+ :style="panelStyle"
role="complementary"
:aria-label="panelTitle"
>
<header class="chat-references-panel__header">
<div class="chat-references-panel__heading">
- <h3 class="chat-references-panel__title">
+ <button
+ v-if="sourceTarget"
+ type="button"
+ class="chat-references-panel__back"
+ :aria-label="t('chat.referenceSourceBack')"
+ @click="backToList"
+ >
+ <t-icon name="chevron-left" size="16px" />
+ <span>{{ t('chat.referenceSourceBack') }}</span>
+ <span v-if="totalCount" class="chat-references-panel__count"> · {{ totalCount }}</span>
+ </button>
+ <h3 v-else class="chat-references-panel__title">
{{ panelTitle }}<span v-if="totalCount" class="chat-references-panel__count"> · {{ totalCount }}</span>
</h3>
</div>
@@ -24,7 +36,15 @@
</button>
</header>
- <div ref="listElement" class="chat-references-panel__body">
+ <ChatReferenceSourceView
+ v-if="sourceTarget"
+ class="chat-references-panel__source"
+ :target="sourceTarget"
+ :active="visible"
+ @unavailable="onSourceUnavailable"
+ />
+
+ <div v-else ref="listElement" class="chat-references-panel__body">
<div v-if="sections.length === 0" class="chat-references-panel__empty">
{{ t('chat.referencesDrawerEmpty') }}
</div>
@@ -71,6 +91,16 @@
<div class="reference-item__document-main">
<div class="reference-item__title-row">
<h5 class="reference-item__title" :title="item.title">{{ item.title }}</h5>
+ <button
+ v-if="canOpenSource(item)"
+ type="button"
+ class="reference-item__open"
+ :title="t('chat.referenceSourceView')"
+ :aria-label="t('chat.referenceSourceView')"
+ @click.stop="openItemSource(item)"
+ >
+ <t-icon name="file-search" size="14px" />
+ </button>
<a
v-if="item.knowledgeBaseId && !embeddedMode"
class="reference-item__open"
@@ -136,11 +166,12 @@
</template>
<script setup lang="ts">
-import { computed, nextTick, reactive, ref, watch } from 'vue'
+import { computed, nextTick, onBeforeUnmount, onMounted, reactive, ref, watch, watchEffect } from 'vue'
import { useI18n } from 'vue-i18n'
import { useRouter } from 'vue-router'
-import { useChatReferencesDrawer } from '@/composables/useChatReferencesDrawer'
+import { REFERENCES_PANEL_WIDTH, useChatReferencesDrawer } from '@/composables/useChatReferencesDrawer'
import ArtifactFileIcon from '@/views/chat/components/ArtifactFileIcon.vue'
+import ChatReferenceSourceView from '@/components/ChatReferenceSourceView.vue'
import {
buildReferenceSections,
formatReferenceSnippet,
@@ -166,13 +197,62 @@ const panelEntered = ref(false)
const visible = computed(() => drawer?.visible.value ?? false)
const references = computed(() => drawer?.references.value ?? [])
const highlight = computed(() => drawer?.highlight.value ?? null)
+// The embed has no access to original files, so it always lists sources.
+const sourceTarget = computed(() => (props.embeddedMode ? null : drawer?.source.value ?? null))
+
+const viewportWidth = ref(typeof window === 'undefined' ? 1440 : window.innerWidth)
+const onViewportResize = () => {
+ viewportWidth.value = window.innerWidth
+}
+onMounted(() => window.addEventListener('resize', onViewportResize))
+onBeforeUnmount(() => window.removeEventListener('resize', onViewportResize))
+
+// Reading an original document needs more room than the source list.
+const panelWidth = computed(() => {
+ if (!sourceTarget.value) return REFERENCES_PANEL_WIDTH
+ return Math.max(480, Math.min(780, Math.round(viewportWidth.value * 0.46)))
+})
+
+watchEffect(() => {
+ if (drawer) drawer.panelWidth.value = panelWidth.value
+})
const useOverlay = computed(() => {
if (props.embeddedMode) return true
- if (typeof window === 'undefined') return false
- return window.innerWidth < (props.overlayBreakpoint ?? 960)
+ return viewportWidth.value < (props.overlayBreakpoint ?? 960)
})
+const panelStyle = computed(() => {
+ if (!sourceTarget.value) return undefined
+ return { width: useOverlay.value ? 'min(760px, 100vw)' : `${panelWidth.value}px` }
+})
+
+function canOpenSource(item: ReferenceListItem) {
+ return !props.embeddedMode && item.kind === 'document' && !!item.knowledgeId && !!item.chunkId
+}
+
+function openItemSource(item: ReferenceListItem) {
+ if (!drawer || !item.knowledgeId || !item.chunkId) return
+ drawer.openSource({
+ chunkId: item.sourceChunkId || item.chunkId,
+ knowledgeId: item.knowledgeId,
+ knowledgeBaseId: item.knowledgeBaseId,
+ title: item.title,
+ fileName: item.fileName,
+ })
+}
+
+function backToList() {
+ drawer?.closeSource()
+ void nextTick(() => scrollToHighlight())
+}
+
+function onSourceUnavailable() {
+ // No original file to show (manual entry, FAQ, deleted file): fall back
+ // to the list with the cited card highlighted.
+ backToList()
+}
+
const sections = computed(() => buildReferenceSections(references.value))
const totalCount = computed(() => sections.value.reduce((sum, section) => sum + section.items.length, 0))
@@ -354,6 +434,31 @@ watch(visible, (open) => {
}
}
+.chat-references-panel__source {
+ flex: 1;
+ min-height: 0;
+}
+
+.chat-references-panel__back {
+ display: inline-flex;
+ align-items: center;
+ gap: 2px;
+ min-width: 0;
+ padding: 4px 6px 4px 2px;
+ border: 0;
+ border-radius: var(--app-radius-md);
+ background: transparent;
+ color: var(--td-text-color-secondary);
+ font-size: var(--app-text-base);
+ font-weight: 500;
+ cursor: pointer;
+
+ &:hover {
+ background: var(--td-bg-color-container-hover);
+ color: var(--td-text-color-primary);
+ }
+}
+
.chat-references-panel__header {
display: flex;
align-items: center;
@@ -575,6 +680,10 @@ watch(visible, (open) => {
.reference-item__open {
flex-shrink: 0;
margin-top: 3px;
+ padding: 0;
+ border: 0;
+ background: transparent;
+ cursor: pointer;
color: var(--td-text-color-placeholder);
line-height: 1;
opacity: 0;
diff --git a/frontend/src/components/chat/SandboxSidePanel.vue b/frontend/src/components/chat/SandboxSidePanel.vue
index 330758ccc..1b2e72cef 100644
--- a/frontend/src/components/chat/SandboxSidePanel.vue
+++ b/frontend/src/components/chat/SandboxSidePanel.vue
@@ -4,7 +4,7 @@
<aside
class="chat-sandbox-panel"
:class="{ 'is-shifted': shifted, 'is-resizing': resizing }"
- :style="{ width: `${panel?.width.value ?? 420}px` }"
+ :style="{ width: `${panel?.width.value ?? 420}px`, '--references-shift': `${shiftWidth}px` }"
role="complementary"
:aria-label="t('chat.sandbox.panelTitle')"
>
@@ -119,12 +119,15 @@ const props = withDefaults(
agentSourceTenantId?: string | number | null
/** 参考来源面板同开时整体左移,避免两块 fixed 面板重叠。 */
shifted?: boolean
+ /** 参考来源面板当前宽度(查看原文时会变宽),左移的距离。 */
+ shiftWidth?: number
artifacts?: SessionArtifactItem[]
artifactsCollecting?: boolean
}>(),
{
artifacts: () => [],
artifactsCollecting: false,
+ shiftWidth: 420,
},
)
@@ -266,7 +269,7 @@ function resizePanel(delta: number) {
&.is-shifted {
@media (min-width: 1400px) {
- right: 420px;
+ right: var(--references-shift, 420px);
}
}
diff --git a/frontend/src/components/document-preview.vue b/frontend/src/components/document-preview.vue
index ded9ad8ac..825dff8de 100644
--- a/frontend/src/components/document-preview.vue
+++ b/frontend/src/components/document-preview.vue
@@ -26,9 +26,27 @@ import {
isValidUTF8,
type FilePreviewKind,
} from '@/utils/filePreview';
+import {
+ narrowTextLines,
+ ngramOverlap,
+ normalizeForMatch,
+ pickBestTexts,
+ type SourceLocateRequest,
+ type SourceLocator,
+} from '@/utils/sourceLocator';
+import {
+ buildTextIndex,
+ findTextRange,
+ highlightElements,
+ highlightRanges,
+ rangeForOffsets,
+ scrollRectIntoContainer,
+} from '@/utils/sourceLocatorDom';
+import { renderEpubPreview } from '@/utils/epubPreview';
const VueOfficePptx = defineAsyncComponent(() => import('@vue-office/pptx'));
+const PdfSourceViewer = defineAsyncComponent(() => import('@/components/source-preview/PdfSourceViewer.vue'));
const { t } = useI18n();
@@ -45,6 +63,19 @@ const props = defineProps<{
fillHeight?: boolean;
/** Place preview actions beside the host header's download button. */
toolbarTarget?: HTMLElement | null;
+ /**
+ * Source mode renders PDFs with pdf.js instead of the browser viewer so a
+ * citation can be scrolled to and highlighted.
+ */
+ sourceMode?: boolean;
+ /** Where to scroll and what to highlight once the file is rendered. */
+ locate?: SourceLocateRequest | null;
+}>();
+
+export type SourceLocateResult = { found: boolean; precise: boolean };
+
+const emit = defineEmits<{
+ located: [result: SourceLocateResult];
}>();
const loading = ref(false);
@@ -58,9 +89,18 @@ const excelHtml = ref('');
const mermaidSvg = ref('');
const htmlViewMode = ref<'render' | 'source'>('render');
const pptxData = shallowRef<ArrayBuffer | null>(null);
+const pdfData = shallowRef<ArrayBuffer | null>(null);
+const epubHtml = ref('');
+let epubObjectUrls: string[] = [];
+let excelSheetNames: string[] = [];
+let previewExt = '';
let pptxSlideCount = 0;
function onPptxRendered(result: unknown) {
- if (!isCompletePptxRender(result, pptxSlideCount)) error.value = t('preview.loadFailed');
+ if (!isCompletePptxRender(result, pptxSlideCount)) {
+ error.value = t('preview.loadFailed');
+ return;
+ }
+ scheduleLocate();
}
const docxContainer = ref<HTMLElement | null>(null);
const imageNaturalWidth = ref(0);
@@ -254,6 +294,7 @@ async function renderExcel(blob: Blob, fileType?: string) {
workbook = XLSX.read(arrayBuffer, { type: 'array' });
}
+ excelSheetNames = [...workbook.SheetNames];
let html = '';
workbook.SheetNames.forEach((name, sheetIdx) => {
const sheet = workbook.Sheets[name];
@@ -401,14 +442,28 @@ async function loadPreview() {
loading.value = false;
await nextTick();
+ previewExt = ft;
switch (kind) {
- case 'pdf':
+ case 'pdf': {
+ if (props.sourceMode) {
+ pdfData.value = await blob.arrayBuffer();
+ } else {
+ blobUrl.value = URL.createObjectURL(blob);
+ }
+ break;
+ }
case 'image':
case 'audio':
case 'video': {
blobUrl.value = URL.createObjectURL(blob);
break;
}
+ case 'epub': {
+ const rendered = await renderEpubPreview(await blob.arrayBuffer());
+ epubObjectUrls = rendered.objectUrls;
+ epubHtml.value = sanitizeHTML(rendered.html);
+ break;
+ }
case 'html': {
if (allowsHtmlScriptPreview()) {
const previewHtml = buildHtmlPreview(await blob.text());
@@ -445,6 +500,8 @@ async function loadPreview() {
break;
}
}
+ // PPTX renders asynchronously and locates from its rendered event.
+ if (kind !== 'pptx') scheduleLocate();
} catch (err: any) {
console.error('Document preview failed:', err);
error.value = err?.message || t('preview.loadFailed');
@@ -465,6 +522,12 @@ function cleanup() {
mermaidSvg.value = '';
htmlViewMode.value = 'render';
pptxData.value = null;
+ pdfData.value = null;
+ for (const url of epubObjectUrls) URL.revokeObjectURL(url);
+ epubObjectUrls = [];
+ epubHtml.value = '';
+ excelSheetNames = [];
+ clearLocateMarks();
pptxSlideCount = 0;
imageNaturalWidth.value = 0;
imageNaturalHeight.value = 0;
@@ -474,6 +537,344 @@ function cleanup() {
}
}
+// ── Citation locating ──
+// Each preview kind reveals a SourceLocateRequest its own way: structural
+// locators first (page, block, slide, rows, section, time), then the quoted
+// text inside whatever they narrowed down, then the quotes anywhere.
+
+type ImageMark = { left: number; top: number; width: number; height: number };
+const imageMarks = ref<ImageMark[]>([]);
+const audioQuote = ref('');
+const audioRange = ref('');
+let clearLocate: Array<() => void> = [];
+
+function clearLocateMarks() {
+ for (const clear of clearLocate) clear();
+ clearLocate = [];
+ imageMarks.value = [];
+ audioQuote.value = '';
+ audioRange.value = '';
+}
+
+let locateScheduled = false;
+function scheduleLocate() {
+ if (!props.locate || locateScheduled) return;
+ locateScheduled = true;
+ void nextTick(() => {
+ locateScheduled = false;
+ applyLocate();
+ });
+}
+
+watch(
+ () => props.locate?.token,
+ () => {
+ if (loadedForId && !loading.value) scheduleLocate();
+ },
+);
+
+const NOT_FOUND: SourceLocateResult = { found: false, precise: false };
+
+function locatorQuotes(request: SourceLocateRequest, locators: SourceLocator[] = request.locators): string[] {
+ return [...locators.map((l) => l.quote || '').filter(Boolean), ...request.quotes];
+}
+
+function findQuoteRange(root: Element | null | undefined, quotes: string[]): Range | null {
+ if (!root) return null;
+ const index = buildTextIndex(root);
+ for (const quote of quotes) {
+ const range = findTextRange(root, quote, index);
+ if (range) return range;
+ }
+ return null;
+}
+
+function reveal(container: Element | null | undefined, target: Range | Element | null | undefined) {
+ if (!container || !target) return;
+ scrollRectIntoContainer(container, target.getBoundingClientRect());
+}
+
+function commitMarks(ranges: Range[], elements: Element[]) {
+ if (ranges.length) clearLocate.push(highlightRanges(ranges));
+ if (elements.length) clearLocate.push(highlightElements(elements));
+}
+
+/** Reveal the first quote found under `root`, scrolling `container`. */
+function locateQuote(root: Element | null | undefined, container: Element | null | undefined, quotes: string[]): SourceLocateResult {
+ const range = findQuoteRange(root, quotes);
+ if (!range) return NOT_FOUND;
+ commitMarks([range], []);
+ reveal(container, range);
+ return { found: true, precise: true };
+}
+
+/** The element among `blocks` near index `at` whose text best matches `quote`. */
+function pickNearby(blocks: Element[], at: number, quote?: string): Element | null {
+ if (!blocks.length) return null;
+ const center = Math.max(0, Math.min(blocks.length - 1, at));
+ const needle = normalizeForMatch(quote || '').slice(0, 120);
+ if (needle.length < 4) return blocks[center];
+ let best = { el: blocks[center], score: -1 };
+ for (let i = Math.max(0, center - 4); i <= Math.min(blocks.length - 1, center + 4); i++) {
+ const score = ngramOverlap(needle, normalizeForMatch(blocks[i].textContent || ''));
+ if (score > best.score) best = { el: blocks[i], score };
+ }
+ return best.score >= 0.4 ? best.el : blocks[center];
+}
+
+/** The candidates that best match the cited sentence, or all of them when none aligns. */
+function narrowToSentence(candidates: Element[], request: SourceLocateRequest): Element[] {
+ const picked = pickBestTexts(candidates.map((el) => el.textContent || ''), request.sentence || '');
+ return picked.length ? picked.map((i) => candidates[i]) : candidates;
+}
+
+function contentsRange(el: Element): Range {
+ const range = el.ownerDocument.createRange();
+ range.selectNodeContents(el);
+ return range;
+}
+
+function locateDocx(request: SourceLocateRequest): SourceLocateResult {
+ const root = docxContainer.value;
+ if (!root) return NOT_FOUND;
+ const locators = request.locators.filter((l) => l.type === 'docx' && l.block);
+ if (locators.length) {
+ // Body paragraphs and tables in order, the unit the backend counts.
+ const blocks = Array.from(root.querySelectorAll('section > article')).flatMap((article) =>
+ Array.from(article.children).filter((c) => c.tagName === 'P' || c.tagName === 'TABLE'),
+ );
+ const targets: Element[] = [];
+ for (const loc of locators) {
+ const el = pickNearby(blocks, (loc.block as number) - 1, loc.quote);
+ if (el && !targets.includes(el)) targets.push(el);
+ }
+ if (targets.length) {
+ const range = findQuoteRange(targets[0], locatorQuotes(request, locators));
+ commitMarks(range ? [range] : [], targets);
+ reveal(root, range || targets[0]);
+ return { found: true, precise: true };
+ }
+ }
+ return locateQuote(root, root, request.quotes);
+}
+
+function locatePptx(request: SourceLocateRequest): SourceLocateResult {
+ const box = previewContent.value;
+ if (!box) return NOT_FOUND;
+ const slides = Array.from(box.querySelectorAll('.pptx-preview-slide-wrapper'));
+ const locators = request.locators.filter((l) => l.type === 'slide' && l.slide);
+ if (locators.length) {
+ const slide = slides[(locators[0].slide as number) - 1];
+ if (slide) {
+ // Master and layout text repeats on every slide; match the slide's own.
+ const own = slide.querySelector('.slide-wrapper') || slide;
+ // A slide locator covers the whole slide; the cited bullet is narrower.
+ const paragraphs = Array.from(own.querySelectorAll('p')).filter((p) => (p.textContent || '').trim());
+ const cited = narrowToSentence(paragraphs, request);
+ if (paragraphs.length && cited.length < paragraphs.length) {
+ const ranges = cited.map(contentsRange);
+ commitMarks(ranges, []);
+ reveal(box, ranges[0]);
+ return { found: true, precise: true };
+ }
+ const range = findQuoteRange(own, locatorQuotes(request, locators));
+ commitMarks(range ? [range] : [], range ? [] : [slide]);
+ reveal(box, range || slide);
+ return { found: true, precise: !!range };
+ }
+ }
+ for (const slide of slides) {
+ const own = slide.querySelector('.slide-wrapper') || slide;
+ const range = findQuoteRange(own, request.quotes);
+ if (range) {
+ commitMarks([range], []);
+ reveal(box, range);
+ return { found: true, precise: true };
+ }
+ }
+ return NOT_FOUND;
+}
+
+/** Map sheet row numbers to table rows using the cell ids SheetJS emits. */
+function sheetRows(table: Element): Map<number, Element> {
+ const rows = new Map<number, Element>();
+ for (const cell of Array.from(table.querySelectorAll('td[id]'))) {
+ const match = /-[A-Z]+(\d+)$/.exec(cell.id);
+ const tr = cell.closest('tr');
+ if (match && tr && !rows.has(Number(match[1]))) rows.set(Number(match[1]), tr);
+ }
+ return rows;
+}
+
+function locateExcel(request: SourceLocateRequest): SourceLocateResult {
+ const box = previewContent.value;
+ if (!box) return NOT_FOUND;
+ const locators = request.locators.filter((l) => l.type === 'sheet' && l.row_start);
+ const targets: Element[] = [];
+ for (const loc of locators) {
+ const byName = loc.sheet ? excelSheetNames.indexOf(loc.sheet) : -1;
+ const sheetIdx = byName >= 0 ? byName : 0;
+ const table = box.querySelector(`#user-content-sheet-${sheetIdx}`) || box.querySelector(`#sheet-${sheetIdx}`);
+ if (!table) continue;
+ const rows = sheetRows(table);
+ const last = Math.min(loc.row_end || loc.row_start || 0, (loc.row_start || 0) + 200);
+ for (let r = loc.row_start as number; r <= last; r++) {
+ const tr = rows.get(r);
+ if (tr && !targets.includes(tr)) targets.push(tr);
+ }
+ }
+ if (targets.length) {
+ // A chunk often spans many rows; keep the ones the sentence is about.
+ // Header rows repeat the column names every answer mentions.
+ const body = targets.filter((tr) => tr.parentElement?.firstElementChild !== tr);
+ const cited = body.length > 1 ? narrowToSentence(body, request) : targets;
+ const rows = cited.length < body.length ? cited : targets;
+ commitMarks([], rows);
+ reveal(box, rows[0]);
+ return { found: true, precise: true };
+ }
+ return locateQuote(box, box, request.quotes);
+}
+
+function locateEpub(request: SourceLocateRequest): SourceLocateResult {
+ const box = previewContent.value;
+ if (!box) return NOT_FOUND;
+ const locators = request.locators.filter((l) => l.type === 'section' && l.section);
+ const section = locators.length
+ ? box.querySelector(`[data-epub-section="${locators[0].section}"]`)
+ : null;
+ if (section) {
+ // A section locator covers the whole chapter; the cited paragraph is narrower.
+ const paragraphs = Array.from(section.querySelectorAll('p, li, h1, h2, h3, h4, h5, h6')).filter(
+ (el) => (el.textContent || '').trim(),
+ );
+ const cited = narrowToSentence(paragraphs, request);
+ if (paragraphs.length && cited.length < paragraphs.length) {
+ const ranges = cited.map(contentsRange);
+ commitMarks(ranges, []);
+ reveal(box, ranges[0]);
+ return { found: true, precise: true };
+ }
+ const range = findQuoteRange(section, locatorQuotes(request, locators));
+ if (range) commitMarks([range], []);
+ reveal(box, range || section);
+ return { found: true, precise: !!range };
+ }
+ return locateQuote(box, box, request.quotes);
+}
+
+function locateText(request: SourceLocateRequest): SourceLocateResult {
+ const box = previewContent.value;
+ const code = box?.querySelector('code') || box;
+ if (!box || !code) return NOT_FOUND;
+ // Offsets index the original file; pretty-printed JSON no longer matches it.
+ const locators = request.locators.filter((l) => l.type === 'text' && (l.end || 0) > (l.start || 0));
+ if (locators.length && previewType.value === 'text' && !shouldPrettyPrintJson(previewExt)) {
+ const spans = locators.map((l): [number, number] => [l.start || 0, l.end || 0]);
+ // A chunk spans several paragraphs; keep the lines the sentence is about.
+ const cited = narrowTextLines(code.textContent || '', spans, request.sentence || '');
+ const ranges = (cited.length ? cited : spans)
+ .map(([start, end]) => rangeForOffsets(code, start, end))
+ .filter((r): r is Range => !!r && !r.collapsed);
+ if (ranges.length) {
+ commitMarks(ranges, []);
+ reveal(box, ranges[0]);
+ return { found: true, precise: true };
+ }
+ }
+ return locateQuote(code, box, locatorQuotes(request));
+}
+
+const MARKDOWN_BLOCKS = 'p, li, tr, h1, h2, h3, h4, h5, h6, pre, blockquote, dt, dd';
+
+/**
+ * Rendered Markdown has no offsets back into the file, so the cited chunk's
+ * text bounds the candidate blocks and the sentence picks among them.
+ */
+function locateMarkdown(request: SourceLocateRequest): SourceLocateResult {
+ const box = previewContent.value;
+ if (!box) return NOT_FOUND;
+ const scope = normalizeForMatch(request.scope || '');
+ if (scope && request.sentence) {
+ const blocks = Array.from(box.querySelectorAll(MARKDOWN_BLOCKS)).filter((el) => {
+ if (el.querySelector(MARKDOWN_BLOCKS)) return false;
+ const text = normalizeForMatch(el.textContent || '');
+ return text.length >= 4 && scope.includes(text);
+ });
+ const cited = pickBestTexts(blocks.map((el) => el.textContent || ''), request.sentence);
+ if (cited.length) {
+ const ranges = cited.map((i) => contentsRange(blocks[i]));
+ commitMarks(ranges, []);
+ reveal(box, ranges[0]);
+ return { found: true, precise: true };
+ }
+ }
+ return locateQuote(box, box, locatorQuotes(request));
+}
+
+function locateImage(request: SourceLocateRequest): SourceLocateResult {
+ const boxes = request.locators
+ .filter((l) => l.type === 'pdf' && (l.page || 1) === 1 && l.bbox?.length === 4)
+ .map((l) => {
+ const [x0, y0, x1, y1] = l.bbox as number[];
+ return { left: x0, top: y0, width: x1 - x0, height: y1 - y0 };
+ });
+ imageMarks.value = boxes;
+ // Without regions the whole image is the cited source.
+ return { found: true, precise: boxes.length > 0 };
+}
+
+function formatClock(ms: number): string {
+ const total = Math.max(0, Math.floor(ms / 1000));
+ const m = Math.floor(total / 60);
+ const s = total % 60;
+ return `${String(m).padStart(2, '0')}:${String(s).padStart(2, '0')}`;
+}
+
+function locateAudio(request: SourceLocateRequest): SourceLocateResult {
+ const audio = previewContent.value as HTMLAudioElement | null;
+ const loc = request.locators.find((l) => l.type === 'time' && (l.end_ms || 0) > (l.start_ms || 0));
+ if (!audio || !loc) return NOT_FOUND;
+ // Seek only; playback stays the reader's choice.
+ const seek = () => {
+ audio.currentTime = (loc.start_ms || 0) / 1000;
+ };
+ if (audio.readyState >= 1) {
+ seek();
+ } else {
+ // A newer locate clears this, so an older citation cannot win the seek.
+ audio.addEventListener('loadedmetadata', seek, { once: true });
+ clearLocate.push(() => audio.removeEventListener('loadedmetadata', seek));
+ }
+ audioRange.value = `${formatClock(loc.start_ms || 0)} – ${formatClock(loc.end_ms || 0)}`;
+ audioQuote.value = loc.quote || '';
+ return { found: true, precise: true };
+}
+
+function applyLocate() {
+ const request = props.locate;
+ clearLocateMarks();
+ // The pdf.js viewer locates on its own and reports through @located.
+ if (!request || previewType.value === 'pdf') return;
+ let result = NOT_FOUND;
+ try {
+ switch (previewType.value) {
+ case 'docx': result = locateDocx(request); break;
+ case 'pptx': result = locatePptx(request); break;
+ case 'excel': result = locateExcel(request); break;
+ case 'epub': result = locateEpub(request); break;
+ case 'text':
+ case 'html': result = locateText(request); break;
+ case 'markdown': result = locateMarkdown(request); break;
+ case 'image': result = locateImage(request); break;
+ case 'audio': result = locateAudio(request); break;
+ }
+ } catch (err) {
+ console.warn('Locating the citation failed:', err);
+ }
+ emit('located', result);
+}
+
watch(
() => [props.active, props.knowledgeId, props.sessionId, props.attachmentId, props.messageId, props.artifactIndex, props.sourceBlob, props.fileName, props.fileType],
([active]) => {
@@ -538,6 +939,9 @@ onUnmounted(() => {
</div>
<!-- PDF -->
+ <div v-else-if="previewType === 'pdf' && pdfData" class="preview-pdf preview-pdf--source">
+ <PdfSourceViewer :data="pdfData" :file-name="fileName" :locate="locate" @located="(r) => emit('located', r)" />
+ </div>
<div v-else-if="previewType === 'pdf' && blobUrl" class="preview-pdf">
<iframe ref="previewContent" tabindex="0" :title="fileName" :src="blobUrl" class="pdf-iframe" @load="onPreviewFrameLoad" />
</div>
@@ -561,7 +965,16 @@ onUnmounted(() => {
<!-- Image -->
<div v-else-if="previewType === 'image' && blobUrl" ref="previewContent" tabindex="0" :aria-label="fileName" class="preview-image">
<div class="image-wrapper">
- <img :src="blobUrl" :alt="fileName" @load="onImageLoad" />
+ <div class="image-frame">
+ <img :src="blobUrl" :alt="fileName" @load="onImageLoad" />
+ <div
+ v-for="(mark, i) in imageMarks"
+ :key="i"
+ class="image-locate-mark"
+ aria-hidden="true"
+ :style="{ left: `${mark.left * 100}%`, top: `${mark.top * 100}%`, width: `${mark.width * 100}%`, height: `${mark.height * 100}%` }"
+ />
+ </div>
<div v-if="imageNaturalWidth" class="image-info">
{{ imageNaturalWidth }} × {{ imageNaturalHeight }} px
</div>
@@ -583,6 +996,11 @@ onUnmounted(() => {
<div ref="previewContent" tabindex="0" :aria-label="fileName" class="excel-container" v-html="excelHtml" />
</div>
+ <!-- EPUB -->
+ <div v-else-if="previewType === 'epub' && epubHtml" ref="previewContent" tabindex="0" :aria-label="fileName" class="preview-markdown preview-epub">
+ <div class="markdown-body" v-html="epubHtml" />
+ </div>
+
<!-- Markdown -->
<div v-else-if="previewType === 'markdown' && markdownHtml" ref="previewContent" tabindex="0" :aria-label="fileName" class="preview-markdown">
<div class="markdown-body" v-html="markdownHtml" />
@@ -606,6 +1024,10 @@ onUnmounted(() => {
<audio ref="previewContent" controls :src="blobUrl" class="audio-element">
{{ $t('preview.audioNotSupported') }}
</audio>
+ <div v-if="audioRange" class="audio-locate">
+ <span class="audio-locate__time">{{ audioRange }}</span>
+ <p v-if="audioQuote" class="audio-locate__quote">{{ audioQuote }}</p>
+ </div>
</div>
</div>
@@ -913,7 +1335,29 @@ onUnmounted(() => {
}
}
+// ── PDF (pdf.js, source mode) ──
+.preview-pdf--source {
+ height: @preview-max-h;
+ border: 1px solid @border-color;
+ border-radius: @border-radius;
+ overflow: hidden;
+}
+
// ── Image ──
+.image-frame {
+ position: relative;
+ display: inline-block;
+ line-height: 0;
+}
+
+.image-locate-mark {
+ position: absolute;
+ pointer-events: none;
+ border-radius: var(--app-radius-xs);
+ background: color-mix(in srgb, var(--td-success-color) 22%, transparent);
+ outline: 1px solid color-mix(in srgb, var(--td-success-color) 50%, transparent);
+}
+
.preview-image {
overflow: auto;
display: flex;
@@ -998,6 +1442,24 @@ onUnmounted(() => {
.audio-filename { font-size: var(--app-text-base); color: @text-primary; margin: 0; }
.audio-element { width: 100%; max-width: 480px; }
}
+ .audio-locate {
+ width: 100%;
+ max-width: 480px;
+ padding: 10px 12px;
+ border-radius: @border-radius;
+ background: color-mix(in srgb, var(--td-success-color) 10%, transparent);
+ color: @text-primary;
+ &__time {
+ font-size: var(--app-text-sm);
+ font-variant-numeric: tabular-nums;
+ color: @text-secondary;
+ }
+ &__quote {
+ margin: 4px 0 0;
+ font-size: var(--app-text-md);
+ line-height: 1.6;
+ }
+ }
}
// ── Video ──
@@ -1245,3 +1707,27 @@ html[theme-mode="dark"] {
}
}
</style>
+
+<style lang="less">
+/* Citation highlights live outside the scoped block: they apply to DOM that
+ third-party renderers (docx-preview, pptx-preview, SheetJS) create. */
+::highlight(source-locate) {
+ background-color: color-mix(in srgb, var(--app-source-highlight) 60%, transparent);
+}
+
+mark.source-locate-mark {
+ background-color: color-mix(in srgb, var(--app-source-highlight) 60%, transparent);
+ color: inherit;
+}
+
+.source-locate-block {
+ background-color: color-mix(in srgb, var(--app-source-highlight) 22%, transparent) !important;
+ outline: 1px solid var(--app-source-highlight);
+ outline-offset: 2px;
+ border-radius: var(--app-radius-xs);
+}
+
+tr.source-locate-block > td {
+ background-color: color-mix(in srgb, var(--app-source-highlight) 45%, transparent) !important;
+}
+</style>
diff --git a/frontend/src/components/source-preview/PdfSourceViewer.vue b/frontend/src/components/source-preview/PdfSourceViewer.vue
new file mode 100644
index 000000000..030125258
--- /dev/null
+++ b/frontend/src/components/source-preview/PdfSourceViewer.vue
@@ -0,0 +1,664 @@
+<script setup lang="ts">
+import { computed, nextTick, onBeforeUnmount, onMounted, reactive, ref, shallowRef, watch } from 'vue'
+import { useI18n } from 'vue-i18n'
+import type { SourceLocateRequest, SourceLocator } from '@/utils/sourceLocator'
+import { findInText } from '@/utils/sourceLocator'
+import { findTextRange } from '@/utils/sourceLocatorDom'
+import { loadPdfJs, type PdfJs } from './pdfjsLoader'
+
+type PdfDocument = Awaited<ReturnType<PdfJs['getDocument']>['promise']>
+type PdfPage = Awaited<ReturnType<PdfDocument['getPage']>>
+
+/** A highlight drawn on a page, in fractions of the page box. */
+type Mark = { left: number; top: number; width: number; height: number; kind: 'box' | 'text' | 'page' }
+
+export type PdfLocateResult = { found: boolean; precise: boolean; page?: number }
+
+const props = defineProps<{
+ data: ArrayBuffer | null
+ fileName?: string
+ locate?: SourceLocateRequest | null
+}>()
+
+const emit = defineEmits<{
+ located: [result: PdfLocateResult]
+ error: [message: string]
+}>()
+
+const { t } = useI18n()
+
+const scroller = ref<HTMLElement | null>(null)
+const pageCount = ref(0)
+const currentPage = ref(1)
+const zoom = ref(1)
+const loading = ref(false)
+const error = ref('')
+/** Height / width of each page; unknown pages borrow the first page's. */
+const ratios = reactive<Record<number, number>>({})
+const marks = reactive<Record<number, Mark[]>>({})
+const pdf = shallowRef<PdfDocument | null>(null)
+
+const pageElements = new Map<number, HTMLElement>()
+type RenderEntry = {
+ scale: number
+ task?: { cancel: () => void }
+ textLayer?: { cancel: () => void }
+ /** Settles once the canvas and text layer are in place (or abandoned). */
+ done?: Promise<void>
+}
+const rendered = new Map<number, RenderEntry>()
+const renderOrder: number[] = []
+const pageTexts = new Map<number, string>()
+let pdfjs: PdfJs | null = null
+let loadingTask: { destroy: () => Promise<void> } | null = null
+let observer: IntersectionObserver | null = null
+let resizeObserver: ResizeObserver | null = null
+let containerWidth = 0
+let disposed = false
+let locateVersion = 0
+
+const MAX_RENDERED_PAGES = 12
+const PAGE_GAP = 12
+
+function defaultRatio(): number {
+ return ratios[1] || 1.414
+}
+
+function pageRatio(page: number): number {
+ return ratios[page] || defaultRatio()
+}
+
+function pageStyle(page: number) {
+ return { aspectRatio: `1 / ${pageRatio(page)}`, width: `${zoom.value * 100}%` }
+}
+
+function setPageRef(page: number, el: unknown) {
+ const node = el as HTMLElement | null
+ const previous = pageElements.get(page)
+ if (previous && previous !== node) observer?.unobserve(previous)
+ if (node) {
+ pageElements.set(page, node)
+ observer?.observe(node)
+ } else {
+ pageElements.delete(page)
+ }
+}
+
+async function openDocument(data: ArrayBuffer) {
+ const lib = await loadPdfJs()
+ pdfjs = lib
+ if (disposed) return
+ // pdf.js takes ownership of the buffer; keep the caller's copy intact.
+ const task = lib.getDocument({ data: new Uint8Array(data.slice(0)) })
+ loadingTask = task
+ const doc = await task.promise
+ if (disposed) {
+ await task.destroy()
+ return
+ }
+ pdf.value = doc
+ pageCount.value = doc.numPages
+ const first = await doc.getPage(1)
+ const vp = first.getViewport({ scale: 1 })
+ ratios[1] = vp.height / vp.width
+}
+
+function renderScale(page: PdfPage): number {
+ const vp = page.getViewport({ scale: 1 })
+ const width = Math.max(200, containerWidth * zoom.value)
+ return width / vp.width
+}
+
+async function renderPage(pageNumber: number): Promise<void> {
+ const doc = pdf.value
+ const el = pageElements.get(pageNumber)
+ if (!doc || !el || !pdfjs) return
+ const page = await doc.getPage(pageNumber)
+ const scale = renderScale(page)
+ const existing = rendered.get(pageNumber)
+ if (existing && Math.abs(existing.scale - scale) < 0.01) {
+ // Wait for a render still in flight, so callers find its text layer.
+ await existing.done
+ return
+ }
+ existing?.task?.cancel()
+ existing?.textLayer?.cancel()
+
+ const viewport = page.getViewport({ scale })
+ ratios[pageNumber] = viewport.height / viewport.width
+
+ const dpr = Math.min(window.devicePixelRatio || 1, 2)
+ const canvas = document.createElement('canvas')
+ canvas.width = Math.floor(viewport.width * dpr)
+ canvas.height = Math.floor(viewport.height * dpr)
+ canvas.className = 'pdf-source-page__canvas'
+
+ const entry: RenderEntry = { scale }
+ rendered.set(pageNumber, entry)
+ touchRendered(pageNumber)
+ entry.done = paintPage(pageNumber, page, el, canvas, viewport, dpr, entry)
+ await entry.done
+}
+
+async function paintPage(
+ pageNumber: number,
+ page: PdfPage,
+ el: HTMLElement,
+ canvas: HTMLCanvasElement,
+ viewport: ReturnType<PdfPage['getViewport']>,
+ dpr: number,
+ entry: RenderEntry,
+): Promise<void> {
+ const lib = pdfjs
+ if (!lib) return
+ const task = page.render({
+ canvas,
+ viewport,
+ transform: dpr !== 1 ? [dpr, 0, 0, dpr, 0, 0] : undefined,
+ })
+ entry.task = task
+ try {
+ await task.promise
+ } catch {
+ return // cancelled by a newer render or unrender
+ }
+ if (rendered.get(pageNumber) !== entry) return
+
+ const textDiv = document.createElement('div')
+ textDiv.className = 'textLayer'
+ el.style.setProperty('--scale-factor', String(entry.scale))
+ el.style.setProperty('--total-scale-factor', String(entry.scale))
+ const oldCanvas = el.querySelector('.pdf-source-page__canvas')
+ const oldText = el.querySelector('.textLayer')
+ oldCanvas?.replaceWith(canvas)
+ if (!oldCanvas) el.prepend(canvas)
+ oldText?.remove()
+ canvas.after(textDiv)
+
+ const textLayer = new lib.TextLayer({
+ textContentSource: page.streamTextContent(),
+ container: textDiv,
+ viewport,
+ })
+ entry.textLayer = textLayer
+ try {
+ await textLayer.render()
+ } catch {
+ // Text layers are best-effort; the page is still readable.
+ }
+}
+
+function touchRendered(pageNumber: number) {
+ const i = renderOrder.indexOf(pageNumber)
+ if (i >= 0) renderOrder.splice(i, 1)
+ renderOrder.push(pageNumber)
+ while (renderOrder.length > MAX_RENDERED_PAGES) {
+ const evict = renderOrder.shift()!
+ unrenderPage(evict)
+ }
+}
+
+function unrenderPage(pageNumber: number) {
+ const entry = rendered.get(pageNumber)
+ entry?.task?.cancel()
+ entry?.textLayer?.cancel()
+ rendered.delete(pageNumber)
+ const el = pageElements.get(pageNumber)
+ el?.querySelector('.pdf-source-page__canvas')?.remove()
+ el?.querySelector('.textLayer')?.remove()
+}
+
+function onIntersect(entries: IntersectionObserverEntry[]) {
+ for (const entry of entries) {
+ if (!entry.isIntersecting) continue
+ const page = Number((entry.target as HTMLElement).dataset.page)
+ if (page) void renderPage(page)
+ }
+}
+
+function onScroll() {
+ const box = scroller.value
+ if (!box) return
+ const mid = box.scrollTop + box.clientHeight / 3
+ for (const [page, el] of pageElements) {
+ if (el.offsetTop <= mid && el.offsetTop + el.offsetHeight + PAGE_GAP > mid) {
+ currentPage.value = page
+ break
+ }
+ }
+}
+
+function rerenderVisible() {
+ const pages = [...rendered.keys()]
+ for (const p of pages) void renderPage(p)
+}
+
+function setZoom(next: number) {
+ const box = scroller.value
+ const anchor = box ? box.scrollTop / Math.max(1, box.scrollHeight) : 0
+ zoom.value = Math.max(0.5, Math.min(3, Math.round(next * 10) / 10))
+ void nextTick(() => {
+ if (box) box.scrollTop = anchor * box.scrollHeight
+ rerenderVisible()
+ })
+}
+
+// ---------------------------------------------------------------------------
+// Locating
+
+function clearMarks() {
+ for (const key of Object.keys(marks)) delete marks[Number(key)]
+}
+
+function addMark(page: number, mark: Mark) {
+ ;(marks[page] ||= []).push(mark)
+}
+
+function scrollToPageFraction(page: number, fraction: number) {
+ const box = scroller.value
+ const el = pageElements.get(page)
+ if (!box || !el) return
+ // Instant, like scrollRectIntoContainer: nothing can cancel it mid-way.
+ box.scrollTop = Math.max(0, el.offsetTop + el.offsetHeight * fraction - box.clientHeight / 3)
+}
+
+async function pageText(pageNumber: number): Promise<string> {
+ const cached = pageTexts.get(pageNumber)
+ if (cached !== undefined) return cached
+ const doc = pdf.value
+ if (!doc) return ''
+ const page = await doc.getPage(pageNumber)
+ const content = await page.getTextContent()
+ const text = content.items.map((item) => ('str' in item ? item.str + (item.hasEOL ? '\n' : ' ') : '')).join('')
+ pageTexts.set(pageNumber, text)
+ return text
+}
+
+/** Draw the match for one of `quotes` on `pageNumber`; returns the top fraction or null. */
+async function markQuoteOnPage(pageNumber: number, quotes: string[], version: number): Promise<number | null> {
+ const text = await pageText(pageNumber)
+ const quote = quotes.find((q) => findInText(text, q))
+ if (!quote || version !== locateVersion) return null
+ await renderPage(pageNumber)
+ // A concurrent render may have replaced the one awaited; wait for the live one.
+ await rendered.get(pageNumber)?.done
+ if (version !== locateVersion) return null
+ const el = pageElements.get(pageNumber)
+ const layer = el?.querySelector('.textLayer')
+ if (!el || !layer) return null
+ const range = findTextRange(layer, quote)
+ if (!range) return null
+ const pageBox = el.getBoundingClientRect()
+ const lines = mergeLineRects([...range.getClientRects()])
+ let top: number | null = null
+ for (const r of lines) {
+ const mark: Mark = {
+ kind: 'text',
+ left: (r.left - pageBox.left) / pageBox.width,
+ top: (r.top - pageBox.top) / pageBox.height,
+ width: r.width / pageBox.width,
+ height: r.height / pageBox.height,
+ }
+ if (mark.width <= 0 || mark.height <= 0) continue
+ addMark(pageNumber, mark)
+ top = top === null ? mark.top : Math.min(top, mark.top)
+ }
+ return top
+}
+
+/** Merge the per-glyph-run rects of a range into one rect per visual line. */
+function mergeLineRects(rects: DOMRect[]): Array<{ left: number; top: number; width: number; height: number }> {
+ const lines: Array<{ left: number; top: number; right: number; bottom: number }> = []
+ for (const r of rects) {
+ if (r.width < 0.5 || r.height < 0.5) continue
+ const line = lines.find((l) => Math.min(l.bottom, r.bottom) - Math.max(l.top, r.top) > Math.min(r.height, l.bottom - l.top) * 0.5)
+ if (line) {
+ line.left = Math.min(line.left, r.left)
+ line.right = Math.max(line.right, r.right)
+ line.top = Math.min(line.top, r.top)
+ line.bottom = Math.max(line.bottom, r.bottom)
+ } else {
+ lines.push({ left: r.left, top: r.top, right: r.right, bottom: r.bottom })
+ }
+ }
+ return lines.map((l) => ({ left: l.left - 2, top: l.top - 1, width: l.right - l.left + 4, height: l.bottom - l.top + 2 }))
+}
+
+async function applyLocate(request: SourceLocateRequest | null | undefined) {
+ const version = ++locateVersion
+ clearMarks()
+ if (!request || !pdf.value) return
+ const total = pageCount.value
+ const pdfLocators = request.locators.filter((l): l is SourceLocator & { page: number } =>
+ l.type === 'pdf' && !!l.page && l.page >= 1 && l.page <= total)
+
+ let firstPage = 0
+ let firstTop = 0
+ let precise = false
+ const note = (page: number, top: number) => {
+ if (!firstPage) {
+ firstPage = page
+ firstTop = top
+ }
+ }
+
+ // Regions reported by the parser are drawn as they are.
+ for (const loc of pdfLocators) {
+ if (loc.bbox && loc.bbox.length === 4) {
+ const [x0, y0, x1, y1] = loc.bbox
+ addMark(loc.page, { kind: 'box', left: x0, top: y0, width: x1 - x0, height: y1 - y0 })
+ note(loc.page, y0)
+ precise = true
+ }
+ }
+
+ // Pages without a region: find the quoted text on the page.
+ if (!precise) {
+ const pages = [...new Set(pdfLocators.map((l) => l.page))]
+ for (const page of pages) {
+ const quotes = [
+ ...pdfLocators.filter((l) => l.page === page && l.quote).map((l) => l.quote as string),
+ ...request.quotes,
+ ]
+ const top = quotes.length ? await markQuoteOnPage(page, quotes, version) : null
+ if (version !== locateVersion) return
+ if (top !== null) {
+ note(page, top)
+ precise = true
+ } else {
+ // Scanned pages have no text: outline the page itself.
+ addMark(page, { kind: 'page', left: 0, top: 0, width: 1, height: 1 })
+ note(page, 0)
+ }
+ }
+ }
+
+ // No page at all: search the document for the quote.
+ if (!firstPage && request.quotes.length) {
+ const limit = Math.min(total, 500)
+ for (let page = 1; page <= limit; page++) {
+ const text = await pageText(page)
+ if (version !== locateVersion) return
+ if (request.quotes.some((q) => findInText(text, q))) {
+ const top = await markQuoteOnPage(page, request.quotes, version)
+ if (version !== locateVersion) return
+ if (top !== null) {
+ note(page, top)
+ precise = true
+ }
+ break
+ }
+ }
+ }
+
+ await nextTick()
+ if (version !== locateVersion) return
+ if (firstPage) {
+ scrollToPageFraction(firstPage, firstTop)
+ currentPage.value = firstPage
+ }
+ emit('located', { found: !!firstPage, precise, page: firstPage || undefined })
+}
+
+watch(
+ () => props.locate?.token,
+ () => {
+ if (pdf.value) void applyLocate(props.locate)
+ },
+)
+
+async function load() {
+ error.value = ''
+ if (!props.data) return
+ loading.value = true
+ try {
+ await openDocument(props.data)
+ await nextTick()
+ containerWidth = scroller.value?.clientWidth ? scroller.value.clientWidth - 24 : 600
+ if (props.locate) await applyLocate(props.locate)
+ } catch (err) {
+ console.error('PDF preview failed:', err)
+ error.value = t('preview.loadFailed')
+ emit('error', error.value)
+ } finally {
+ loading.value = false
+ }
+}
+
+onMounted(() => {
+ observer = new IntersectionObserver(onIntersect, { root: scroller.value, rootMargin: '600px 0px' })
+ for (const el of pageElements.values()) observer.observe(el)
+ resizeObserver = new ResizeObserver(() => {
+ const width = scroller.value?.clientWidth ? scroller.value.clientWidth - 24 : 0
+ if (width && Math.abs(width - containerWidth) > 8) {
+ containerWidth = width
+ rerenderVisible()
+ }
+ })
+ if (scroller.value) resizeObserver.observe(scroller.value)
+ void load()
+})
+
+onBeforeUnmount(() => {
+ disposed = true
+ locateVersion++
+ observer?.disconnect()
+ resizeObserver?.disconnect()
+ for (const p of [...rendered.keys()]) unrenderPage(p)
+ // Destroying the loading task also destroys the document and its worker.
+ void loadingTask?.destroy()
+})
+
+const pageLabel = computed(() => (pageCount.value ? `${currentPage.value} / ${pageCount.value}` : ''))
+
+defineExpose({ relocate: () => applyLocate(props.locate) })
+</script>
+
+<template>
+ <div class="pdf-source-viewer">
+ <div v-if="error" class="pdf-source-viewer__error">{{ error }}</div>
+ <div
+ ref="scroller"
+ class="pdf-source-viewer__scroller"
+ tabindex="0"
+ :aria-label="fileName"
+ @scroll.passive="onScroll"
+ >
+ <div v-if="loading && !pageCount" class="pdf-source-viewer__loading">
+ <t-loading size="small" />
+ </div>
+ <div
+ v-for="page in pageCount"
+ :key="page"
+ :ref="(el) => setPageRef(page, el)"
+ class="pdf-source-page"
+ :data-page="page"
+ :style="pageStyle(page)"
+ >
+ <div class="pdf-source-page__marks" aria-hidden="true">
+ <div
+ v-for="(mark, i) in marks[page] || []"
+ :key="i"
+ class="pdf-source-mark"
+ :class="`pdf-source-mark--${mark.kind}`"
+ :style="{
+ left: `${mark.left * 100}%`,
+ top: `${mark.top * 100}%`,
+ width: `${mark.width * 100}%`,
+ height: `${mark.height * 100}%`,
+ }"
+ />
+ </div>
+ </div>
+ </div>
+ <div v-if="pageCount" class="pdf-source-viewer__bar">
+ <span class="pdf-source-viewer__page">{{ pageLabel }}</span>
+ <t-button
+ theme="default" variant="text" size="small" shape="square"
+ :aria-label="t('preview.zoomOut')" :title="t('preview.zoomOut')"
+ @click="setZoom(zoom - 0.2)"
+ >
+ <template #icon><t-icon name="zoom-out" /></template>
+ </t-button>
+ <span class="pdf-source-viewer__zoom">{{ Math.round(zoom * 100) }}%</span>
+ <t-button
+ theme="default" variant="text" size="small" shape="square"
+ :aria-label="t('preview.zoomIn')" :title="t('preview.zoomIn')"
+ @click="setZoom(zoom + 0.2)"
+ >
+ <template #icon><t-icon name="zoom-in" /></template>
+ </t-button>
+ </div>
+ </div>
+</template>
+
+<style scoped lang="less">
+.pdf-source-viewer {
+ position: relative;
+ display: flex;
+ flex-direction: column;
+ height: 100%;
+ min-height: 0;
+ background: var(--td-bg-color-secondarycontainer);
+}
+
+.pdf-source-viewer__scroller {
+ // Positioned so page offsetTop is measured from the scroll content.
+ position: relative;
+ flex: 1;
+ min-height: 0;
+ overflow: auto;
+ padding: 12px;
+ outline: none;
+}
+
+.pdf-source-viewer__loading,
+.pdf-source-viewer__error {
+ padding: 24px;
+ text-align: center;
+ color: var(--td-text-color-placeholder);
+}
+
+.pdf-source-page {
+ position: relative;
+ // Block flow with auto margins keeps zoomed pages scrollable to the left.
+ margin: 0 auto 12px;
+ max-width: none;
+ background: #fff;
+ box-shadow: 0 1px 3px rgba(0, 0, 0, 0.12);
+ // Text layers size themselves in these units.
+ --scale-round-x: 1px;
+ --scale-round-y: 1px;
+
+ :deep(.pdf-source-page__canvas) {
+ position: absolute;
+ inset: 0;
+ width: 100%;
+ height: 100%;
+ }
+}
+
+.pdf-source-page__marks {
+ position: absolute;
+ inset: 0;
+ z-index: 2;
+ pointer-events: none;
+}
+
+.pdf-source-mark {
+ position: absolute;
+ border-radius: var(--app-radius-xs);
+ background: color-mix(in srgb, var(--app-source-highlight) 55%, transparent);
+ mix-blend-mode: multiply;
+
+ &--box {
+ outline: 1px solid var(--app-source-highlight);
+ }
+
+ &--page {
+ background: transparent;
+ outline: 2px solid var(--app-source-highlight);
+ mix-blend-mode: normal;
+ }
+}
+
+.pdf-source-viewer__bar {
+ position: absolute;
+ right: 16px;
+ bottom: 12px;
+ z-index: 3;
+ display: flex;
+ align-items: center;
+ gap: 2px;
+ padding: 2px 6px;
+ border-radius: var(--app-radius-md);
+ background: var(--td-bg-color-container);
+ box-shadow: 0 2px 8px rgba(0, 0, 0, 0.12);
+ font-size: var(--app-text-sm);
+ color: var(--td-text-color-secondary);
+}
+
+.pdf-source-viewer__page {
+ padding: 0 6px;
+ font-variant-numeric: tabular-nums;
+}
+
+.pdf-source-viewer__zoom {
+ min-width: 40px;
+ text-align: center;
+ font-variant-numeric: tabular-nums;
+}
+</style>
+
+<style lang="less">
+/* Minimal pdf.js text layer rules (from pdfjs-dist/web/pdf_viewer.css), so
+ text is selectable and searchable without the full viewer stylesheet. */
+.pdf-source-page .textLayer {
+ position: absolute;
+ text-align: initial;
+ inset: 0;
+ overflow: clip;
+ opacity: 1;
+ line-height: 1;
+ letter-spacing: normal;
+ word-spacing: normal;
+ text-size-adjust: none;
+ forced-color-adjust: none;
+ transform-origin: 0 0;
+ z-index: 1;
+ --min-font-size: 1;
+ --text-scale-factor: calc(var(--total-scale-factor) * var(--min-font-size));
+ --min-font-size-inv: calc(1 / var(--min-font-size));
+
+ :is(span, br) {
+ color: transparent;
+ position: absolute;
+ white-space: pre;
+ cursor: text;
+ transform-origin: 0% 0%;
+ }
+
+ > :not(.markedContent),
+ .markedContent span:not(.markedContent) {
+ z-index: 1;
+ --font-height: 0;
+ font-size: calc(var(--text-scale-factor) * var(--font-height));
+ --scale-x: 1;
+ --rotate: 0deg;
+ transform: rotate(var(--rotate)) scaleX(var(--scale-x)) scale(var(--min-font-size-inv));
+ }
+
+ .markedContent {
+ display: contents;
+ }
+
+ ::selection {
+ background: color-mix(in srgb, AccentColor, transparent 50%);
+ color: transparent;
+ }
+
+ &[data-main-rotation="90"] { transform: rotate(90deg) translateY(-100%); }
+ &[data-main-rotation="180"] { transform: rotate(180deg) translate(-100%, -100%); }
+ &[data-main-rotation="270"] { transform: rotate(270deg) translateX(-100%); }
+}
+</style>
diff --git a/frontend/src/components/source-preview/pdfjsLoader.ts b/frontend/src/components/source-preview/pdfjsLoader.ts
new file mode 100644
index 000000000..c83f100b3
--- /dev/null
+++ b/frontend/src/components/source-preview/pdfjsLoader.ts
@@ -0,0 +1,24 @@
+// Bundled as a worker chunk with a .js name, so any static server hands it
+// out with a JavaScript MIME type (module workers refuse anything else).
+import PdfWorker from 'pdfjs-dist/legacy/build/pdf.worker.min.mjs?worker'
+
+export type PdfJs = typeof import('pdfjs-dist/legacy/build/pdf.mjs')
+
+let loading: Promise<PdfJs> | null = null
+
+/**
+ * Load pdf.js once per page. Every document shares one worker: pdf.js keeps
+ * a caller-provided port alive when a document is destroyed.
+ */
+export function loadPdfJs(): Promise<PdfJs> {
+ if (!loading) {
+ loading = import('pdfjs-dist/legacy/build/pdf.mjs').then((lib) => {
+ lib.GlobalWorkerOptions.workerPort = new PdfWorker()
+ return lib
+ })
+ loading.catch(() => {
+ loading = null
+ })
+ }
+ return loading
+}
diff --git a/frontend/src/composables/useChatReferencesDrawer.ts b/frontend/src/composables/useChatReferencesDrawer.ts
index 72dab0779..3fcba700b 100644
--- a/frontend/src/composables/useChatReferencesDrawer.ts
+++ b/frontend/src/composables/useChatReferencesDrawer.ts
@@ -1,5 +1,10 @@
import { inject, provide, ref, type InjectionKey, type Ref } from 'vue'
-import type { KnowledgeReferenceLike, ReferenceHighlightTarget } from '@/utils/referenceSources'
+import {
+ resolveReferenceSource,
+ type KnowledgeReferenceLike,
+ type ReferenceHighlightTarget,
+ type ReferenceSourceTarget,
+} from '@/utils/referenceSources'
export type ChatReferencesDrawerOpenOptions = {
references: KnowledgeReferenceLike[]
@@ -8,16 +13,25 @@ export type ChatReferencesDrawerOpenOptions = {
sourceKey?: string
}
+/** Default width of the references panel, which the chat column makes room for. */
+export const REFERENCES_PANEL_WIDTH = 420
+
export type ChatReferencesDrawerContext = {
visible: Ref<boolean>
references: Ref<KnowledgeReferenceLike[]>
highlight: Ref<ReferenceHighlightTarget | null>
messageId: Ref<string>
sourceKey: Ref<string>
+ /** The cited chunk shown inside its original document, or null for the list. */
+ source: Ref<ReferenceSourceTarget | null>
+ /** Current panel width in px; the source view is wider than the list. */
+ panelWidth: Ref<number>
open: (options: ChatReferencesDrawerOpenOptions) => void
toggle: (options: ChatReferencesDrawerOpenOptions) => boolean
close: () => void
setHighlight: (highlight: ReferenceHighlightTarget | null) => void
+ openSource: (target: ReferenceSourceTarget) => void
+ closeSource: () => void
}
const CHAT_REFERENCES_DRAWER_KEY: InjectionKey<ChatReferencesDrawerContext> = Symbol(
@@ -30,6 +44,8 @@ export function provideChatReferencesDrawer(): ChatReferencesDrawerContext {
const highlight = ref<ReferenceHighlightTarget | null>(null)
const messageId = ref('')
const sourceKey = ref('')
+ const source = ref<ReferenceSourceTarget | null>(null)
+ const panelWidth = ref(REFERENCES_PANEL_WIDTH)
const getFallbackSourceKey = (options: ChatReferencesDrawerOpenOptions) => {
if (options.sourceKey) return options.sourceKey
@@ -40,11 +56,16 @@ export function provideChatReferencesDrawer(): ChatReferencesDrawerContext {
.join('|')
}
+ // Citation clicks ask for the original document; other openings show the list.
+ const sourceFor = (next: ReferenceHighlightTarget | null | undefined) =>
+ next?.openSource ? resolveReferenceSource(references.value, next) : null
+
const open = (options: ChatReferencesDrawerOpenOptions) => {
references.value = Array.isArray(options.references) ? options.references : []
highlight.value = options.highlight ?? null
messageId.value = options.messageId || ''
sourceKey.value = getFallbackSourceKey(options)
+ source.value = sourceFor(options.highlight)
visible.value = true
}
@@ -62,25 +83,40 @@ export function provideChatReferencesDrawer(): ChatReferencesDrawerContext {
visible.value = false
highlight.value = null
sourceKey.value = ''
+ source.value = null
}
const setHighlight = (next: ReferenceHighlightTarget | null) => {
highlight.value = next
+ if (next?.openSource) source.value = sourceFor(next)
if (next && references.value.length) {
visible.value = true
}
}
+ const openSource = (target: ReferenceSourceTarget) => {
+ source.value = target
+ visible.value = true
+ }
+
+ const closeSource = () => {
+ source.value = null
+ }
+
const ctx: ChatReferencesDrawerContext = {
visible,
references,
highlight,
messageId,
sourceKey,
+ source,
+ panelWidth,
open,
toggle,
close,
setHighlight,
+ openSource,
+ closeSource,
}
provide(CHAT_REFERENCES_DRAWER_KEY, ctx)
diff --git a/frontend/src/composables/useCitationPopover.test.ts b/frontend/src/composables/useCitationPopover.test.ts
index 09267cb5d..5c5d6c5f4 100644
--- a/frontend/src/composables/useCitationPopover.test.ts
+++ b/frontend/src/composables/useCitationPopover.test.ts
@@ -78,6 +78,7 @@ function fixture(mode: 'chat' | 'embed', options: { drawer?: boolean; refs?: cit
if (path === 'vue-i18n') return { useI18n: () => ({ t: (key: string) => key }) }
if (path === '@/utils/citationMarkdown') return citationMarkdown
if (path === '@/utils/citationChunkCache') return load('../utils/citationChunkCache')
+ if (path === '@/utils/citationAnchor') return { citationAnchorText: () => '' }
if (path === '@/composables/useChatReferencesDrawer') return { useChatReferencesDrawer: () => options.drawer ? { open: (args: any) => drawerCalls.push(args) } : null }
if (path === './useCitationPopover') return load('useCitationPopover')
if (path === '@/api/knowledge-base') return { getChunkByIdOnly: (...args: string[]) => request('chat', ...args) }
diff --git a/frontend/src/composables/useCitationPopover.ts b/frontend/src/composables/useCitationPopover.ts
index 1fedde098..3340c96c9 100644
--- a/frontend/src/composables/useCitationPopover.ts
+++ b/frontend/src/composables/useCitationPopover.ts
@@ -5,6 +5,7 @@ import {
setCitationChunkCache,
} from '@/utils/citationChunkCache'
import { useChatReferencesDrawer } from '@/composables/useChatReferencesDrawer'
+import { citationAnchorText } from '@/utils/citationAnchor'
export type CitationFloatState = {
visible: boolean
@@ -170,7 +171,7 @@ export function useCitationPopover(rootRef: Ref<HTMLElement | null>, options: Ci
scheduleClose()
}
- const openDrawerForCitation = (payload: { url?: string; chunkId?: string }) => {
+ const openDrawerForCitation = (payload: { url?: string; chunkId?: string; anchorText?: string; openSource?: boolean }) => {
const refs = options?.getKnowledgeReferences?.() || []
if (!referencesDrawer || !refs.length) return false
referencesDrawer.open({
@@ -197,7 +198,10 @@ export function useCitationPopover(rootRef: Ref<HTMLElement | null>, options: Ci
e.preventDefault()
e.stopPropagation()
const chunkId = resolveChunkId(kbEl)
- if (openDrawerForCitation({ chunkId })) return
+ // Chat opens the cited document at the cited passage; the embed has no
+ // access to original files and keeps the reference list.
+ const source = isChat ? { anchorText: citationAnchorText(kbEl), openSource: true } : {}
+ if (openDrawerForCitation({ chunkId, ...source })) return
void openKb(kbEl)
return
}
diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts
index ff9675d05..1c7e6dcf1 100755
--- a/frontend/src/i18n/locales/en-US.ts
+++ b/frontend/src/i18n/locales/en-US.ts
@@ -4071,6 +4071,13 @@ export default {
channelIm: 'IM',
chunkLabel: 'Chunk {index}:',
navigateToDocument: 'View document details',
+ referenceSourceBack: 'All sources',
+ referenceSourceView: 'View in original',
+ referenceSourceRelocate: 'Locate again',
+ referenceSourceLocating: 'Locating the cited passage…',
+ referenceSourceFoundPage: 'Found on page {page}',
+ referenceSourceNotFound: 'Could not pinpoint the cited passage; the original is open',
+ referenceSourceOpenWeb: 'Open the web page at this passage',
chunkIdLabel: 'Chunk ID:',
documentIdLabel: 'Document ID:',
faqIdLabel: 'FAQ ID:',
@@ -6004,6 +6011,8 @@ export default {
retry: 'Retry',
unsupported: 'This file type does not support online preview',
unsupportedHint: 'Please download and open with a local application',
+ zoomIn: 'Zoom in',
+ zoomOut: 'Zoom out',
fullscreen: 'Fullscreen',
exitFullscreen: 'Exit Fullscreen',
htmlRendered: 'Rendered preview',
diff --git a/frontend/src/i18n/locales/ja-JP.ts b/frontend/src/i18n/locales/ja-JP.ts
index df0c1562d..ac272ac64 100644
--- a/frontend/src/i18n/locales/ja-JP.ts
+++ b/frontend/src/i18n/locales/ja-JP.ts
@@ -4071,6 +4071,13 @@ export default {
channelIm: 'IM',
chunkLabel: 'チャンク{index}:',
navigateToDocument: 'ドキュメント詳細を表示',
+ referenceSourceBack: 'すべての出典',
+ referenceSourceView: '原文を表示',
+ referenceSourceRelocate: '再度位置を特定',
+ referenceSourceLocating: '引用箇所を特定しています…',
+ referenceSourceFoundPage: '{page} ページで見つかりました',
+ referenceSourceNotFound: '引用箇所を特定できなかったため、原文を開きました',
+ referenceSourceOpenWeb: '元の Web ページで該当箇所を開く',
chunkIdLabel: 'チャンクID:',
documentIdLabel: 'ドキュメントID:',
faqIdLabel: 'FAQ ID:',
@@ -6004,6 +6011,8 @@ export default {
retry: '再試行',
unsupported: 'このファイル形式はオンラインプレビューに対応していません',
unsupportedHint: 'ダウンロードして、ローカルのアプリケーションで開いてください',
+ zoomIn: '拡大',
+ zoomOut: '縮小',
fullscreen: '全画面表示',
exitFullscreen: '全画面表示を終了',
htmlRendered: 'レンダリング表示',
diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts
index 7d3a5f4ac..ead7f7139 100755
--- a/frontend/src/i18n/locales/ko-KR.ts
+++ b/frontend/src/i18n/locales/ko-KR.ts
@@ -2008,6 +2008,8 @@ export default {
retry: '재시도',
unsupported: '이 파일 유형은 온라인 미리보기를 지원하지 않습니다',
unsupportedHint: '파일을 다운로드하여 로컬 앱으로 열어주세요',
+ zoomIn: '확대',
+ zoomOut: '축소',
fullscreen: '전체 화면',
exitFullscreen: '전체 화면 종료',
htmlRendered: '렌더링 미리보기',
@@ -3948,6 +3950,13 @@ export default {
channelIm: 'IM',
chunkLabel: '청크 {index}:',
navigateToDocument: '문서 상세 보기',
+ referenceSourceBack: '전체 출처',
+ referenceSourceView: '원문 보기',
+ referenceSourceRelocate: '다시 찾기',
+ referenceSourceLocating: '인용 위치를 찾는 중…',
+ referenceSourceFoundPage: '{page}페이지에서 찾았습니다',
+ referenceSourceNotFound: '인용 위치를 정확히 찾지 못해 원문을 열었습니다',
+ referenceSourceOpenWeb: '원본 웹페이지에서 해당 위치 열기',
chunkIdLabel: '청크 ID:',
documentIdLabel: '문서 ID:',
faqIdLabel: 'FAQ ID:',
diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts
index 916776d35..4f075e0bd 100755
--- a/frontend/src/i18n/locales/ru-RU.ts
+++ b/frontend/src/i18n/locales/ru-RU.ts
@@ -2008,6 +2008,8 @@ export default {
retry: 'Повторить',
unsupported: 'Этот тип файла не поддерживает онлайн-просмотр',
unsupportedHint: 'Скачайте файл и откройте локально',
+ zoomIn: 'Увеличить',
+ zoomOut: 'Уменьшить',
fullscreen: 'Полноэкранный режим',
exitFullscreen: 'Выйти из полноэкранного режима',
htmlRendered: 'Отображение',
@@ -3948,6 +3950,13 @@ export default {
channelIm: 'IM',
chunkLabel: 'Фрагмент {index}:',
navigateToDocument: 'Просмотр документа',
+ referenceSourceBack: 'Все источники',
+ referenceSourceView: 'Открыть оригинал',
+ referenceSourceRelocate: 'Найти снова',
+ referenceSourceLocating: 'Поиск цитируемого фрагмента…',
+ referenceSourceFoundPage: 'Найдено на странице {page}',
+ referenceSourceNotFound: 'Не удалось точно найти фрагмент; открыт оригинал',
+ referenceSourceOpenWeb: 'Открыть веб-страницу на этом фрагменте',
chunkIdLabel: 'ID фрагмента:',
documentIdLabel: 'ID документа:',
faqIdLabel: 'FAQ ID:',
diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts
index 79fc6b967..073f86492 100755
--- a/frontend/src/i18n/locales/zh-CN.ts
+++ b/frontend/src/i18n/locales/zh-CN.ts
@@ -2010,6 +2010,8 @@ export default {
retry: '重试',
unsupported: '该文件类型暂不支持在线预览',
unsupportedHint: '请下载文件后使用本地应用查看',
+ zoomIn: '放大',
+ zoomOut: '缩小',
fullscreen: '全屏预览',
exitFullscreen: '退出全屏',
htmlRendered: '渲染预览',
@@ -3950,6 +3952,13 @@ export default {
channelIm: 'IM',
chunkLabel: '片段{index}:',
navigateToDocument: '查看文档详情',
+ referenceSourceBack: '全部引用',
+ referenceSourceView: '查看原文',
+ referenceSourceRelocate: '重新定位',
+ referenceSourceLocating: '正在定位引用位置…',
+ referenceSourceFoundPage: '已定位到第 {page} 页',
+ referenceSourceNotFound: '未能精确定位引用内容,已为你打开原文',
+ referenceSourceOpenWeb: '打开原网页并定位',
chunkIdLabel: '片段ID:',
documentIdLabel: '文档ID:',
faqIdLabel: 'FAQ ID:',
diff --git a/frontend/src/utils/citationAnchor.ts b/frontend/src/utils/citationAnchor.ts
new file mode 100644
index 000000000..ddd6e9ab3
--- /dev/null
+++ b/frontend/src/utils/citationAnchor.ts
@@ -0,0 +1,84 @@
+/**
+ * The answer sentence a citation marker supports: the text of the marker's
+ * block (paragraph, list item, table cell) up to the marker, without other
+ * citation markers, trimmed to its last sentence. A marker on a line of its
+ * own ("依据:[1]") borrows the blocks just before it.
+ */
+
+const BLOCK_SELECTOR = 'li, p, td, th, blockquote, h1, h2, h3, h4, h5, h6, dd, dt'
+const CITATION_SELECTOR = '.citation, .citation-kb, .citation-web, .citation-wiki'
+const MIN_SENTENCE = 8
+const MAX_LOOKBACK = 3
+// Lookback never leaves the rendered answer.
+const ANSWER_ROOT_SELECTOR = '.markdown-content, body, [contenteditable]'
+
+function letterCount(text: string): number {
+ return text.replace(/[\s\p{P}\p{S}]/gu, '').length
+}
+
+/** Last sentence of `text`, extended backwards while it is too short to align. */
+export function lastSentence(text: string): string {
+ const pieces = String(text || '')
+ .replace(/\s+/g, ' ')
+ .split(/(?<=[。!?!?;;])/u)
+ .map((s) => s.trim())
+ .filter(Boolean)
+ let out = ''
+ for (let i = pieces.length - 1; i >= 0; i--) {
+ out = pieces[i] + out
+ if (letterCount(out) >= MIN_SENTENCE) break
+ }
+ return out.slice(-300)
+}
+
+/**
+ * Anchor for a marker whose own text is `own`, prepending the texts of the
+ * blocks before it (nearest first) while the sentence is too short to align.
+ */
+export function anchorWithContext(own: string, previous: string[]): string {
+ let text = own
+ for (const prev of previous) {
+ if (letterCount(lastSentence(text)) >= MIN_SENTENCE) break
+ text = `${prev} ${text}`
+ }
+ return lastSentence(text)
+}
+
+function textWithoutCitations(node: Node): string {
+ const clone = node.cloneNode(true) as Element | DocumentFragment
+ clone.querySelectorAll?.(CITATION_SELECTOR).forEach((n) => n.remove())
+ return clone.textContent || ''
+}
+
+/**
+ * Text preceding `block` at each enclosing level, nearest first: earlier
+ * siblings, and for a nested list the text of the item that holds it.
+ */
+function previousBlockTexts(block: Element): string[] {
+ const out: string[] = []
+ for (let node: Element | null = block; node?.parentElement && out.length < MAX_LOOKBACK; node = node.parentElement) {
+ if (node.matches(ANSWER_ROOT_SELECTOR)) break
+ const range = node.ownerDocument.createRange()
+ range.setStart(node.parentElement, 0)
+ range.setEndBefore(node)
+ const text = textWithoutCitations(range.cloneContents()).trim()
+ if (text) out.push(text)
+ }
+ return out
+}
+
+export function citationAnchorText(marker: Element): string {
+ try {
+ const block = marker.closest(BLOCK_SELECTOR) || marker.parentElement
+ if (!block) return ''
+ const range = marker.ownerDocument.createRange()
+ range.setStart(block, 0)
+ range.setEndBefore(marker)
+ const own = textWithoutCitations(range.cloneContents())
+ const enough = letterCount(lastSentence(own)) >= MIN_SENTENCE
+ return anchorWithContext(own, enough ? [] : previousBlockTexts(block))
+ } catch {
+ // Alignment only narrows the highlight; without a sentence the whole chunk is shown.
+ return ''
+ }
+}
diff --git a/frontend/src/utils/epubPreview.ts b/frontend/src/utils/epubPreview.ts
new file mode 100644
index 000000000..c6f7f2b30
--- /dev/null
+++ b/frontend/src/utils/epubPreview.ts
@@ -0,0 +1,113 @@
+/**
+ * Minimal EPUB rendering for previews: every spine item becomes a
+ * `<section data-epub-section="N">` (N is the 1-based spine position, the same
+ * numbering the backend uses in section locators), with images served from
+ * blob URLs. The result must still be sanitized before it reaches the DOM.
+ */
+import JSZip from 'jszip'
+
+export type EpubPreview = {
+ html: string
+ /** Blob URLs created for images; revoke them when the preview goes away. */
+ objectUrls: string[]
+}
+
+const IMAGE_MIME: Record<string, string> = {
+ png: 'image/png',
+ jpg: 'image/jpeg',
+ jpeg: 'image/jpeg',
+ gif: 'image/gif',
+ webp: 'image/webp',
+ svg: 'image/svg+xml',
+ bmp: 'image/bmp',
+}
+
+/** Resolve `href` against the directory of `base` inside the archive. */
+export function resolveEpubPath(base: string, href: string): string {
+ const clean = decodeURIComponent(href.split('#')[0].split('?')[0])
+ if (clean.startsWith('/')) return clean.slice(1)
+ const parts = base.split('/').slice(0, -1)
+ for (const seg of clean.split('/')) {
+ if (seg === '..') parts.pop()
+ else if (seg && seg !== '.') parts.push(seg)
+ }
+ return parts.join('/')
+}
+
+function parseXml(text: string): Document {
+ return new DOMParser().parseFromString(text, 'application/xml')
+}
+
+function parseChapter(text: string): Document {
+ const xhtml = new DOMParser().parseFromString(text, 'application/xhtml+xml')
+ if (!xhtml.getElementsByTagName('parsererror').length && xhtml.body) return xhtml
+ return new DOMParser().parseFromString(text, 'text/html')
+}
+
+function byLocalName(root: Document | Element, name: string): Element[] {
+ return Array.from(root.getElementsByTagName('*')).filter((el) => el.localName === name)
+}
+
+export async function renderEpubPreview(data: ArrayBuffer): Promise<EpubPreview> {
+ const zip = await JSZip.loadAsync(data)
+ const containerXml = await zip.file('META-INF/container.xml')?.async('string')
+ if (!containerXml) throw new Error('EPUB: missing META-INF/container.xml')
+ const rootfile = byLocalName(parseXml(containerXml), 'rootfile')[0]?.getAttribute('full-path')
+ if (!rootfile) throw new Error('EPUB: no rootfile')
+ const opfText = await zip.file(rootfile)?.async('string')
+ if (!opfText) throw new Error('EPUB: missing package document')
+ const opf = parseXml(opfText)
+
+ const manifest = new Map<string, string>()
+ for (const item of byLocalName(opf, 'item')) {
+ const id = item.getAttribute('id')
+ const href = item.getAttribute('href')
+ if (id && href) manifest.set(id, resolveEpubPath(rootfile, href))
+ }
+ // Keep unresolved entries so section numbers match spine positions.
+ const spine = byLocalName(opf, 'itemref').map((ref) => manifest.get(ref.getAttribute('idref') || ''))
+
+ const objectUrls: string[] = []
+ const imageUrls = new Map<string, string>()
+ const imageUrl = async (path: string): Promise<string> => {
+ const cached = imageUrls.get(path)
+ if (cached) return cached
+ const file = zip.file(path)
+ if (!file) return ''
+ const ext = path.split('.').pop()?.toLowerCase() || ''
+ const blob = new Blob([await file.async('arraybuffer')], { type: IMAGE_MIME[ext] || 'application/octet-stream' })
+ const url = URL.createObjectURL(blob)
+ objectUrls.push(url)
+ imageUrls.set(path, url)
+ return url
+ }
+
+ const sections: string[] = []
+ for (let i = 0; i < spine.length; i++) {
+ const path = spine[i]
+ if (!path) continue
+ const text = await zip.file(path)?.async('string')
+ if (!text) continue
+ const doc = parseChapter(text)
+ const body = doc.body || doc.documentElement
+ for (const img of Array.from(body.querySelectorAll('img'))) {
+ const src = img.getAttribute('src')
+ img.setAttribute('src', src ? await imageUrl(resolveEpubPath(path, src)) : '')
+ }
+ for (const image of byLocalName(body as Element, 'image')) {
+ const href = image.getAttribute('href') || image.getAttribute('xlink:href')
+ if (href) {
+ const url = await imageUrl(resolveEpubPath(path, href))
+ image.setAttribute('href', url)
+ image.removeAttribute('xlink:href')
+ }
+ }
+ // Chapter links point inside the archive; keep the text, drop the target.
+ for (const a of Array.from(body.querySelectorAll('a[href]'))) {
+ const href = a.getAttribute('href') || ''
+ if (!/^https?:/i.test(href)) a.removeAttribute('href')
+ }
+ sections.push(`<section class="epub-section" data-epub-section="${i + 1}">${body.innerHTML}</section>`)
+ }
+ return { html: sections.join('\n'), objectUrls }
+}
diff --git a/frontend/src/utils/filePreview.ts b/frontend/src/utils/filePreview.ts
index b451c525f..065c833ff 100644
--- a/frontend/src/utils/filePreview.ts
+++ b/frontend/src/utils/filePreview.ts
@@ -20,6 +20,7 @@ export type FilePreviewKind =
| 'audio'
| 'video'
| 'mermaid'
+ | 'epub'
| 'unsupported'
export type FilePreviewSniff = {
@@ -52,6 +53,7 @@ register('html', ['html', 'htm', 'xhtml'])
register('audio', ['mp3', 'wav', 'm4a', 'flac', 'ogg', 'oga', 'aac', 'opus', 'weba'])
register('video', ['mp4', 'webm', 'ogv', 'm4v', 'mov'])
register('mermaid', ['mmd', 'mermaid'])
+register('epub', ['epub'])
register('text', [
'txt', 'text', 'log', 'out', 'err',
'json', 'jsonc', 'json5', 'geojson', 'ipynb', 'jsonl', 'ndjson', 'har',
@@ -83,6 +85,7 @@ const MIME_BY_EXT: Record<string, string> = {
docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
doc: 'application/msword',
pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+ epub: 'application/epub+zip',
ppt: 'application/vnd.ms-powerpoint',
xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
xls: 'application/vnd.ms-excel',
@@ -225,6 +228,7 @@ const DEFAULT_EXT_BY_KIND: Record<FilePreviewKind, string> = {
audio: 'mp3',
video: 'mp4',
mermaid: 'mmd',
+ epub: 'epub',
unsupported: '',
}
diff --git a/frontend/src/utils/referenceSources.test.mjs b/frontend/src/utils/referenceSources.test.mjs
index 43cb53d1d..fabe9ae57 100644
--- a/frontend/src/utils/referenceSources.test.mjs
+++ b/frontend/src/utils/referenceSources.test.mjs
@@ -54,6 +54,15 @@ test('buildReferenceList aggregates chunks from the same document', () => {
assert.match(items[0].content || '', /shipping rules/)
})
+test('buildReferenceList opens a document at its first chunk with positions', () => {
+ const [item] = buildReferenceList([
+ { id: 'summary', knowledge_id: 'k1', knowledge_title: 'Scan', chunk_type: 'summary', content: 'Summary' },
+ { id: 'ocr-1', knowledge_id: 'k1', knowledge_title: 'Scan', chunk_type: 'image_ocr', content: 'Page 1', source_locators: [{ type: 'pdf', page: 1 }] },
+ ])
+ assert.equal(item.chunkId, 'summary')
+ assert.equal(item.sourceChunkId, 'ocr-1')
+})
+
test('buildReferenceSections keeps tool results in their own section', () => {
const sections = buildReferenceSections([
{
diff --git a/frontend/src/utils/referenceSources.ts b/frontend/src/utils/referenceSources.ts
index 3b1e97620..4a164e92f 100644
--- a/frontend/src/utils/referenceSources.ts
+++ b/frontend/src/utils/referenceSources.ts
@@ -1,3 +1,5 @@
+import type { SourceLocator } from './sourceLocator'
+
export type ReferenceItemKind = 'web' | 'document' | 'tool'
export type KnowledgeReferenceLike = {
@@ -11,6 +13,12 @@ export type KnowledgeReferenceLike = {
chunk_type?: string
content?: string
metadata?: Record<string, string>
+ /** Where the knowledge came from: an upload, "manual", or the imported URL. */
+ knowledge_source?: string
+ /** Positions of the chunk in the original file. */
+ source_locators?: SourceLocator[]
+ /** After merging a document's chunks: the first one with positions. */
+ source_chunk_id?: string
}
export type ReferenceListItem = {
@@ -25,6 +33,8 @@ export type ReferenceListItem = {
snippet?: string
chunkId?: string
chunkIds?: string[]
+ /** The chunk to open the original file at (summaries have no position). */
+ sourceChunkId?: string
knowledgeId?: string
knowledgeBaseId?: string
content?: string
@@ -174,6 +184,7 @@ function buildDocumentItem(item: KnowledgeReferenceLike, index: number): Referen
title,
chunkId,
chunkIds: item.chunk_ids,
+ sourceChunkId: item.source_chunk_id,
knowledgeId: item.knowledge_id,
knowledgeBaseId: item.knowledge_base_id,
snippet: truncateText(item.content || '', 220) || undefined,
@@ -212,6 +223,7 @@ function mergeDocumentReferences(refs: KnowledgeReferenceLike[]): KnowledgeRefer
const key = getDocumentGroupKey(item, index)
const content = String(item.content || '').trim()
const chunkIds = Array.from(new Set([...(item.chunk_ids || []), ...(item.id ? [item.id] : [])]))
+ const located = item.id && item.source_locators?.length ? item.id : undefined
const existing = groups.get(key)
if (!existing) {
@@ -220,10 +232,12 @@ function mergeDocumentReferences(refs: KnowledgeReferenceLike[]): KnowledgeRefer
id: item.id || key,
content_parts: content ? [content] : [],
chunk_ids: chunkIds,
+ source_chunk_id: item.source_chunk_id || located,
})
return
}
+ if (!existing.source_chunk_id && located) existing.source_chunk_id = located
if (!existing.knowledge_id && item.knowledge_id) existing.knowledge_id = item.knowledge_id
if (!existing.knowledge_title && item.knowledge_title) existing.knowledge_title = item.knowledge_title
if (!existing.knowledge_filename && item.knowledge_filename) existing.knowledge_filename = item.knowledge_filename
@@ -321,6 +335,60 @@ export type ReferenceHighlightTarget = {
documentTitle?: string
knowledgeBaseId?: string
key?: string
+ /** The answer sentence the citation supports. */
+ anchorText?: string
+ /** Open the cited document at the cited place instead of the list. */
+ openSource?: boolean
+}
+
+/** A cited chunk to show inside its original document. */
+export type ReferenceSourceTarget = {
+ chunkId: string
+ knowledgeId: string
+ knowledgeBaseId?: string
+ title?: string
+ fileName?: string
+ knowledgeSource?: string
+ chunkType?: string
+ /** The chunk's own text; absent when only its document is known. */
+ content?: string
+ locators?: SourceLocator[]
+ anchorText?: string
+}
+
+/**
+ * Resolve a citation to the document and chunk it points at. Web results,
+ * tool output and FAQ entries have no original file and resolve to null.
+ */
+export function resolveReferenceSource(
+ refs: KnowledgeReferenceLike[] | null | undefined,
+ target: ReferenceHighlightTarget | null | undefined,
+): ReferenceSourceTarget | null {
+ const chunkId = String(target?.chunkId || '').trim()
+ if (!chunkId) return null
+ const list = Array.isArray(refs) ? refs.filter(Boolean) : []
+ const exact = list.find((r) => r.id === chunkId)
+ const grouped = exact || list.find((r) => r.chunk_ids?.includes(chunkId))
+ let ref = grouped
+ if (!ref && target?.documentTitle) {
+ const key = resolveReferenceHighlightKey(list, target)
+ const item = key ? buildReferenceList(list).find((i) => i.key === key && i.kind === 'document') : undefined
+ if (item?.knowledgeId) ref = list.find((r) => r.knowledge_id === item.knowledgeId)
+ }
+ if (!ref?.knowledge_id || isWebReference(ref) || isToolReference(ref) || ref.chunk_type === 'faq') return null
+ return {
+ chunkId,
+ knowledgeId: ref.knowledge_id,
+ knowledgeBaseId: ref.knowledge_base_id,
+ title: ref.knowledge_title,
+ fileName: ref.knowledge_filename,
+ knowledgeSource: ref.knowledge_source,
+ chunkType: ref.chunk_type,
+ // A grouped match belongs to another chunk of the same document.
+ content: exact ? exact.content : undefined,
+ locators: exact ? exact.source_locators : undefined,
+ anchorText: target?.anchorText,
+ }
}
export function resolveReferenceHighlightKey(
diff --git a/frontend/src/utils/sourceLocator.test.ts b/frontend/src/utils/sourceLocator.test.ts
new file mode 100644
index 000000000..75e520f4b
--- /dev/null
+++ b/frontend/src/utils/sourceLocator.test.ts
@@ -0,0 +1,132 @@
+import test from 'node:test'
+import assert from 'node:assert/strict'
+
+import {
+ findInText,
+ findNormalized,
+ locatorPages,
+ narrowTextLines,
+ ngramOverlap,
+ normalizeForMatch,
+ parseSourceLocators,
+ pickBestTexts,
+ selectLocatorsForSentence,
+ selectQuoteForSentence,
+ textFragmentUrl,
+} from './sourceLocator.ts'
+import { anchorWithContext, lastSentence } from './citationAnchor.ts'
+import { resolveReferenceSource } from './referenceSources.ts'
+import { resolveEpubPath } from './epubPreview.ts'
+import { resolvePreviewKind } from './filePreview.ts'
+
+test('normalization keeps letters and digits only, folding width and case', () => {
+ assert.equal(normalizeForMatch('8.14 索夹,Hello!'), '814索夹hello')
+})
+
+test('findInText maps a normalized match back to the raw text', () => {
+ const text = '8.14.11 索夹与吊索施工应符合下列规定:\n1 在满足施工需要的前提下,应减小猫道面层开孔面积。'
+ const hit = findInText(text, '在满足施工需要的前提下,应减小猫道面层开孔面积')
+ assert.ok(hit)
+ assert.equal(text.slice(hit.start, hit.end), '在满足施工需要的前提下,应减小猫道面层开孔面积')
+})
+
+test('findNormalized tolerates a differing opening via mid and tail anchors', () => {
+ const hay = normalizeForMatch('前文。紧固同一索夹的螺栓时,各螺栓受力应均匀,并应按规定顺序施拧。后文')
+ const m = findNormalized(hay, '(二)紧固同一索夹的螺栓时,各螺栓受力应均匀,并应按规定顺序施拧')
+ assert.ok(m)
+ assert.ok(m.score >= 0.5)
+ assert.equal(findNormalized(hay, '完全无关的一段内容不应该被匹配到这里来吧'), null)
+})
+
+test('ngramOverlap scores shared bigrams', () => {
+ assert.equal(ngramOverlap('abcd', 'xxabcdxx'), 1)
+ assert.equal(ngramOverlap('abcd', 'wxyz'), 0)
+})
+
+test('selectLocatorsForSentence narrows to the locator that supports the sentence', () => {
+ const locs = [
+ { type: 'pdf', page: 57, quote: '8.14.11 索夹与吊索施工应符合下列规定' },
+ { type: 'pdf', page: 57, quote: '1 在满足施工需要的前提下,应减小猫道面层开孔面积,并应在开孔位置四周绑扎防滑木条,设立警示标志。' },
+ { type: 'pdf', page: 57, quote: '2 索夹在主缆上定位后,应紧固螺栓。' },
+ ]
+ const picked = selectLocatorsForSentence(locs, '在满足施工需要的前提下,应减小猫道面层开孔面积,并应在开孔位置四周绑扎防滑木条,设立警示标志。')
+ assert.equal(picked.length, 1)
+ assert.equal(picked[0], locs[1])
+ assert.equal(selectLocatorsForSentence(locs, '毫不相干的一句话').length, 3)
+})
+
+test('selectQuoteForSentence prefers the supporting sentence of the chunk', () => {
+ const content = '索夹在主缆上定位后,应紧固螺栓。吊运物体时,作业人员不得沿主缆顶面行走。'
+ const [first] = selectQuoteForSentence(content, '吊运物体时作业人员不得沿主缆顶面行走')
+ assert.equal(first, '吊运物体时,作业人员不得沿主缆顶面行走。')
+ assert.equal(selectQuoteForSentence(content, '')[0], '索夹在主缆上定位后,应紧固螺栓。')
+})
+
+test('parseSourceLocators and locatorPages', () => {
+ const locs = parseSourceLocators([{ type: 'pdf', page: 3 }, null, { page: 2 }, { type: 'pdf', page: 3 }, { type: 'pdf', page: 1 }])
+ assert.equal(locs.length, 3)
+ assert.deepEqual(locatorPages(locs), [3, 1])
+ assert.deepEqual(parseSourceLocators('nope'), [])
+})
+
+test('textFragmentUrl builds a scroll-to-text link', () => {
+ assert.equal(
+ textFragmentUrl('https://example.com/a#old', 'safety rules - part 1'),
+ 'https://example.com/a#:~:text=safety%20rules%20%2D%20part%201',
+ )
+ const long = textFragmentUrl('https://example.com/a', 'x'.repeat(50) + ' ' + 'y'.repeat(50))
+ assert.match(long, /#:~:text=x+,.*y+$/)
+})
+
+test('pickBestTexts narrows a slide or sheet to the cited line', () => {
+ const bullets = ['风险与对策', '台风季可能影响高空作业:已准备应急停工预案', '钢材价格上涨8%:已锁定四季度采购价']
+ assert.deepEqual(pickBestTexts(bullets, '应对措施:已准备应急停工预案。依据:'), [1])
+ const rows = ['液压拉伸器 HT-300 6 张工 在用', '超声波轴力计 UB-20 2 李工 送检', '缆索吊机 CL-500 1 赵工 维修中']
+ assert.deepEqual(pickBestTexts(rows, '状态:维修中 负责人:赵工 依据:'), [2])
+ assert.deepEqual(pickBestTexts(bullets, '完全无关的一句话内容'), [])
+})
+
+test('narrowTextLines keeps the cited lines of a text range', () => {
+ const text = '一、安全方面\n上周共排查隐患17项,剩余2项为临边防护网破损。\n二、进度方面\n主缆猫道铺设滞后1天。'
+ const [[start, end]] = narrowTextLines(text, [[0, Array.from(text).length]], '临边防护网破损需要在周五前整改')
+ assert.equal(Array.from(text).slice(start, end).join(''), '上周共排查隐患17项,剩余2项为临边防护网破损。')
+ assert.deepEqual(narrowTextLines(text, [[0, 5]], '完全无关的内容句子'), [])
+})
+
+test('lastSentence keeps the cited sentence and extends short ones', () => {
+ assert.equal(lastSentence('第一句很长很长的内容。第二句也是完整的句子。'), '第二句也是完整的句子。')
+ assert.equal(lastSentence('前一句足够长的内容。短句。'), '前一句足够长的内容。短句。')
+})
+
+test('anchorWithContext borrows the previous block for a marker on a label line', () => {
+ const para = '在索夹吊装作业开始前,必须对主缆表面进行清理。'
+ assert.equal(anchorWithContext('依据:', [para, '2. 清理主缆表面']), `${para}依据:`)
+ assert.equal(anchorWithContext('索夹就位后应立即进行初拧。', [para]), '索夹就位后应立即进行初拧。')
+ assert.equal(anchorWithContext('依据:', []), '依据:')
+})
+
+test('resolveReferenceSource picks the cited chunk of a document reference', () => {
+ const refs = [
+ { id: 'c1', knowledge_id: 'k1', knowledge_filename: 'a.pdf', knowledge_title: 'A', content: 'x', source_locators: [{ type: 'pdf', page: 2 }] },
+ { id: 'c2', chunk_ids: ['c2', 'c3'], knowledge_id: 'k2', knowledge_title: 'B' },
+ { id: 'w', chunk_type: 'web_search', knowledge_id: 'k3' },
+ { id: 'f', chunk_type: 'faq', knowledge_id: 'k4' },
+ ]
+ const exact = resolveReferenceSource(refs, { chunkId: 'c1', anchorText: 's' })
+ assert.equal(exact?.knowledgeId, 'k1')
+ assert.equal(exact?.locators?.[0].page, 2)
+ assert.equal(exact?.anchorText, 's')
+ const grouped = resolveReferenceSource(refs, { chunkId: 'c3' })
+ assert.equal(grouped?.knowledgeId, 'k2')
+ assert.equal(grouped?.chunkId, 'c3')
+ assert.equal(grouped?.locators, undefined, 'a grouped match belongs to another chunk')
+ assert.equal(resolveReferenceSource(refs, { chunkId: 'w' }), null)
+ assert.equal(resolveReferenceSource(refs, { chunkId: 'f' }), null)
+ assert.equal(resolveReferenceSource(refs, { url: 'https://x' }), null)
+})
+
+test('EPUB paths resolve against their document and preview as epub', () => {
+ assert.equal(resolveEpubPath('OEBPS/text/ch1.xhtml', '../images/a%20b.png#x'), 'OEBPS/images/a b.png')
+ assert.equal(resolveEpubPath('OEBPS/content.opf', 'ch1.xhtml'), 'OEBPS/ch1.xhtml')
+ assert.equal(resolvePreviewKind('epub'), 'epub')
+})
diff --git a/frontend/src/utils/sourceLocator.ts b/frontend/src/utils/sourceLocator.ts
new file mode 100644
index 000000000..4cf3d92fb
--- /dev/null
+++ b/frontend/src/utils/sourceLocator.ts
@@ -0,0 +1,281 @@
+/**
+ * Source locators point from a cited chunk back into the original file.
+ * The backend fills them at ingestion time (see internal/types/source_locator.go);
+ * ordinals are 1-based and PDF boxes are fractions of the displayed page with
+ * the origin at its top-left corner.
+ */
+export type SourceLocatorType = 'pdf' | 'docx' | 'slide' | 'sheet' | 'text' | 'time' | 'section'
+
+export type SourceLocator = {
+ type: SourceLocatorType | string
+ page?: number
+ bbox?: [number, number, number, number] | number[]
+ block?: number
+ slide?: number
+ sheet?: string
+ row_start?: number
+ row_end?: number
+ start?: number
+ end?: number
+ start_ms?: number
+ end_ms?: number
+ section?: number
+ title?: string
+ quote?: string
+}
+
+/** What the preview should reveal: structural targets plus text to match. */
+export type SourceLocateRequest = {
+ /** Locators to reveal, most relevant first. */
+ locators: SourceLocator[]
+ /** Text to find when no locator narrows it down (or to refine one). */
+ quotes: string[]
+ /** Changes on every request so re-clicking the same citation re-scrolls. */
+ token: number
+ /** The answer sentence the citation supports, to narrow a coarse locator. */
+ sentence?: string
+ /** The cited chunk's text, bounding where a narrowed match may come from. */
+ scope?: string
+}
+
+// ---------------------------------------------------------------------------
+// Normalization: letters and digits only, lower-cased, full-width folded. The
+// same projection the backend aligner uses, so Markdown syntax, whitespace and
+// punctuation differences between parsed text and the rendered original do not
+// break matching.
+
+const LETTER_OR_DIGIT = /[\p{L}\p{N}]/u
+
+export function foldChar(ch: string): string {
+ if (!LETTER_OR_DIGIT.test(ch)) return ''
+ return ch.normalize('NFKC').toLowerCase()
+}
+
+export function normalizeForMatch(text: string): string {
+ let out = ''
+ for (const ch of String(text || '')) out += foldChar(ch)
+ return out
+}
+
+/**
+ * Normalized projection of `text` with, for each normalized character, the
+ * UTF-16 offset in `text` of the character it came from.
+ */
+export function normalizeWithPositions(text: string): { norm: string; pos: number[] } {
+ let norm = ''
+ const pos: number[] = []
+ let i = 0
+ for (const ch of String(text || '')) {
+ const folded = foldChar(ch)
+ for (const f of folded) {
+ norm += f
+ pos.push(i)
+ }
+ i += ch.length
+ }
+ return { norm, pos }
+}
+
+// ---------------------------------------------------------------------------
+// Fuzzy location of a quote inside a normalized haystack.
+
+const KEY_LEN = 24
+
+export type NormalizedMatch = { start: number; end: number; score: number }
+
+/**
+ * Find `quote` (raw text) inside `haystack` (already normalized). Tries the
+ * whole quote, then anchors from its head, middle and tail, and extends the
+ * match to the quote's length. Returns normalized offsets or null.
+ */
+export function findNormalized(haystack: string, quote: string): NormalizedMatch | null {
+ const needle = normalizeForMatch(quote)
+ if (needle.length < 2 || !haystack) return null
+ const whole = haystack.indexOf(needle)
+ if (whole >= 0) return { start: whole, end: whole + needle.length, score: 1 }
+ if (needle.length <= KEY_LEN) return null
+
+ const anchors: Array<{ offset: number; key: string }> = [
+ { offset: 0, key: needle.slice(0, KEY_LEN) },
+ { offset: Math.floor(needle.length / 2), key: needle.slice(Math.floor(needle.length / 2), Math.floor(needle.length / 2) + KEY_LEN) },
+ { offset: needle.length - KEY_LEN, key: needle.slice(-KEY_LEN) },
+ ]
+ let best: NormalizedMatch | null = null
+ for (const anchor of anchors) {
+ let from = 0
+ // Every occurrence of the anchor is a candidate; score by how much of
+ // the quote agrees around it.
+ for (let guard = 0; guard < 50; guard++) {
+ const at = haystack.indexOf(anchor.key, from)
+ if (at < 0) break
+ from = at + 1
+ const start = Math.max(0, at - anchor.offset)
+ const end = Math.min(haystack.length, start + needle.length)
+ const score = ngramOverlap(needle, haystack.slice(start, end))
+ if (!best || score > best.score) best = { start, end, score }
+ }
+ }
+ return best && best.score >= 0.5 ? best : null
+}
+
+/** Share of the character bigrams of `a` that also occur in `b`. */
+export function ngramOverlap(a: string, b: string): number {
+ if (a.length < 2 || b.length < 2) return a && b && a === b ? 1 : 0
+ const counts = new Map<string, number>()
+ for (let i = 0; i + 1 < b.length; i++) {
+ const g = b.slice(i, i + 2)
+ counts.set(g, (counts.get(g) || 0) + 1)
+ }
+ let hit = 0
+ for (let i = 0; i + 1 < a.length; i++) {
+ const g = a.slice(i, i + 2)
+ const n = counts.get(g) || 0
+ if (n > 0) {
+ hit++
+ counts.set(g, n - 1)
+ }
+ }
+ return hit / (a.length - 1)
+}
+
+/** Locate a quote in raw text; returns UTF-16 offsets into `text`. */
+export function findInText(text: string, quote: string): { start: number; end: number } | null {
+ const { norm, pos } = normalizeWithPositions(text)
+ const m = findNormalized(norm, quote)
+ if (!m || !pos.length) return null
+ const start = pos[m.start]
+ const lastIdx = Math.min(m.end, pos.length) - 1
+ const last = pos[lastIdx]
+ // Extend to cover the whole last character (surrogate pairs included).
+ const end = last + (text.codePointAt(last)! > 0xffff ? 2 : 1)
+ return { start, end }
+}
+
+// ---------------------------------------------------------------------------
+// Picking what to reveal for a citation.
+
+/** Split text into sentence-sized pieces for alignment. */
+export function splitSentences(text: string): string[] {
+ return String(text || '')
+ .split(/(?<=[。!?!?;;])\s*|\n+/u)
+ .map((s) => s.trim())
+ .filter((s) => normalizeForMatch(s).length >= 4)
+}
+
+const ALIGN_THRESHOLD = 0.35
+
+/**
+ * Narrow a chunk's locators to the ones that support the cited sentence. The
+ * answer paraphrases the source, so each locator's quote is scored by how
+ * many of the sentence's character bigrams it contains; locators well below
+ * the best are dropped. When nothing scores, every locator is kept.
+ */
+export function selectLocatorsForSentence(locators: SourceLocator[], sentence: string): SourceLocator[] {
+ const list = Array.isArray(locators) ? locators.filter(Boolean) : []
+ const needle = normalizeForMatch(sentence)
+ if (list.length <= 1 || needle.length < 4) return list
+ const scored = list.map((loc) => ({ loc, score: ngramOverlap(needle, normalizeForMatch(loc.quote || '')) }))
+ const best = Math.max(...scored.map((s) => s.score))
+ if (best < ALIGN_THRESHOLD) return list
+ return scored.filter((s) => s.score >= best * 0.8).map((s) => s.loc)
+}
+
+/**
+ * Indices of the texts that best support `sentence`: every text within 80% of
+ * the best once it clears the alignment threshold, or else the single best
+ * when it clearly stands out (table rows share few bigrams with prose).
+ * Empty when nothing aligns, so the caller keeps the whole region.
+ */
+export function pickBestTexts(texts: string[], sentence: string): number[] {
+ const needle = normalizeForMatch(sentence || '')
+ if (needle.length < 4 || texts.length < 2) return []
+ const scores = texts.map((t) => ngramOverlap(needle, normalizeForMatch(t)))
+ const best = Math.max(...scores)
+ if (best >= ALIGN_THRESHOLD) return scores.flatMap((s, i) => (s >= best * 0.8 ? [i] : []))
+ const top = scores.indexOf(best)
+ const second = Math.max(0, ...scores.filter((_, i) => i !== top))
+ return best >= 0.2 && second <= best * 0.5 ? [top] : []
+}
+
+/**
+ * Code point ranges of the lines within `spans` of `text` that best support
+ * `sentence`, or empty when none aligns. Offsets are code points, the unit
+ * text locators use.
+ */
+export function narrowTextLines(text: string, spans: Array<[number, number]>, sentence: string): Array<[number, number]> {
+ const chars = Array.from(text || '')
+ const lines: Array<{ text: string; start: number; end: number }> = []
+ for (const [from, to] of spans) {
+ const stop = Math.min(to, chars.length)
+ let lineStart = Math.max(0, from)
+ for (let i = lineStart; i <= stop; i++) {
+ if (i < stop && chars[i] !== '\n') continue
+ const line = chars.slice(lineStart, i).join('')
+ if (line.trim()) lines.push({ text: line, start: lineStart, end: i })
+ lineStart = i + 1
+ }
+ }
+ return pickBestTexts(
+ lines.map((l) => l.text),
+ sentence,
+ ).map((i) => [lines[i].start, lines[i].end])
+}
+
+/**
+ * The piece of a chunk's text that best supports the cited sentence, used as
+ * the quote to search when there are no locators. Falls back to the chunk's
+ * opening when the sentence matches nothing.
+ */
+export function selectQuoteForSentence(content: string, sentence: string): string[] {
+ const pieces = splitSentences(content)
+ if (!pieces.length) return content ? [content.slice(0, 200)] : []
+ const needle = normalizeForMatch(sentence)
+ if (needle.length >= 4) {
+ let best = { piece: '', score: 0 }
+ for (const piece of pieces) {
+ const score = ngramOverlap(needle, normalizeForMatch(piece))
+ if (score > best.score) best = { piece, score }
+ }
+ if (best.score >= ALIGN_THRESHOLD) return [best.piece, pieces[0]]
+ }
+ return [pieces[0], pieces.slice(0, 3).join('')]
+}
+
+/** Normalize locator payloads from the API, dropping malformed entries. */
+export function parseSourceLocators(raw: unknown): SourceLocator[] {
+ if (!Array.isArray(raw)) return []
+ return raw.filter((item): item is SourceLocator => !!item && typeof item === 'object' && typeof (item as SourceLocator).type === 'string')
+}
+
+/** 1-based pages targeted by PDF locators, in first-seen order. */
+export function locatorPages(locators: SourceLocator[]): number[] {
+ const pages: number[] = []
+ for (const loc of locators) {
+ if (loc.type === 'pdf' && loc.page && !pages.includes(loc.page)) pages.push(loc.page)
+ }
+ return pages
+}
+
+/** Text fragment URL (`#:~:text=`) that makes browsers scroll to and mark a quote. */
+export function textFragmentUrl(url: string, quote: string): string {
+ const clean = String(quote || '').replace(/\s+/g, ' ').trim()
+ if (!url || !clean) return url
+ const words = clean.split(' ')
+ let fragment: string
+ if (clean.length <= 80) {
+ fragment = encodeTextFragment(clean)
+ } else {
+ // start,end form keeps the URL short for long passages.
+ const head = clean.slice(0, 40).trim()
+ const tail = clean.slice(-40).trim()
+ fragment = words.length > 1 || /[ -鿿]/u.test(clean)
+ ? `${encodeTextFragment(head)},${encodeTextFragment(tail)}`
+ : encodeTextFragment(head)
+ }
+ const base = url.split('#')[0]
+ return `${base}#:~:text=${fragment}`
+}
+
+function encodeTextFragment(text: string): string {
+ return encodeURIComponent(text).replace(/-/g, '%2D').replace(/,/g, '%2C').replace(/&/g, '%26')
+}
diff --git a/frontend/src/utils/sourceLocatorDom.ts b/frontend/src/utils/sourceLocatorDom.ts
new file mode 100644
index 000000000..0842f251b
--- /dev/null
+++ b/frontend/src/utils/sourceLocatorDom.ts
@@ -0,0 +1,200 @@
+import { findInText } from './sourceLocator'
+
+/** Name of the CSS custom highlight used for cited text. */
+export const SOURCE_HIGHLIGHT_NAME = 'source-locate'
+/** Class added to whole elements (paragraphs, table rows, slides) that are cited. */
+export const SOURCE_BLOCK_CLASS = 'source-locate-block'
+const MARK_CLASS = 'source-locate-mark'
+
+type TextIndex = {
+ text: string
+ nodes: Array<{ node: Text; start: number }>
+}
+
+const SKIP_TAGS = new Set(['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE'])
+
+/** Concatenate the text nodes under `root`, remembering where each starts. */
+export function buildTextIndex(root: Node): TextIndex {
+ const nodes: TextIndex['nodes'] = []
+ let text = ''
+ const doc = root.ownerDocument || (root as Document)
+ const walker = doc.createTreeWalker(root, NodeFilter.SHOW_TEXT, {
+ acceptNode(node) {
+ let el = node.parentElement
+ while (el && el !== root) {
+ if (SKIP_TAGS.has(el.tagName)) return NodeFilter.FILTER_REJECT
+ el = el.parentElement
+ }
+ return node.nodeValue ? NodeFilter.FILTER_ACCEPT : NodeFilter.FILTER_REJECT
+ },
+ })
+ for (let n = walker.nextNode(); n; n = walker.nextNode()) {
+ const t = n as Text
+ nodes.push({ node: t, start: text.length })
+ // A separator keeps words from adjacent blocks from fusing; it is not a
+ // letter or digit, so matching ignores it.
+ text += t.nodeValue + ' '
+ }
+ return { text, nodes }
+}
+
+function pointAt(index: TextIndex, offset: number): { node: Text; offset: number } | null {
+ if (!index.nodes.length) return null
+ let lo = 0
+ let hi = index.nodes.length - 1
+ while (lo < hi) {
+ const mid = (lo + hi + 1) >> 1
+ if (index.nodes[mid].start <= offset) lo = mid
+ else hi = mid - 1
+ }
+ const entry = index.nodes[lo]
+ const len = entry.node.nodeValue?.length || 0
+ return { node: entry.node, offset: Math.max(0, Math.min(len, offset - entry.start)) }
+}
+
+/** Range of `root` holding the best match for `quote`, or null. */
+export function findTextRange(root: Node, quote: string, index = buildTextIndex(root)): Range | null {
+ const hit = findInText(index.text, quote)
+ if (!hit) return null
+ const from = pointAt(index, hit.start)
+ const to = pointAt(index, hit.end)
+ if (!from || !to) return null
+ const range = (root.ownerDocument || (root as Document)).createRange()
+ range.setStart(from.node, from.offset)
+ range.setEnd(to.node, to.offset)
+ return range
+}
+
+/**
+ * Range covering the code point offsets [start, end) of `root`'s text
+ * content (the unit the backend reports for plain-text originals).
+ */
+export function rangeForOffsets(root: Node, startCp: number, endCp: number): Range | null {
+ const index = buildTextIndex(root)
+ if (!index.nodes.length) return null
+ const full = index.nodes.map((n) => n.node.nodeValue || '').join('')
+ const toUnits = (cp: number) => {
+ let units = 0
+ let count = 0
+ for (const ch of full) {
+ if (count >= cp) break
+ units += ch.length
+ count++
+ }
+ return units
+ }
+ const start = toUnits(startCp)
+ const end = toUnits(endCp)
+ let consumed = 0
+ let startPoint: { node: Text; offset: number } | null = null
+ let endPoint: { node: Text; offset: number } | null = null
+ for (const { node } of index.nodes) {
+ const len = node.nodeValue?.length || 0
+ if (!startPoint && start <= consumed + len) startPoint = { node, offset: Math.max(0, start - consumed) }
+ if (!endPoint && end <= consumed + len) {
+ endPoint = { node, offset: Math.max(0, end - consumed) }
+ break
+ }
+ consumed += len
+ }
+ if (!startPoint) return null
+ if (!endPoint) {
+ const last = index.nodes[index.nodes.length - 1].node
+ endPoint = { node: last, offset: last.nodeValue?.length || 0 }
+ }
+ const range = (root.ownerDocument || (root as Document)).createRange()
+ range.setStart(startPoint.node, startPoint.offset)
+ range.setEnd(endPoint.node, endPoint.offset)
+ return range
+}
+
+type HighlightRegistry = { set(name: string, h: unknown): void; delete(name: string): void }
+
+function highlightRegistry(): { registry: HighlightRegistry; Highlight: new (...r: Range[]) => unknown } | null {
+ const css = (globalThis as unknown as { CSS?: { highlights?: HighlightRegistry } }).CSS
+ const Highlight = (globalThis as unknown as { Highlight?: new (...r: Range[]) => unknown }).Highlight
+ if (css?.highlights && Highlight) return { registry: css.highlights, Highlight }
+ return null
+}
+
+/**
+ * Highlight ranges without touching the document when the CSS Custom
+ * Highlight API exists; otherwise wrap the text in <mark>. Returns a function
+ * that removes the highlight.
+ */
+export function highlightRanges(ranges: Range[]): () => void {
+ const live = ranges.filter((r) => r && !r.collapsed)
+ if (!live.length) return () => {}
+ const api = highlightRegistry()
+ if (api) {
+ api.registry.set(SOURCE_HIGHLIGHT_NAME, new api.Highlight(...live))
+ return () => api.registry.delete(SOURCE_HIGHLIGHT_NAME)
+ }
+ const marks: HTMLElement[] = []
+ for (const range of live) marks.push(...wrapRange(range))
+ return () => {
+ for (const mark of marks) {
+ const parent = mark.parentNode
+ if (!parent) continue
+ while (mark.firstChild) parent.insertBefore(mark.firstChild, mark)
+ parent.removeChild(mark)
+ parent.normalize()
+ }
+ }
+}
+
+function wrapRange(range: Range): HTMLElement[] {
+ const doc = range.startContainer.ownerDocument
+ if (!doc) return []
+ const texts: Text[] = []
+ const walker = doc.createTreeWalker(range.commonAncestorContainer, NodeFilter.SHOW_TEXT)
+ for (let n = walker.currentNode as Node | null; n; n = walker.nextNode()) {
+ if (n.nodeType === Node.TEXT_NODE && range.intersectsNode(n)) texts.push(n as Text)
+ }
+ const marks: HTMLElement[] = []
+ for (const node of texts) {
+ let target = node
+ const start = node === range.startContainer ? range.startOffset : 0
+ const end = node === range.endContainer ? range.endOffset : node.length
+ if (end <= start) continue
+ if (end < target.length) target.splitText(end)
+ if (start > 0) target = target.splitText(start)
+ const mark = doc.createElement('mark')
+ mark.className = MARK_CLASS
+ target.parentNode?.insertBefore(mark, target)
+ mark.appendChild(target)
+ marks.push(mark)
+ }
+ return marks
+}
+
+/** Mark whole elements as cited. Returns a function that unmarks them. */
+export function highlightElements(elements: Element[]): () => void {
+ const list = elements.filter(Boolean)
+ for (const el of list) el.classList.add(SOURCE_BLOCK_CLASS)
+ return () => {
+ for (const el of list) el.classList.remove(SOURCE_BLOCK_CLASS)
+ }
+}
+
+/**
+ * Scroll `container` so `rect` (in viewport coordinates) sits about a third
+ * of the way down, without moving any outer scroll container. The jump is
+ * instant: a smooth scroll is cancelled by the focus handling and reflows
+ * that happen while a preview is still settling.
+ */
+export function scrollRectIntoContainer(container: Element, rect: DOMRect | { top: number; bottom: number }) {
+ const box = container.getBoundingClientRect()
+ const target = container.scrollTop + (rect.top - box.top) - Math.max(24, box.height / 3)
+ container.scrollTop = Math.max(0, target)
+}
+
+/** The nearest scrollable ancestor of `el` (or `fallback`). */
+export function scrollParent(el: Element | null, fallback: Element): Element {
+ for (let node = el?.parentElement; node; node = node.parentElement) {
+ const style = getComputedStyle(node)
+ if (/(auto|scroll)/.test(style.overflowY) && node.scrollHeight > node.clientHeight) return node
+ if (node === fallback) break
+ }
+ return fallback
+}
diff --git a/frontend/src/views/chat/components/AgentStreamDisplay.vue b/frontend/src/views/chat/components/AgentStreamDisplay.vue
index d97984c55..76862d98e 100644
--- a/frontend/src/views/chat/components/AgentStreamDisplay.vue
+++ b/frontend/src/views/chat/components/AgentStreamDisplay.vue
@@ -654,6 +654,7 @@ import { useChatCitationPopover } from '@/composables/useChatCitationPopover';
import { useChatReferencesDrawer } from '@/composables/useChatReferencesDrawer';
import type { KnowledgeReferenceLike, ReferenceHighlightTarget } from '@/utils/referenceSources';
import { resolveCitationChunkId } from '@/utils/citationMarkdown';
+import { citationAnchorText } from '@/utils/citationAnchor';
import { getWikiPage, type WikiPage } from '@/api/wiki';
import { MessagePlugin } from 'tdesign-vue-next';
import { useUIStore } from '@/stores/ui';
@@ -2403,6 +2404,8 @@ const onRootClick = (e: Event) => {
chunkId,
documentTitle: title,
knowledgeBaseId: kbId,
+ anchorText: citationAnchorText(kbEl),
+ openSource: !props.embeddedMode,
})) {
return;
}
@@ -2488,6 +2491,8 @@ const onRootKeydown = (e: KeyboardEvent) => {
chunkId,
documentTitle: title,
knowledgeBaseId: kbId,
+ anchorText: citationAnchorText(kbEl),
+ openSource: !props.embeddedMode,
})) {
return;
}
diff --git a/frontend/src/views/chat/components/chatLinksNewTab.test.mjs b/frontend/src/views/chat/components/chatLinksNewTab.test.mjs
index 12de7694c..5018833f1 100644
--- a/frontend/src/views/chat/components/chatLinksNewTab.test.mjs
+++ b/frontend/src/views/chat/components/chatLinksNewTab.test.mjs
@@ -85,5 +85,6 @@ test('references drawer smoothly shifts the chat area while opening', () => {
assert.match(referenceDrawer, /\.chat-references-panel \{[\s\S]*?position: fixed;/)
assert.match(chatView, /'has-references-panel': referencesDrawerVisible/)
assert.match(chatView, /transition: padding-right (?:0\.3s|var\(--app-motion-slow\)) cubic-bezier\(0\.22, 0\.61, 0\.36, 1\)/)
- assert.match(chatView, /&\.has-references-panel:not\(\.is-embedded\)[\s\S]*?padding-right:\s*420px/)
+ // The panel widens while showing an original document; 420px is the list width.
+ assert.match(chatView, /&\.has-references-panel:not\(\.is-embedded\)[\s\S]*?padding-right:\s*var\(--references-panel-width, 420px\)/)
})
diff --git a/frontend/src/views/chat/index.vue b/frontend/src/views/chat/index.vue
index b5cbe91ca..fe5557876 100644
--- a/frontend/src/views/chat/index.vue
+++ b/frontend/src/views/chat/index.vue
@@ -3,7 +3,10 @@
'is-embedded': embeddedMode,
'has-references-panel': referencesDrawerVisible,
'has-sandbox-panel': sandboxPanel.visible.value,
- }" :style="{ '--sandbox-panel-width': `${sandboxPanel.width.value}px` }">
+ }" :style="{
+ '--sandbox-panel-width': `${sandboxPanel.width.value}px`,
+ '--references-panel-width': `${referencesPanelWidth}px`,
+ }">
<div v-if="!embeddedMode" class="chat-topbar">
<ChatHeader :session="currentSession" />
<div v-if="!sandboxPanel.visible.value" class="sandbox-header-toggle">
@@ -187,6 +190,7 @@
:agent-id="useSettingsStoreInstance.selectedAgentId"
:agent-source-tenant-id="useSettingsStoreInstance.selectedAgentSourceTenantId"
:shifted="referencesDrawerVisible"
+ :shift-width="referencesPanelWidth"
:artifacts="sessionArtifacts" :artifacts-collecting="sessionArtifactsCollecting"
@artifact-deleted="handleArtifactDeleted" />
</template>
@@ -246,7 +250,7 @@ import { isCollectingSkillArtifacts } from '@/utils/skillArtifacts';
const referencesDrawer = provideChatReferencesDrawer();
provideChatAttachmentPreviewDrawer();
const sandboxPanel = provideChatSandboxPanel();
-const { visible: referencesDrawerVisible } = referencesDrawer;
+const { visible: referencesDrawerVisible, panelWidth: referencesPanelWidth } = referencesDrawer;
const props = defineProps({
session_id: { type: String, default: '' },
@@ -1776,7 +1780,7 @@ onBeforeRouteUpdate((to, from, next) => {
&.has-references-panel:not(.is-embedded) {
@media (min-width: 960px) {
- padding-right: 420px;
+ padding-right: var(--references-panel-width, 420px);
box-sizing: border-box;
.chat_scroll_box {
@@ -1796,7 +1800,7 @@ onBeforeRouteUpdate((to, from, next) => {
&.has-sandbox-panel.has-references-panel:not(.is-embedded) {
@media (min-width: 1400px) {
- padding-right: calc(420px + var(--sandbox-panel-width, 420px));
+ padding-right: calc(var(--references-panel-width, 420px) + var(--sandbox-panel-width, 420px));
}
@media (max-width: 1399.98px) and (min-width: 960px) {
diff --git a/internal/application/service/chat_pipeline/merge_overlap.go b/internal/application/service/chat_pipeline/merge_overlap.go
index 025e6c692..1425de8cd 100644
--- a/internal/application/service/chat_pipeline/merge_overlap.go
+++ b/internal/application/service/chat_pipeline/merge_overlap.go
@@ -6,6 +6,7 @@ import (
"sort"
"github.com/Tencent/WeKnora/internal/searchutil"
+ "github.com/Tencent/WeKnora/internal/sourceloc"
"github.com/Tencent/WeKnora/internal/types"
)
@@ -51,6 +52,8 @@ func (p *PluginMerge) mergeSequentialChunks(
recordMergedChild(ctx, knowledgeID, lastChunk, current, "image_merge")
}
+ lastChunk.SourceLocators = sourceloc.MergeLocators(lastChunk.SourceLocators, current.SourceLocators)
+
// Extend the merged group's span and keep the higher score.
if current.ChunkIndex > last.lastIndex {
last.lastIndex = current.ChunkIndex
diff --git a/internal/application/service/chat_pipeline/search_entity.go b/internal/application/service/chat_pipeline/search_entity.go
index 3ab64c83f..25cd98678 100644
--- a/internal/application/service/chat_pipeline/search_entity.go
+++ b/internal/application/service/chat_pipeline/search_entity.go
@@ -261,5 +261,6 @@ func chunk2SearchResult(chunk *types.Chunk, knowledge *types.Knowledge) *types.S
KnowledgeChannel: knowledge.Channel,
ChunkMetadata: chunk.Metadata,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
+ SourceLocators: chunk.SourceLocators,
}
}
diff --git a/internal/application/service/image_multimodal.go b/internal/application/service/image_multimodal.go
index 2b11c3f23..aeb1cdc48 100644
--- a/internal/application/service/image_multimodal.go
+++ b/internal/application/service/image_multimodal.go
@@ -355,6 +355,7 @@ func (s *ImageMultimodalService) processImage(
Content: imageInfo.OCRText,
ChunkType: types.ChunkTypeImageOCR,
ParentChunkID: payload.ChunkID,
+ SourceLocators: payload.SourceLocators,
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
ImageInfo: string(imageInfoJSON),
@@ -372,6 +373,7 @@ func (s *ImageMultimodalService) processImage(
Content: imageInfo.Caption,
ChunkType: types.ChunkTypeImageCaption,
ParentChunkID: payload.ChunkID,
+ SourceLocators: payload.SourceLocators,
IsEnabled: true,
Flags: types.ChunkFlagRecommended,
ImageInfo: string(imageInfoJSON),
diff --git a/internal/application/service/knowledge_clone_move.go b/internal/application/service/knowledge_clone_move.go
index 7390a1976..596589761 100644
--- a/internal/application/service/knowledge_clone_move.go
+++ b/internal/application/service/knowledge_clone_move.go
@@ -357,6 +357,7 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl
Metadata: sourceChunk.Metadata,
ContentHash: sourceChunk.ContentHash,
ImageInfo: newImageInfo,
+ SourceLocators: sourceChunk.SourceLocators,
CreatedAt: now,
UpdatedAt: now,
}
diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go
index b3edb8d61..59f2b6a99 100644
--- a/internal/application/service/knowledge_process.go
+++ b/internal/application/service/knowledge_process.go
@@ -22,6 +22,7 @@ import (
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/models/embedding"
"github.com/Tencent/WeKnora/internal/searchutil"
+ "github.com/Tencent/WeKnora/internal/sourceloc"
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
@@ -527,6 +528,7 @@ func (s *knowledgeService) processChunks(ctx context.Context,
StartAt: pc.Start,
EndAt: pc.End,
ChunkType: types.ChunkTypeParentText,
+ SourceLocators: pc.SourceLocators,
}
}
// Set prev/next links for parent chunks
@@ -567,6 +569,7 @@ func (s *knowledgeService) processChunks(ctx context.Context,
StartAt: int(chunkData.Start),
EndAt: int(chunkData.End),
ChunkType: types.ChunkTypeText,
+ SourceLocators: chunkData.SourceLocators,
}
// Wire up ParentChunkID for child chunks
@@ -3742,19 +3745,18 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
return fmt.Errorf("audio transcription failed: %w", err)
}
- var transcribedText string
- if transcriptionResult != nil {
- transcribedText = transcriptionResult.Text
- }
+ transcribedText, timeBlocks := transcriptWithSegments(transcriptionResult)
if transcribedText == "" {
logger.Warn(ctx, "[ASR] Transcription returned empty text")
transcribedText = "[No speech detected in audio file]"
+ timeBlocks = nil
}
logger.Infof(ctx, "[ASR] Transcription completed, text length=%d", len(transcribedText))
// Replace the audio placeholder with the transcribed text
convertResult.MarkdownContent = transcribedText
+ convertResult.SourceBlocks = timeBlocks
convertResult.IsAudio = false
convertResult.AudioData = nil
}
@@ -3767,6 +3769,14 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
// <img> tags to `![...](...)` and then escaping them in a later
// HTML→GFM pass. Line endings are normalized first so injected row
// breaks are plain LF.
+ // Source blocks were recorded against the parser's markdown; keep it so
+ // they can be re-aimed after the rewrites below.
+ var parsedMarkdown string
+ var sourceBlocks []types.SourceBlock
+ if convertResult != nil {
+ parsedMarkdown = convertResult.MarkdownContent
+ sourceBlocks = convertResult.SourceBlocks
+ }
sanitizeReadResult(convertResult)
if convertResult != nil {
convertResult.MarkdownContent = chunker.NormalizeLineEndings(convertResult.MarkdownContent)
@@ -3810,6 +3820,11 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
// unbound extracted image renders broken for org-shared KB viewers (#3342).
s.bindStoredImages(ctx, knowledge, storedImages)
+ var sourceIndex *sourceloc.Index
+ if convertResult != nil {
+ sourceIndex = buildSourceIndex(parsedMarkdown, sourceBlocks, convertResult.MarkdownContent, storedImages)
+ }
+
// Step 3: Split into chunks using Go chunker. Line endings and inline
// HTML tables were normalized before image resolution above.
chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)
@@ -3831,17 +3846,21 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
chunks = make([]types.ParsedChunk, len(pcResult.Children))
for i, c := range pcResult.Children {
chunks[i] = types.ParsedChunk{
- Content: c.Content,
- ContextHeader: c.ContextHeader,
- Seq: c.Seq,
- Start: c.Start,
- End: c.End,
- ParentIndex: c.ParentIndex,
+ Content: c.Content,
+ ContextHeader: c.ContextHeader,
+ Seq: c.Seq,
+ Start: c.Start,
+ End: c.End,
+ ParentIndex: c.ParentIndex,
+ SourceLocators: sourceIndex.Locators(c.Start, c.End),
}
}
parentChunks := make([]types.ParsedParentChunk, len(pcResult.Parents))
for i, p := range pcResult.Parents {
- parentChunks[i] = types.ParsedParentChunk{Content: p.Content, Seq: p.Seq, Start: p.Start, End: p.End}
+ parentChunks[i] = types.ParsedParentChunk{
+ Content: p.Content, Seq: p.Seq, Start: p.Start, End: p.End,
+ SourceLocators: sourceIndex.Locators(p.Start, p.End),
+ }
}
processOpts.ParentChunks = parentChunks
logger.Infof(ctx, "Split document into %d parent + %d child chunks for knowledge %s",
@@ -3851,11 +3870,12 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
chunks = make([]types.ParsedChunk, len(splitChunks))
for i, c := range splitChunks {
chunks[i] = types.ParsedChunk{
- Content: c.Content,
- ContextHeader: c.ContextHeader,
- Seq: c.Seq,
- Start: c.Start,
- End: c.End,
+ Content: c.Content,
+ ContextHeader: c.ContextHeader,
+ Seq: c.Seq,
+ Start: c.Start,
+ End: c.End,
+ SourceLocators: sourceIndex.Locators(c.Start, c.End),
}
}
logger.Infof(ctx, "Split document into %d chunks for knowledge %s", len(chunks), knowledge.ID)
@@ -4018,6 +4038,9 @@ func (s *knowledgeService) convert(
werrors.ErrCodeDocReaderParseFailed, result.Error, nil)
return nil, nil
}
+ if !isURL {
+ attachStructureBlocks(ctx, fileType, req.FileContent, result)
+ }
docOutput := types.JSONMap{
"text_length": len(result.MarkdownContent),
"images_found": len(result.ImageRefs),
@@ -4193,6 +4216,7 @@ func (s *knowledgeService) enqueueImageMultimodalTasks(
ImageSourceType: metadata["image_source_type"],
Attempt: attempt,
ImageIndex: idx,
+ SourceLocators: img.SourceLocators,
}
langfuse.InjectTracing(ctx, &payload)
diff --git a/internal/application/service/knowledge_source_locators.go b/internal/application/service/knowledge_source_locators.go
new file mode 100644
index 000000000..89e9489e2
--- /dev/null
+++ b/internal/application/service/knowledge_source_locators.go
@@ -0,0 +1,133 @@
+package service
+
+import (
+ "context"
+ "sort"
+ "strings"
+ "time"
+ "unicode/utf8"
+
+ "github.com/Tencent/WeKnora/internal/infrastructure/docparser"
+ "github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/models/asr"
+ "github.com/Tencent/WeKnora/internal/sourceloc"
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+// attachStructureBlocks gives a parse result source blocks when the parser
+// reported none: office files are aligned against their own structure, and
+// plain-text files that the parser passed through verbatim map onto
+// themselves. Failures only cost the locators, never the parse.
+func attachStructureBlocks(ctx context.Context, fileType string, data []byte, result *types.ReadResult) {
+ if result == nil || len(result.SourceBlocks) > 0 || result.MarkdownContent == "" || len(data) == 0 {
+ return
+ }
+ ft := strings.ToLower(strings.TrimPrefix(fileType, "."))
+ switch {
+ case sourceloc.SupportsStructure(ft):
+ started := time.Now()
+ blocks, err := sourceloc.AlignStructure(ft, data, result.MarkdownContent)
+ if err != nil {
+ logger.Warnf(ctx, "[SourceLocator] reading %s structure failed: %v", ft, err)
+ return
+ }
+ result.SourceBlocks = blocks
+ logger.Infof(ctx, "[SourceLocator] aligned %d %s blocks in %s", len(blocks), ft, time.Since(started))
+ case ft == "md" || ft == "markdown" || ft == "txt" || ft == "text":
+ if result.MarkdownContent == string(data) {
+ result.SourceBlocks = passThroughTextBlocks(result.MarkdownContent)
+ }
+ }
+}
+
+// passThroughTextBlocks maps a verbatim text original onto itself. Text
+// locators count from after a UTF-8 BOM, since viewers decode the file
+// without it; the blocks still cover the markdown, BOM included.
+func passThroughTextBlocks(content string) []types.SourceBlock {
+ body, hasBOM := strings.CutPrefix(content, "\ufeff")
+ blocks := sourceloc.TextBlocks(body)
+ if hasBOM {
+ for i := range blocks {
+ blocks[i].Start++
+ blocks[i].End++
+ }
+ }
+ return blocks
+}
+
+// transcriptWithSegments renders an ASR result with one line per timed
+// segment, and the time blocks that point each line back into the audio.
+// Without segments the plain transcript is returned and no blocks.
+func transcriptWithSegments(result *asr.TranscriptionResult) (string, []types.SourceBlock) {
+ if result == nil {
+ return "", nil
+ }
+ if len(result.Segments) == 0 {
+ return result.Text, nil
+ }
+ var (
+ sb strings.Builder
+ blocks []types.SourceBlock
+ pos int
+ )
+ for _, seg := range result.Segments {
+ text := strings.TrimSpace(seg.Text)
+ if text == "" {
+ continue
+ }
+ if sb.Len() > 0 {
+ sb.WriteByte('\n')
+ pos++
+ }
+ n := len([]rune(text))
+ sb.WriteString(text)
+ blocks = append(blocks, types.SourceBlock{
+ Start: pos, End: pos + n,
+ Locator: types.SourceLocator{
+ Type: types.SourceLocatorTime,
+ StartMs: int64(seg.Start * 1000),
+ EndMs: int64(seg.End * 1000),
+ },
+ })
+ pos += n
+ }
+ if len(blocks) == 0 {
+ return result.Text, nil
+ }
+ return sb.String(), blocks
+}
+
+// buildSourceIndex re-aims the parser's source blocks at the markdown the
+// chunker will split, and places every stored image in the original file.
+// parsed is the markdown the blocks were recorded against.
+func buildSourceIndex(
+ parsed string, blocks []types.SourceBlock, final string, images []docparser.StoredImage,
+) *sourceloc.Index {
+ if len(blocks) == 0 {
+ return nil
+ }
+ idx := sourceloc.NewIndex(final, sourceloc.RemapBlocks(blocks, parsed, final))
+ if idx == nil {
+ return nil
+ }
+ // Visit the images in document order so rune offsets are counted in one
+ // pass over the markdown instead of once per image.
+ type placed struct{ image, at int }
+ var found []placed
+ for i := range images {
+ if images[i].ServingURL == "" {
+ continue
+ }
+ if at := strings.Index(final, images[i].ServingURL); at >= 0 {
+ found = append(found, placed{image: i, at: at})
+ }
+ }
+ sort.Slice(found, func(a, b int) bool { return found[a].at < found[b].at })
+ bytePos, runePos := 0, 0
+ for _, f := range found {
+ runePos += utf8.RuneCountInString(final[bytePos:f.at])
+ bytePos = f.at
+ images[f.image].SourceLocators = idx.LocatorsAt(runePos)
+ }
+ return idx
+}
diff --git a/internal/application/service/knowledge_source_locators_test.go b/internal/application/service/knowledge_source_locators_test.go
new file mode 100644
index 000000000..c2137daca
--- /dev/null
+++ b/internal/application/service/knowledge_source_locators_test.go
@@ -0,0 +1,80 @@
+package service
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/Tencent/WeKnora/internal/infrastructure/docparser"
+ "github.com/Tencent/WeKnora/internal/models/asr"
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+func TestAttachStructureBlocksForPassThroughText(t *testing.T) {
+ content := "# 标题\n\n正文一段\n\n正文二段"
+ result := &types.ReadResult{MarkdownContent: content}
+ attachStructureBlocks(context.Background(), "md", []byte(content), result)
+ require.Len(t, result.SourceBlocks, 3)
+ assert.Equal(t, types.SourceLocatorText, result.SourceBlocks[1].Locator.Type)
+
+ // A parser that rewrote the text gets no text locators.
+ rewritten := &types.ReadResult{MarkdownContent: "different"}
+ attachStructureBlocks(context.Background(), "txt", []byte(content), rewritten)
+ assert.Empty(t, rewritten.SourceBlocks)
+
+ // Blocks a parser already reported are kept.
+ kept := &types.ReadResult{MarkdownContent: content, SourceBlocks: []types.SourceBlock{{Start: 0, End: 1}}}
+ attachStructureBlocks(context.Background(), "md", []byte(content), kept)
+ assert.Len(t, kept.SourceBlocks, 1)
+
+ // Viewers decode without the BOM, so text offsets start after it while
+ // the blocks still cover the markdown that holds it.
+ withBOM := "\ufeff正文一段\n\n正文二段"
+ bom := &types.ReadResult{MarkdownContent: withBOM}
+ attachStructureBlocks(context.Background(), "txt", []byte(withBOM), bom)
+ require.Len(t, bom.SourceBlocks, 2)
+ assert.Equal(t, 1, bom.SourceBlocks[0].Start)
+ assert.Equal(t, 0, bom.SourceBlocks[0].Locator.Start)
+ assert.Equal(t, 6, bom.SourceBlocks[1].Locator.Start)
+ assert.Equal(t, 7, bom.SourceBlocks[1].Start)
+}
+
+func TestTranscriptWithSegments(t *testing.T) {
+ text, blocks := transcriptWithSegments(&asr.TranscriptionResult{
+ Text: "你好世界。再见。",
+ Segments: []asr.Segment{
+ {Start: 0, End: 1.5, Text: " 你好世界。"},
+ {Start: 1.5, End: 2.25, Text: "再见。"},
+ },
+ })
+ assert.Equal(t, "你好世界。\n再见。", text)
+ require.Len(t, blocks, 2)
+ assert.Equal(t, 6, blocks[1].Start)
+ assert.Equal(t, int64(1500), blocks[1].Locator.StartMs)
+ assert.Equal(t, int64(2250), blocks[1].Locator.EndMs)
+
+ text, blocks = transcriptWithSegments(&asr.TranscriptionResult{Text: "plain"})
+ assert.Equal(t, "plain", text)
+ assert.Nil(t, blocks)
+}
+
+func TestBuildSourceIndexFollowsRewritesAndPlacesImages(t *testing.T) {
+ parsed := "第一页文字\r\n\r\n![p2](images/p2.jpg)"
+ final := "第一页文字\n\n![p2](local://tenant/p2.jpg)"
+ blocks := []types.SourceBlock{
+ {Start: 0, End: 5, Locator: types.SourceLocator{Type: types.SourceLocatorPDF, Page: 1}},
+ {Start: 9, End: len([]rune(parsed)), Locator: types.SourceLocator{Type: types.SourceLocatorPDF, Page: 2}},
+ }
+ images := []docparser.StoredImage{{ServingURL: "local://tenant/p2.jpg"}}
+ idx := buildSourceIndex(parsed, blocks, final, images)
+ require.NotNil(t, idx)
+ locs := idx.Locators(0, 5)
+ require.Len(t, locs, 1)
+ assert.Equal(t, "第一页文字", locs[0].Quote)
+ require.Len(t, images[0].SourceLocators, 1)
+ assert.Equal(t, 2, images[0].SourceLocators[0].Page)
+
+ assert.Nil(t, buildSourceIndex("x", nil, "x", nil))
+}
diff --git a/internal/application/service/knowledgebase_search_results.go b/internal/application/service/knowledgebase_search_results.go
index 6930e3cd2..096853eac 100644
--- a/internal/application/service/knowledgebase_search_results.go
+++ b/internal/application/service/knowledgebase_search_results.go
@@ -359,6 +359,7 @@ func (s *knowledgeBaseService) buildSearchResult(chunk *types.Chunk,
ContextHeader: chunk.ContextHeader,
MatchedContent: matchedContent,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
+ SourceLocators: chunk.SourceLocators,
}
}
diff --git a/internal/database/migration_sqlite_versioned_schema_test.go b/internal/database/migration_sqlite_versioned_schema_test.go
index 073e53037..2e60e8ced 100644
--- a/internal/database/migration_sqlite_versioned_schema_test.go
+++ b/internal/database/migration_sqlite_versioned_schema_test.go
@@ -56,6 +56,7 @@ var versionedSQLiteColumns = map[string][]string{
"tenant_invitations": {"token", "accepted_count"}, // 000054
"embed_channels": {"allow_memory"}, // 000060
"im_channels": {"locale"}, // 000030
+ "chunks": {"source_locators"}, // 000033
"mcp_oauth_tokens": {"principal_type", "principal_id"}, // 000064
"mcp_tool_approvals": {"enabled"}, // 000091
"message_artifacts": {"deleted_at"}, // 000107
@@ -68,7 +69,7 @@ var versionedSQLiteColumns = map[string][]string{
}, // 000028
}
-const expectedSQLiteMigrationVersion = 32
+const expectedSQLiteMigrationVersion = 33
func TestSQLiteMigrationsCreateVersionedSchema(t *testing.T) {
repoRoot := sqliteRepoRoot(t)
diff --git a/internal/handler/session/helpers.go b/internal/handler/session/helpers.go
index c33b29dba..ca08215e6 100644
--- a/internal/handler/session/helpers.go
+++ b/internal/handler/session/helpers.go
@@ -2,6 +2,7 @@ package session
import (
"context"
+ "encoding/json"
"fmt"
"strings"
"time"
@@ -488,6 +489,14 @@ func searchResultFromMap(refMap map[string]interface{}) *types.SearchResult {
}
sr.Metadata = metadata
}
+ if raw, ok := refMap["source_locators"]; ok && raw != nil {
+ if b, err := json.Marshal(raw); err == nil {
+ var locators types.SourceLocators
+ if json.Unmarshal(b, &locators) == nil {
+ sr.SourceLocators = locators
+ }
+ }
+ }
return sr
}
diff --git a/internal/handler/session/helpers_test.go b/internal/handler/session/helpers_test.go
index f9537da12..709369ba7 100644
--- a/internal/handler/session/helpers_test.go
+++ b/internal/handler/session/helpers_test.go
@@ -79,6 +79,9 @@ func TestSearchResultFromMap_RoundTrip(t *testing.T) {
KnowledgeDescription: "desc",
KnowledgeBaseID: "kb-1",
Metadata: map[string]string{"page": "3"},
+ SourceLocators: types.SourceLocators{
+ {Type: types.SourceLocatorPDF, Page: 4, BBox: []float64{0.1, 0.2, 0.9, 0.3}, Quote: "q"},
+ },
}
raw, err := json.Marshal(original)
@@ -105,6 +108,7 @@ func TestSearchResultFromMap_RoundTrip(t *testing.T) {
assert.Equal(t, original.KnowledgeDescription, got.KnowledgeDescription)
assert.Equal(t, original.KnowledgeBaseID, got.KnowledgeBaseID)
assert.Equal(t, original.Metadata, got.Metadata)
+ assert.Equal(t, original.SourceLocators, got.SourceLocators)
}
func TestCreateAgentQueryEventIncludesPersistedTimestamps(t *testing.T) {
diff --git a/internal/infrastructure/docparser/grpc_parser.go b/internal/infrastructure/docparser/grpc_parser.go
index 9c19ae24c..ac6e6ccd1 100644
--- a/internal/infrastructure/docparser/grpc_parser.go
+++ b/internal/infrastructure/docparser/grpc_parser.go
@@ -174,6 +174,7 @@ func (p *GRPCDocumentReader) readStream(
result.ImageDirPath = meta.GetImageDirPath()
result.Metadata = meta.GetMetadata()
result.Error = meta.GetError()
+ result.SourceBlocks = sourceBlocksFromProto(meta.GetSourceBlocks())
if n := meta.GetImageCount(); n > 0 {
result.ImageRefs = make([]types.ImageRef, 0, n)
}
@@ -212,6 +213,7 @@ func (p *GRPCDocumentReader) readUnary(
ImageDirPath: resp.GetImageDirPath(),
Metadata: resp.GetMetadata(),
Error: resp.GetError(),
+ SourceBlocks: sourceBlocksFromProto(resp.GetSourceBlocks()),
}
if refs := resp.GetImageRefs(); len(refs) > 0 {
result.ImageRefs = make([]types.ImageRef, 0, len(refs))
diff --git a/internal/infrastructure/docparser/http_parser.go b/internal/infrastructure/docparser/http_parser.go
index aeca68e60..218fc8451 100644
--- a/internal/infrastructure/docparser/http_parser.go
+++ b/internal/infrastructure/docparser/http_parser.go
@@ -53,6 +53,7 @@ type httpReadResponse struct {
ImageDirPath string `json:"image_dir_path,omitempty"`
Metadata map[string]string `json:"metadata,omitempty"`
Error string `json:"error,omitempty"`
+ SourceBlocks []wireSourceBlock `json:"source_blocks,omitempty"`
}
// HTTPDocumentReader implements DocumentReader over HTTP/JSON.
@@ -180,6 +181,7 @@ func fromHTTPReadResponse(resp *httpReadResponse) *types.ReadResult {
ImageDirPath: resp.ImageDirPath,
Metadata: resp.Metadata,
Error: resp.Error,
+ SourceBlocks: sourceBlocksFromWire(resp.SourceBlocks),
}
for _, ref := range resp.ImageRefs {
result.ImageRefs = append(result.ImageRefs, types.ImageRef{
diff --git a/internal/infrastructure/docparser/image_resolver.go b/internal/infrastructure/docparser/image_resolver.go
index e8db4297e..a4631a07b 100644
--- a/internal/infrastructure/docparser/image_resolver.go
+++ b/internal/infrastructure/docparser/image_resolver.go
@@ -60,6 +60,9 @@ type StoredImage struct {
OriginalRef string // reference in the original markdown
ServingURL string // provider:// URL (e.g. local://images/xxx.png, minio://bucket/key)
MimeType string
+ // SourceLocators place the image in the original file (e.g. the scanned
+ // page it was rendered from). Filled by ingestion after resolution.
+ SourceLocators types.SourceLocators
}
// ImageResolver reads images from a DocReader ReadResult (inline bytes only)
diff --git a/internal/infrastructure/docparser/mineru_cloud_converter.go b/internal/infrastructure/docparser/mineru_cloud_converter.go
index d3993c965..b2aa820e7 100644
--- a/internal/infrastructure/docparser/mineru_cloud_converter.go
+++ b/internal/infrastructure/docparser/mineru_cloud_converter.go
@@ -85,7 +85,7 @@ func (c *MinerUCloudReader) Read(ctx context.Context, req *types.ReadRequest) (*
return nil, fmt.Errorf("MinerU Cloud file upload: %w", err)
}
- mdContent, imageRefs, err := c.pollBatchResult(ctx, batchID)
+ mdContent, imageRefs, contentList, err := c.pollBatchResult(ctx, batchID)
if err != nil {
return nil, fmt.Errorf("MinerU Cloud poll: %w", err)
}
@@ -95,6 +95,7 @@ func (c *MinerUCloudReader) Read(ctx context.Context, req *types.ReadRequest) (*
return &types.ReadResult{
MarkdownContent: mdContent,
ImageRefs: imageRefs,
+ SourceBlocks: minerUSourceBlocks(mdContent, contentList, req.FileType),
}, nil
}
@@ -207,7 +208,9 @@ type extractResultItem struct {
FullZipURL string `json:"full_zip_url"`
}
-func (c *MinerUCloudReader) pollBatchResult(ctx context.Context, batchID string) (string, []types.ImageRef, error) {
+func (c *MinerUCloudReader) pollBatchResult(
+ ctx context.Context, batchID string,
+) (string, []types.ImageRef, []byte, error) {
deadline := time.Now().Add(defaultCloudTimeout)
pollCount := 0
headers := map[string]string{
@@ -220,7 +223,7 @@ func (c *MinerUCloudReader) pollBatchResult(ctx context.Context, batchID string)
// cancelled ctx, so without this guard the loop busy-hammers the cloud
// API and floods logs until the deadline.
if err := ctx.Err(); err != nil {
- return "", nil, err
+ return "", nil, nil, err
}
pollCount++
@@ -248,7 +251,7 @@ func (c *MinerUCloudReader) pollBatchResult(ctx context.Context, batchID string)
}
if state == "failed" {
- return "", nil, fmt.Errorf("MinerU Cloud task failed: %s", item.ErrMsg)
+ return "", nil, nil, fmt.Errorf("MinerU Cloud task failed: %s", item.ErrMsg)
}
if state == "done" {
@@ -258,7 +261,7 @@ func (c *MinerUCloudReader) pollBatchResult(ctx context.Context, batchID string)
sleepCtx(ctx, defaultPollInterval)
}
- return "", nil, fmt.Errorf("MinerU Cloud task timed out after %d polls", pollCount)
+ return "", nil, nil, fmt.Errorf("MinerU Cloud task timed out after %d polls", pollCount)
}
func (c *MinerUCloudReader) fetchBatchStatus(ctx context.Context, batchID string, headers map[string]string) ([]extractResultItem, error) {
@@ -328,47 +331,54 @@ func (c *MinerUCloudReader) fetchBatchStatus(ctx context.Context, batchID string
// extractDoneResult extracts markdown and images from a completed batch item.
// Prefers inline markdown/content fields; falls back to downloading full_zip_url.
-func (c *MinerUCloudReader) extractDoneResult(_ context.Context, item *extractResultItem) (string, []types.ImageRef, error) {
+func (c *MinerUCloudReader) extractDoneResult(
+ _ context.Context, item *extractResultItem,
+) (string, []types.ImageRef, []byte, error) {
text := firstNonEmpty(item.Markdown, item.Content, item.Text)
if text != "" {
logger.Infof(context.Background(), "[MinerUCloud] parsed (inline), length=%d", len(text))
- return text, nil, nil
+ return text, nil, nil, nil
}
if item.FullZipURL == "" {
- return "", nil, fmt.Errorf("MinerU Cloud state=done but no markdown/content or full_zip_url")
+ return "", nil, nil, fmt.Errorf("MinerU Cloud state=done but no markdown/content or full_zip_url")
}
- md, imageRefs, err := downloadAndExtractZip(item.FullZipURL)
+ md, imageRefs, contentList, err := downloadAndExtractZip(item.FullZipURL)
if err != nil {
- return "", nil, fmt.Errorf("extract zip: %w", err)
+ return "", nil, nil, fmt.Errorf("extract zip: %w", err)
}
logger.Infof(context.Background(), "[MinerUCloud] parsed (zip), markdown=%d chars, images=%d", len(md), len(imageRefs))
- return md, imageRefs, nil
+ return md, imageRefs, contentList, nil
}
// --- ZIP handling ---
-func downloadAndExtractZip(zipURL string) (string, []types.ImageRef, error) {
+// downloadAndExtractZip also returns the content_list of the package, if any.
+func downloadAndExtractZip(zipURL string) (string, []types.ImageRef, []byte, error) {
if err := utils.ValidateURLForSSRF(zipURL); err != nil {
- return "", nil, fmt.Errorf("zip URL blocked by SSRF check: %v", err)
+ return "", nil, nil, fmt.Errorf("zip URL blocked by SSRF check: %v", err)
}
client := utils.NewSSRFSafeHTTPClient(utils.SSRFSafeHTTPClientConfig{Timeout: 120 * time.Second, MaxRedirects: 5})
resp, err := client.Get(zipURL)
if err != nil {
- return "", nil, fmt.Errorf("download zip: %w", err)
+ return "", nil, nil, fmt.Errorf("download zip: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
- return "", nil, fmt.Errorf("download zip status %d", resp.StatusCode)
+ return "", nil, nil, fmt.Errorf("download zip status %d", resp.StatusCode)
}
zipData, err := io.ReadAll(resp.Body)
if err != nil {
- return "", nil, fmt.Errorf("read zip body: %w", err)
+ return "", nil, nil, fmt.Errorf("read zip body: %w", err)
}
- return extractMarkdownZip(zipData, "MinerUCloud")
+ md, imageRefs, err := extractMarkdownZip(zipData, "MinerUCloud")
+ if err != nil {
+ return "", nil, nil, err
+ }
+ return md, imageRefs, minerUContentListFromZip(zipData), nil
}
// extractMarkdownZip reads the shallowest .md file of a MinerU result package
diff --git a/internal/infrastructure/docparser/mineru_converter.go b/internal/infrastructure/docparser/mineru_converter.go
index ead9ac5d2..71db1c587 100644
--- a/internal/infrastructure/docparser/mineru_converter.go
+++ b/internal/infrastructure/docparser/mineru_converter.go
@@ -113,7 +113,7 @@ func (c *MinerUReader) Read(ctx context.Context, req *types.ReadRequest) (*types
logger.Infof(ctx, "[MinerU] Parsing file=%s size=%d via %s (legacy /file_parse)",
req.FileName, len(content), c.endpoint)
- mdContent, imagesB64, err := c.callFileParse(ctx, content, req.FileName, req.FileType)
+ mdContent, imagesB64, contentList, err := c.callFileParse(ctx, content, req.FileName, req.FileType)
if err != nil {
return nil, fmt.Errorf("MinerU file_parse: %w", err)
}
@@ -133,12 +133,14 @@ func (c *MinerUReader) Read(ctx context.Context, req *types.ReadRequest) (*types
return &types.ReadResult{
MarkdownContent: mdContent,
ImageRefs: imageRefs,
+ SourceBlocks: minerUSourceBlocks(mdContent, contentList, req.FileType),
}, nil
}
func (c *MinerUReader) readV1(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) {
client := newMinerUV1Client(c.endpoint, c.apiKey, mineruTimeout)
- mdContent, imageRefs, err := client.Parse(ctx, req.FileContent, minerUUploadFileName(req.FileName, req.FileType),
+ mdContent, imageRefs, contentList, err := client.ParseWithLayout(ctx, req.FileContent,
+ minerUUploadFileName(req.FileName, req.FileType),
minerUV1ParseOptions{Tier: c.tier, OCRMode: c.parseMethod})
if err != nil {
return nil, fmt.Errorf("MinerU V1: %w", err)
@@ -152,12 +154,16 @@ func (c *MinerUReader) readV1(ctx context.Context, req *types.ReadRequest) (*typ
return &types.ReadResult{
MarkdownContent: mdContent,
ImageRefs: imageRefs,
+ SourceBlocks: minerUSourceBlocks(mdContent, contentList, req.FileType),
}, nil
}
type mineruFileEntry struct {
MDContent string `json:"md_content"`
Images map[string]string `json:"images"` // path -> "data:image/png;base64,..." or raw base64
+ // ContentList is the layout block list (page and box per block), as a
+ // JSON array or a string holding one depending on the MinerU version.
+ ContentList json.RawMessage `json:"content_list"`
}
func minerUCleanFileType(fileType string) string {
@@ -239,12 +245,24 @@ func parseMinerUFileParseResponse(respBody []byte, uploadFileName string) (strin
return "", nil, "", nil
}
+// minerUFileParseContentList returns the content_list of the result entry
+// that supplied the markdown.
+func minerUFileParseContentList(respBody []byte, resultKey string) json.RawMessage {
+ var envelope struct {
+ Results map[string]mineruFileEntry `json:"results"`
+ }
+ if json.Unmarshal(respBody, &envelope) != nil {
+ return nil
+ }
+ return envelope.Results[resultKey].ContentList
+}
+
func (c *MinerUReader) callFileParse(
ctx context.Context,
content []byte,
fileName string,
fileType string,
-) (string, map[string]string, error) {
+) (string, map[string]string, json.RawMessage, error) {
var body bytes.Buffer
writer := multipart.NewWriter(&body)
@@ -278,16 +296,16 @@ func (c *MinerUReader) callFileParse(
// File part
part, err := writer.CreateFormFile("files", uploadFileName)
if err != nil {
- return "", nil, fmt.Errorf("create form file: %w", err)
+ return "", nil, nil, fmt.Errorf("create form file: %w", err)
}
if _, err := part.Write(content); err != nil {
- return "", nil, fmt.Errorf("write file content: %w", err)
+ return "", nil, nil, fmt.Errorf("write file content: %w", err)
}
writer.Close()
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, c.endpoint+"/file_parse", &body)
if err != nil {
- return "", nil, fmt.Errorf("create request: %w", err)
+ return "", nil, nil, fmt.Errorf("create request: %w", err)
}
httpReq.Header.Set("Content-Type", writer.FormDataContentType())
@@ -297,18 +315,18 @@ func (c *MinerUReader) callFileParse(
})
resp, err := client.Do(httpReq)
if err != nil {
- return "", nil, fmt.Errorf("HTTP request: %w", err)
+ return "", nil, nil, fmt.Errorf("HTTP request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
respBody, _ := io.ReadAll(resp.Body)
- return "", nil, fmt.Errorf("MinerU API status %d: %s", resp.StatusCode, string(respBody))
+ return "", nil, nil, fmt.Errorf("MinerU API status %d: %s", resp.StatusCode, string(respBody))
}
respBody, err := io.ReadAll(resp.Body)
if err != nil {
- return "", nil, fmt.Errorf("read response body: %w", err)
+ return "", nil, nil, fmt.Errorf("read response body: %w", err)
}
// Dump raw response for debugging (truncate if too large)
@@ -327,15 +345,15 @@ func (c *MinerUReader) callFileParse(
mdContent, imagesB64, resultKey, err := parseMinerUFileParseResponse(respBody, uploadFileName)
if err != nil {
- return "", nil, err
+ return "", nil, nil, err
}
if resultKey != "" {
logger.Infof(context.Background(), "[MinerU] Using response path: results.%s", resultKey)
- return mdContent, imagesB64, nil
+ return mdContent, imagesB64, minerUFileParseContentList(respBody, resultKey), nil
}
logger.Errorf(context.Background(), "[MinerU] Response has no markdown/images under results")
- return "", nil, nil
+ return "", nil, nil, nil
}
// processImages decodes base64 images from MinerU response and returns ImageRef list.
diff --git a/internal/infrastructure/docparser/mineru_layout.go b/internal/infrastructure/docparser/mineru_layout.go
new file mode 100644
index 000000000..a461e5796
--- /dev/null
+++ b/internal/infrastructure/docparser/mineru_layout.go
@@ -0,0 +1,154 @@
+package docparser
+
+import (
+ "archive/zip"
+ "bytes"
+ "encoding/json"
+ "regexp"
+ "sort"
+ "strings"
+
+ "github.com/Tencent/WeKnora/internal/sourceloc"
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+// minerUBBoxScale is the coordinate range of content_list bounding boxes:
+// MinerU normalizes them to 0-1000 of the page, origin at the top-left.
+const minerUBBoxScale = 1000.0
+
+// minerUContentItem is one layout block of MinerU's content_list output.
+type minerUContentItem struct {
+ Type string `json:"type"`
+ Text string `json:"text"`
+ PageIdx int `json:"page_idx"`
+ BBox []float64 `json:"bbox"`
+ TableBody string `json:"table_body"`
+ TableCaption json.RawMessage `json:"table_caption"`
+ ImageCaption json.RawMessage `json:"image_caption"`
+ ListItems json.RawMessage `json:"list_items"`
+ CodeBody string `json:"code_body"`
+}
+
+var minerUHTMLTagRe = regexp.MustCompile(`<[^>]+>`)
+
+// text is the block's searchable text, in the order MinerU renders it to
+// markdown: captions above tables and figures, then the body.
+func (it minerUContentItem) text() string {
+ parts := []string{it.Text}
+ parts = append(parts, rawStrings(it.TableCaption)...)
+ parts = append(parts, rawStrings(it.ImageCaption)...)
+ if it.TableBody != "" {
+ parts = append(parts, minerUHTMLTagRe.ReplaceAllString(it.TableBody, " "))
+ }
+ parts = append(parts, it.CodeBody)
+ parts = append(parts, rawStrings(it.ListItems)...)
+ return strings.TrimSpace(strings.Join(parts, " "))
+}
+
+// rawStrings reads a JSON string or array of strings.
+func rawStrings(raw json.RawMessage) []string {
+ if len(raw) == 0 {
+ return nil
+ }
+ var list []string
+ if json.Unmarshal(raw, &list) == nil {
+ return list
+ }
+ var one string
+ if json.Unmarshal(raw, &one) == nil {
+ return []string{one}
+ }
+ return nil
+}
+
+// decodeMinerUContentList accepts the list itself or the list serialized as
+// a JSON string, which is how some MinerU API versions return it.
+func decodeMinerUContentList(raw []byte) []minerUContentItem {
+ raw = bytes.TrimSpace(raw)
+ if len(raw) == 0 {
+ return nil
+ }
+ if raw[0] == '"' {
+ var inner string
+ if json.Unmarshal(raw, &inner) != nil {
+ return nil
+ }
+ raw = []byte(inner)
+ }
+ var items []minerUContentItem
+ if json.Unmarshal(raw, &items) != nil {
+ return nil
+ }
+ return items
+}
+
+// minerUContentListFromZip returns the shallowest *content_list.json of a
+// MinerU result package, or nil.
+func minerUContentListFromZip(zipData []byte) []byte {
+ zr, err := zip.NewReader(bytes.NewReader(zipData), int64(len(zipData)))
+ if err != nil {
+ return nil
+ }
+ var candidates []*zip.File
+ for _, f := range zr.File {
+ name := strings.ToLower(f.Name)
+ if strings.HasSuffix(name, "content_list.json") && !strings.Contains(name, "content_list_v2") {
+ candidates = append(candidates, f)
+ }
+ }
+ if len(candidates) == 0 {
+ return nil
+ }
+ sort.Slice(candidates, func(i, j int) bool {
+ return strings.Count(candidates[i].Name, "/") < strings.Count(candidates[j].Name, "/")
+ })
+ data, err := readZipEntryBytes(candidates[0])
+ if err != nil {
+ return nil
+ }
+ return data
+}
+
+// minerUSourceBlocks aligns MinerU's layout blocks against its markdown so
+// each paragraph, table and figure points back at its page and region. Only
+// PDFs and images have pages a viewer can show; for office files MinerU's
+// pages belong to an intermediate PDF, so those are left to the structure
+// aligner.
+func minerUSourceBlocks(markdown string, contentList []byte, fileType string) []types.SourceBlock {
+ ft := strings.ToLower(strings.TrimPrefix(fileType, "."))
+ if ft != "pdf" && !IsImageFormat(ft) {
+ return nil
+ }
+ items := decodeMinerUContentList(contentList)
+ if len(items) == 0 {
+ return nil
+ }
+ units := make([]sourceloc.Unit, 0, len(items))
+ for _, it := range items {
+ loc := types.SourceLocator{Type: types.SourceLocatorPDF, Page: it.PageIdx + 1}
+ if bbox, ok := minerUBBox(it.BBox); ok {
+ loc.BBox = bbox
+ }
+ units = append(units, sourceloc.Unit{Text: it.text(), Locator: loc})
+ }
+ return sourceloc.Align(markdown, units)
+}
+
+// minerUBBox converts a 0-1000 content_list box to page fractions. Boxes
+// outside that range come from MinerU versions that report other units and
+// are dropped rather than drawn in the wrong place.
+func minerUBBox(b []float64) ([]float64, bool) {
+ if len(b) != 4 || b[2] <= b[0] || b[3] <= b[1] {
+ return nil, false
+ }
+ for _, v := range b {
+ if v < 0 || v > minerUBBoxScale {
+ return nil, false
+ }
+ }
+ out := make([]float64, 4)
+ for i, v := range b {
+ out[i] = v / minerUBBoxScale
+ }
+ return out, true
+}
diff --git a/internal/infrastructure/docparser/mineru_v1_client.go b/internal/infrastructure/docparser/mineru_v1_client.go
index 0eae3915e..232d4c501 100644
--- a/internal/infrastructure/docparser/mineru_v1_client.go
+++ b/internal/infrastructure/docparser/mineru_v1_client.go
@@ -214,17 +214,29 @@ func (c *minerUV1Client) Parse(
fileName string,
opts minerUV1ParseOptions,
) (string, []types.ImageRef, error) {
+ md, refs, _, err := c.ParseWithLayout(ctx, content, fileName, opts)
+ return md, refs, err
+}
+
+// ParseWithLayout is Parse plus the package's content_list, the per-block
+// page and box layout (nil when the package carried none).
+func (c *minerUV1Client) ParseWithLayout(
+ ctx context.Context,
+ content []byte,
+ fileName string,
+ opts minerUV1ParseOptions,
+) (string, []types.ImageRef, []byte, error) {
ctx, cancel := context.WithTimeout(ctx, c.jobTimeout)
defer cancel()
fileID, err := c.upload(ctx, content, fileName)
if err != nil {
- return "", nil, fmt.Errorf("upload: %w", err)
+ return "", nil, nil, fmt.Errorf("upload: %w", err)
}
jobID, err := c.createJob(ctx, fileID, opts)
if err != nil {
- return "", nil, fmt.Errorf("create parse job: %w", err)
+ return "", nil, nil, fmt.Errorf("create parse job: %w", err)
}
logger.Infof(ctx, "[%s] parse job created: job_id=%s file_id=%s tier=%q ocr_mode=%s",
c.logLabel, jobID, fileID, opts.Tier, opts.OCRMode)
@@ -232,24 +244,24 @@ func (c *minerUV1Client) Parse(
job, err := c.waitJob(ctx, jobID)
if err != nil {
c.cancelJob(jobID)
- return "", nil, err
+ return "", nil, nil, err
}
zipRef, err := minerUV1ZipOutput(job)
if err != nil {
- return "", nil, err
+ return "", nil, nil, err
}
zipData, err := c.downloadFile(ctx, zipRef.FileID)
if err != nil {
- return "", nil, fmt.Errorf("download zip artifact: %w", err)
+ return "", nil, nil, fmt.Errorf("download zip artifact: %w", err)
}
md, imageRefs, err := extractMarkdownZip(zipData, c.logLabel)
if err != nil {
- return "", nil, fmt.Errorf("extract zip artifact: %w", err)
+ return "", nil, nil, fmt.Errorf("extract zip artifact: %w", err)
}
- return md, imageRefs, nil
+ return md, imageRefs, minerUContentListFromZip(zipData), nil
}
func (c *minerUV1Client) upload(ctx context.Context, content []byte, fileName string) (string, error) {
diff --git a/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go b/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go
index c05c9561a..3546a7e96 100644
--- a/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go
+++ b/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go
@@ -14,6 +14,7 @@ import (
"time"
"github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/sourceloc"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/utils"
)
@@ -73,7 +74,7 @@ func (c *PaddleOCRVLCloudReader) Read(ctx context.Context, req *types.ReadReques
return nil, fmt.Errorf("PaddleOCR-VL Cloud poll: %w", err)
}
- mdContent, imagesURL, err := c.fetchResults(jsonlURL)
+ mdContent, imagesURL, layoutUnits, err := c.fetchResults(jsonlURL)
if err != nil {
return nil, fmt.Errorf("PaddleOCR-VL Cloud fetch results: %w", err)
}
@@ -89,6 +90,7 @@ func (c *PaddleOCRVLCloudReader) Read(ctx context.Context, req *types.ReadReques
return &types.ReadResult{
MarkdownContent: mdContent,
ImageRefs: imageRefs,
+ SourceBlocks: paddleOCRVLSourceBlocks(mdContent, layoutUnits, req.FileType),
}, nil
}
@@ -252,35 +254,32 @@ func (c *PaddleOCRVLCloudReader) pollJob(ctx context.Context, jobID string) (str
type paddleOCRVLCloudResultLine struct {
Result struct {
- LayoutParsingResults []struct {
- Markdown struct {
- Text string `json:"text"`
- Images map[string]string `json:"images"`
- } `json:"markdown"`
- } `json:"layoutParsingResults"`
+ LayoutParsingResults []paddleOCRVLPage `json:"layoutParsingResults"`
} `json:"result"`
}
-func (c *PaddleOCRVLCloudReader) fetchResults(jsonlURL string) (string, map[string]string, error) {
+func (c *PaddleOCRVLCloudReader) fetchResults(jsonlURL string) (string, map[string]string, []sourceloc.Unit, error) {
if err := utils.ValidateURLForSSRF(jsonlURL); err != nil {
- return "", nil, fmt.Errorf("jsonl URL blocked by SSRF check: %v", err)
+ return "", nil, nil, fmt.Errorf("jsonl URL blocked by SSRF check: %v", err)
}
client := utils.NewSSRFSafeHTTPClient(utils.SSRFSafeHTTPClientConfig{Timeout: 120 * time.Second, MaxRedirects: 5})
resp, err := client.Get(jsonlURL)
if err != nil {
- return "", nil, fmt.Errorf("download jsonl: %w", err)
+ return "", nil, nil, fmt.Errorf("download jsonl: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
- return "", nil, fmt.Errorf("download jsonl status %d", resp.StatusCode)
+ return "", nil, nil, fmt.Errorf("download jsonl status %d", resp.StatusCode)
}
data, err := io.ReadAll(resp.Body)
if err != nil {
- return "", nil, fmt.Errorf("read jsonl body: %w", err)
+ return "", nil, nil, fmt.Errorf("read jsonl body: %w", err)
}
texts := make([]string, 0)
images := make(map[string]string)
+ var units []sourceloc.Unit
+ pageNumber := 0
for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") {
line = strings.TrimSpace(line)
if line == "" {
@@ -292,6 +291,8 @@ func (c *PaddleOCRVLCloudReader) fetchResults(jsonlURL string) (string, map[stri
continue
}
for _, p := range parsed.Result.LayoutParsingResults {
+ pageNumber++
+ units = append(units, paddleOCRVLPageUnits(pageNumber, p)...)
if t := strings.TrimSpace(p.Markdown.Text); t != "" {
texts = append(texts, p.Markdown.Text)
}
@@ -304,7 +305,7 @@ func (c *PaddleOCRVLCloudReader) fetchResults(jsonlURL string) (string, map[stri
}
logger.Infof(context.Background(), "[PaddleOCR-VL Cloud] fetched %d page(s), images=%d", len(texts), len(images))
- return strings.Join(texts, "\n\n"), images, nil
+ return strings.Join(texts, "\n\n"), images, units, nil
}
// downloadImages fetches each referenced image URL and builds ImageRef entries.
diff --git a/internal/infrastructure/docparser/paddleocr_vl_converter.go b/internal/infrastructure/docparser/paddleocr_vl_converter.go
index f16d1020d..a5546a3e6 100644
--- a/internal/infrastructure/docparser/paddleocr_vl_converter.go
+++ b/internal/infrastructure/docparser/paddleocr_vl_converter.go
@@ -15,6 +15,7 @@ import (
"time"
"github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/sourceloc"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/utils"
)
@@ -75,7 +76,7 @@ func (c *PaddleOCRVLReader) Read(ctx context.Context, req *types.ReadRequest) (*
logger.Infof(context.Background(), "[PaddleOCR-VL] Parsing file=%s size=%d via %s",
req.FileName, len(content), c.endpoint)
- mdContent, imagesB64, err := c.callLayoutParsing(ctx, req, content)
+ mdContent, imagesB64, layoutUnits, err := c.callLayoutParsing(ctx, req, content)
if err != nil {
return nil, fmt.Errorf("PaddleOCR-VL layout-parsing: %w", err)
}
@@ -95,6 +96,7 @@ func (c *PaddleOCRVLReader) Read(ctx context.Context, req *types.ReadRequest) (*
return &types.ReadResult{
MarkdownContent: mdContent,
ImageRefs: imageRefs,
+ SourceBlocks: paddleOCRVLSourceBlocks(mdContent, layoutUnits, req.FileType),
}, nil
}
@@ -149,18 +151,13 @@ type paddleOCRVLResponse struct {
ErrorCode int `json:"errorCode"`
ErrorMsg string `json:"errorMsg"`
Result struct {
- LayoutParsingResults []struct {
- Markdown struct {
- Text string `json:"text"`
- Images map[string]string `json:"images"`
- } `json:"markdown"`
- } `json:"layoutParsingResults"`
+ LayoutParsingResults []paddleOCRVLPage `json:"layoutParsingResults"`
} `json:"result"`
}
func (c *PaddleOCRVLReader) callLayoutParsing(
ctx context.Context, req *types.ReadRequest, content []byte,
-) (string, map[string]string, error) {
+) (string, map[string]string, []sourceloc.Unit, error) {
payload := paddleOCRVLRecognitionParams(c.useSeal, c.useChart)
payload["file"] = base64.StdEncoding.EncodeToString(content)
payload["fileType"] = fileTypeCode(req)
@@ -168,14 +165,14 @@ func (c *PaddleOCRVLReader) callLayoutParsing(
body, err := json.Marshal(payload)
if err != nil {
- return "", nil, fmt.Errorf("marshal payload: %w", err)
+ return "", nil, nil, fmt.Errorf("marshal payload: %w", err)
}
httpReq, err := http.NewRequestWithContext(
ctx, http.MethodPost, c.endpoint+"/layout-parsing", bytes.NewReader(body),
)
if err != nil {
- return "", nil, fmt.Errorf("create request: %w", err)
+ return "", nil, nil, fmt.Errorf("create request: %w", err)
}
httpReq.Header.Set("Content-Type", "application/json")
@@ -185,37 +182,39 @@ func (c *PaddleOCRVLReader) callLayoutParsing(
})
resp, err := client.Do(httpReq)
if err != nil {
- return "", nil, fmt.Errorf("HTTP request: %w", err)
+ return "", nil, nil, fmt.Errorf("HTTP request: %w", err)
}
defer resp.Body.Close()
respBody, err := io.ReadAll(resp.Body)
if err != nil {
- return "", nil, fmt.Errorf("read response body: %w", err)
+ return "", nil, nil, fmt.Errorf("read response body: %w", err)
}
if resp.StatusCode != http.StatusOK {
- return "", nil, fmt.Errorf("PaddleOCR-VL API status %d: %s", resp.StatusCode, string(respBody))
+ return "", nil, nil, fmt.Errorf("PaddleOCR-VL API status %d: %s", resp.StatusCode, string(respBody))
}
var result paddleOCRVLResponse
if err := json.Unmarshal(respBody, &result); err != nil {
- return "", nil, fmt.Errorf("decode response: %w", err)
+ return "", nil, nil, fmt.Errorf("decode response: %w", err)
}
if result.ErrorCode != 0 {
- return "", nil, fmt.Errorf("PaddleOCR-VL error %d: %s", result.ErrorCode, result.ErrorMsg)
+ return "", nil, nil, fmt.Errorf("PaddleOCR-VL error %d: %s", result.ErrorCode, result.ErrorMsg)
}
pages := result.Result.LayoutParsingResults
if len(pages) == 0 {
logger.Errorf(context.Background(), "[PaddleOCR-VL] response has no layoutParsingResults")
- return "", nil, nil
+ return "", nil, nil, nil
}
// Merge per-page markdown and image dicts into one document.
texts := make([]string, 0, len(pages))
images := make(map[string]string)
- for _, p := range pages {
+ var units []sourceloc.Unit
+ for i, p := range pages {
+ units = append(units, paddleOCRVLPageUnits(i+1, p)...)
if t := strings.TrimSpace(p.Markdown.Text); t != "" {
texts = append(texts, p.Markdown.Text)
}
@@ -227,7 +226,7 @@ func (c *PaddleOCRVLReader) callLayoutParsing(
}
logger.Infof(context.Background(), "[PaddleOCR-VL] parsed %d page(s), images=%d", len(pages), len(images))
- return strings.Join(texts, "\n\n"), images, nil
+ return strings.Join(texts, "\n\n"), images, units, nil
}
// processImages decodes the inline base64 images returned by PaddleOCR-VL and
diff --git a/internal/infrastructure/docparser/paddleocr_vl_layout.go b/internal/infrastructure/docparser/paddleocr_vl_layout.go
new file mode 100644
index 000000000..c4a6388c9
--- /dev/null
+++ b/internal/infrastructure/docparser/paddleocr_vl_layout.go
@@ -0,0 +1,82 @@
+package docparser
+
+import (
+ "encoding/json"
+ "strings"
+
+ "github.com/Tencent/WeKnora/internal/sourceloc"
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+// paddleOCRVLPage is one page of a PaddleOCR-VL layout-parsing result.
+type paddleOCRVLPage struct {
+ Markdown struct {
+ Text string `json:"text"`
+ Images map[string]string `json:"images"`
+ } `json:"markdown"`
+ // PrunedResult carries the page's layout blocks; decoded lazily and
+ // only for the fields source locators need.
+ PrunedResult json.RawMessage `json:"prunedResult"`
+}
+
+type paddleOCRVLLayout struct {
+ Width float64 `json:"width"`
+ Height float64 `json:"height"`
+ ParsingResList []struct {
+ BlockContent string `json:"block_content"`
+ BlockBBox []float64 `json:"block_bbox"`
+ } `json:"parsing_res_list"`
+}
+
+// paddleOCRVLPageUnits lists a page's layout blocks, each boxed in page
+// fractions, for aligning against the merged markdown. Without layout
+// blocks the page's markdown paragraphs are listed with the page alone.
+func paddleOCRVLPageUnits(pageNumber int, page paddleOCRVLPage) []sourceloc.Unit {
+ pageOnly := types.SourceLocator{Type: types.SourceLocatorPDF, Page: pageNumber}
+ var layout paddleOCRVLLayout
+ if len(page.PrunedResult) > 0 && json.Unmarshal(page.PrunedResult, &layout) == nil &&
+ layout.Width > 0 && layout.Height > 0 && len(layout.ParsingResList) > 0 {
+ units := make([]sourceloc.Unit, 0, len(layout.ParsingResList))
+ for _, b := range layout.ParsingResList {
+ loc := pageOnly
+ if len(b.BlockBBox) == 4 && b.BlockBBox[2] > b.BlockBBox[0] && b.BlockBBox[3] > b.BlockBBox[1] {
+ loc.BBox = []float64{
+ clampUnit(b.BlockBBox[0] / layout.Width), clampUnit(b.BlockBBox[1] / layout.Height),
+ clampUnit(b.BlockBBox[2] / layout.Width), clampUnit(b.BlockBBox[3] / layout.Height),
+ }
+ }
+ units = append(units, sourceloc.Unit{
+ Text: minerUHTMLTagRe.ReplaceAllString(b.BlockContent, " "),
+ Locator: loc,
+ })
+ }
+ return units
+ }
+ var units []sourceloc.Unit
+ for _, para := range strings.Split(page.Markdown.Text, "\n\n") {
+ if strings.TrimSpace(para) != "" {
+ units = append(units, sourceloc.Unit{Text: para, Locator: pageOnly})
+ }
+ }
+ return units
+}
+
+// paddleOCRVLSourceBlocks aligns layout units against the final markdown
+// for file types a viewer shows page by page.
+func paddleOCRVLSourceBlocks(markdown string, units []sourceloc.Unit, fileType string) []types.SourceBlock {
+ ft := strings.ToLower(strings.TrimPrefix(fileType, "."))
+ if ft != "pdf" && !IsImageFormat(ft) {
+ return nil
+ }
+ return sourceloc.Align(markdown, units)
+}
+
+func clampUnit(v float64) float64 {
+ if v < 0 {
+ return 0
+ }
+ if v > 1 {
+ return 1
+ }
+ return v
+}
diff --git a/internal/infrastructure/docparser/source_blocks.go b/internal/infrastructure/docparser/source_blocks.go
new file mode 100644
index 000000000..0c6a9ec73
--- /dev/null
+++ b/internal/infrastructure/docparser/source_blocks.go
@@ -0,0 +1,57 @@
+package docparser
+
+import (
+ "encoding/json"
+
+ "github.com/Tencent/WeKnora/docreader/proto"
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+// wireSourceBlock is the transport shape of a source block: a markdown range
+// plus the JSON-encoded locator, shared by the gRPC and HTTP docreaders.
+type wireSourceBlock struct {
+ Start uint32 `json:"start"`
+ End uint32 `json:"end"`
+ LocatorJSON string `json:"locator_json"`
+}
+
+// decodeSourceBlock converts one transported block, dropping it when the
+// range is empty or the locator cannot be decoded.
+func decodeSourceBlock(b wireSourceBlock) (types.SourceBlock, bool) {
+ if b.End <= b.Start || b.LocatorJSON == "" {
+ return types.SourceBlock{}, false
+ }
+ var loc types.SourceLocator
+ if err := json.Unmarshal([]byte(b.LocatorJSON), &loc); err != nil || loc.Type == "" {
+ return types.SourceBlock{}, false
+ }
+ return types.SourceBlock{Start: int(b.Start), End: int(b.End), Locator: loc}, true
+}
+
+func sourceBlocksFromProto(pbs []*proto.SourceBlock) []types.SourceBlock {
+ if len(pbs) == 0 {
+ return nil
+ }
+ out := make([]types.SourceBlock, 0, len(pbs))
+ for _, pb := range pbs {
+ if b, ok := decodeSourceBlock(wireSourceBlock{
+ Start: pb.GetStart(), End: pb.GetEnd(), LocatorJSON: pb.GetLocatorJson(),
+ }); ok {
+ out = append(out, b)
+ }
+ }
+ return out
+}
+
+func sourceBlocksFromWire(wire []wireSourceBlock) []types.SourceBlock {
+ if len(wire) == 0 {
+ return nil
+ }
+ out := make([]types.SourceBlock, 0, len(wire))
+ for _, w := range wire {
+ if b, ok := decodeSourceBlock(w); ok {
+ out = append(out, b)
+ }
+ }
+ return out
+}
diff --git a/internal/infrastructure/docparser/source_blocks_test.go b/internal/infrastructure/docparser/source_blocks_test.go
new file mode 100644
index 000000000..cda8db199
--- /dev/null
+++ b/internal/infrastructure/docparser/source_blocks_test.go
@@ -0,0 +1,92 @@
+package docparser
+
+import (
+ "archive/zip"
+ "bytes"
+ "encoding/json"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/Tencent/WeKnora/docreader/proto"
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+const minerUContentListJSON = `[
+ {"type":"text","text":"8.14.11 索夹与吊索施工应符合下列规定:","text_level":2,"page_idx":56,"bbox":[80,100,900,120]},
+ {"type":"text","text":"1 在满足施工需要的前提下,应减小猫道面层开孔面积。","page_idx":56,"bbox":[80,130,920,170]},
+ {"type":"table","table_body":"<table><tr><td>高度</td><td>12m</td></tr></table>",
+ "table_caption":["表 1"],"page_idx":57,"bbox":[50,10,950,2000]}
+]`
+
+func TestMinerUSourceBlocks(t *testing.T) {
+ md := "## 8.14.11 索夹与吊索施工应符合下列规定:\n\n1 在满足施工需要的前提下,应减小猫道面层开孔面积。\n\n表 1\n\n| 高度 | 12m |\n| --- | --- |\n"
+ for name, raw := range map[string][]byte{
+ "array": []byte(minerUContentListJSON),
+ "string": mustJSON(t, minerUContentListJSON),
+ } {
+ t.Run(name, func(t *testing.T) {
+ blocks := minerUSourceBlocks(md, raw, "pdf")
+ require.Len(t, blocks, 3)
+ assert.Equal(t, 57, blocks[0].Locator.Page)
+ assert.InDeltaSlice(t, []float64{0.08, 0.1, 0.9, 0.12}, blocks[0].Locator.BBox, 1e-9)
+ assert.Equal(t, 58, blocks[2].Locator.Page)
+ assert.Nil(t, blocks[2].Locator.BBox, "out-of-range boxes are dropped")
+ })
+ }
+ assert.Nil(t, minerUSourceBlocks(md, []byte(minerUContentListJSON), "docx"))
+}
+
+func TestMinerUContentListFromZip(t *testing.T) {
+ var buf bytes.Buffer
+ zw := zip.NewWriter(&buf)
+ for name, body := range map[string]string{
+ "doc/doc.md": "x",
+ "doc/doc_content_list.json": "[1]",
+ "doc/auto/doc_content_list.json": "[2]",
+ } {
+ w, err := zw.Create(name)
+ require.NoError(t, err)
+ _, _ = w.Write([]byte(body))
+ }
+ require.NoError(t, zw.Close())
+ assert.Equal(t, "[1]", string(minerUContentListFromZip(buf.Bytes())))
+}
+
+func TestPaddleOCRVLPageUnits(t *testing.T) {
+ var page paddleOCRVLPage
+ require.NoError(t, json.Unmarshal([]byte(`{
+ "markdown":{"text":"第一段\n\n第二段"},
+ "prunedResult":{"width":1000,"height":2000,"parsing_res_list":[
+ {"block_label":"text","block_content":"第一段","block_bbox":[100,200,900,400]}]}
+ }`), &page))
+ units := paddleOCRVLPageUnits(3, page)
+ require.Len(t, units, 1)
+ assert.Equal(t, []float64{0.1, 0.1, 0.9, 0.2}, units[0].Locator.BBox)
+ assert.Equal(t, 3, units[0].Locator.Page)
+
+ page.PrunedResult = nil
+ units = paddleOCRVLPageUnits(4, page)
+ require.Len(t, units, 2)
+ assert.Nil(t, units[1].Locator.BBox)
+ assert.True(t, strings.Contains(units[1].Text, "第二段"))
+}
+
+func TestSourceBlocksFromProto(t *testing.T) {
+ blocks := sourceBlocksFromProto([]*proto.SourceBlock{
+ {Start: 0, End: 5, LocatorJson: `{"type":"pdf","page":2,"bbox":[0.1,0.2,0.3,0.4]}`},
+ {Start: 5, End: 5, LocatorJson: `{"type":"pdf","page":2}`},
+ {Start: 5, End: 9, LocatorJson: `not json`},
+ })
+ require.Len(t, blocks, 1)
+ assert.Equal(t, types.SourceLocator{Type: "pdf", Page: 2, BBox: []float64{0.1, 0.2, 0.3, 0.4}}, blocks[0].Locator)
+}
+
+func mustJSON(t *testing.T, s string) []byte {
+ t.Helper()
+ b, err := json.Marshal(s)
+ require.NoError(t, err)
+ return b
+}
diff --git a/internal/searchutil/imageinfo.go b/internal/searchutil/imageinfo.go
index d2c5b96d8..cc5468a40 100644
--- a/internal/searchutil/imageinfo.go
+++ b/internal/searchutil/imageinfo.go
@@ -8,6 +8,7 @@ import (
"sort"
"strings"
+ "github.com/Tencent/WeKnora/internal/sourceloc"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
@@ -81,13 +82,46 @@ func collectImageInfoByChunkIDs(
chunkIDs []string,
listChildren func(parentIDs []string) ([]*types.Chunk, error),
) map[string]string {
+ info, _ := collectChildEvidence(chunkIDs, listChildren)
+ return info
+}
+
+// collectChildEvidence reads the image children of chunkIDs once and returns
+// their merged image_info JSON and, per chunk, the recognized text of each
+// scanned PDF page (1-based page → quote) taken from children that carry a
+// page locator. OCR text wins over a caption for the same page.
+func collectChildEvidence(
+ chunkIDs []string,
+ listChildren func(parentIDs []string) ([]*types.Chunk, error),
+) (map[string]string, map[string]map[int]string) {
if len(chunkIDs) == 0 {
- return nil
+ return nil, nil
}
children, err := listChildren(chunkIDs)
if err != nil || len(children) == 0 {
- return nil
+ return nil, nil
+ }
+
+ pageQuotes := make(map[string]map[int]string)
+ addPageQuote := func(targetID string, child *types.Chunk) {
+ quote := sourceloc.CleanQuote(child.Content)
+ if quote == "" {
+ return
+ }
+ for _, loc := range child.SourceLocators {
+ if loc.Type != types.SourceLocatorPDF || loc.Page <= 0 {
+ continue
+ }
+ pages, ok := pageQuotes[targetID]
+ if !ok {
+ pages = make(map[int]string)
+ pageQuotes[targetID] = pages
+ }
+ if _, taken := pages[loc.Page]; !taken || child.ChunkType == types.ChunkTypeImageOCR {
+ pages[loc.Page] = quote
+ }
+ }
}
type imageAgg struct {
@@ -141,6 +175,7 @@ func collectImageInfoByChunkIDs(
switch child.ChunkType {
case types.ChunkTypeImageOCR, types.ChunkTypeImageCaption:
addInfo(child.ParentChunkID, child)
+ addPageQuote(child.ParentChunkID, child)
case types.ChunkTypeText:
textChildIDs = append(textChildIDs, child.ID)
textToParent[child.ID] = child.ParentChunkID
@@ -159,6 +194,7 @@ func collectImageInfoByChunkIDs(
}
if parentTextID, ok := textToParent[gc.ParentChunkID]; ok {
addInfo(parentTextID, gc)
+ addPageQuote(parentTextID, gc)
}
}
}
@@ -179,19 +215,21 @@ func collectImageInfoByChunkIDs(
}
out[id] = string(data)
}
- return out
+ return out, pageQuotes
}
// EnrichSearchResultsImageInfo fills in ImageInfo for SearchResults that have
-// none by batch-querying child image chunks.
+// none by batch-querying child image chunks. The same children give scanned
+// PDF pages their recognized text as locator quotes, so a citation can be
+// matched to the page it came from.
func EnrichSearchResultsImageInfo(
ctx context.Context,
chunkRepo interfaces.ChunkRepository,
tenantID uint64,
results []*types.SearchResult,
) {
- enrichSearchResultsImageInfo(results, func(chunkIDs []string) map[string]string {
- return CollectImageInfoByChunkIDs(ctx, chunkRepo, tenantID, chunkIDs)
+ enrichSearchResultsImageInfo(results, func(parentIDs []string) ([]*types.Chunk, error) {
+ return chunkRepo.ListChunksByParentIDs(ctx, tenantID, parentIDs)
})
}
@@ -204,19 +242,21 @@ func EnrichSearchResultsImageInfoOnly(
chunkRepo interfaces.ChunkRepository,
results []*types.SearchResult,
) {
- enrichSearchResultsImageInfo(results, func(chunkIDs []string) map[string]string {
- return CollectImageInfoByChunkIDsOnly(ctx, chunkRepo, chunkIDs)
+ enrichSearchResultsImageInfo(results, func(parentIDs []string) ([]*types.Chunk, error) {
+ return chunkRepo.ListChunksByParentIDsOnly(ctx, parentIDs)
})
}
func enrichSearchResultsImageInfo(
results []*types.SearchResult,
- collect func(chunkIDs []string) map[string]string,
+ listChildren func(parentIDs []string) ([]*types.Chunk, error),
) {
var chunkIDs []string
seen := make(map[string]bool)
for _, r := range results {
- if r.ImageInfo != "" {
+ // Results that already carry image info only need their children
+ // again when they are scanned pages waiting for recognized text.
+ if r.ImageInfo != "" && !scannedPagesOnly(r.SourceLocators) {
continue
}
if !seen[r.ID] {
@@ -228,19 +268,67 @@ func enrichSearchResultsImageInfo(
return
}
- infoMap := collect(chunkIDs)
- if len(infoMap) == 0 {
- return
- }
+ infoMap, pageQuotes := collectChildEvidence(chunkIDs, listChildren)
for _, r := range results {
- if r.ImageInfo != "" {
+ if merged, ok := infoMap[r.ID]; ok && r.ImageInfo == "" {
+ r.ImageInfo = merged
+ }
+ if pages, ok := pageQuotes[r.ID]; ok {
+ r.SourceLocators = withPageQuotes(r.SourceLocators, pages)
+ }
+ }
+}
+
+// needsPageQuotes reports whether a chunk has whole-page PDF locators without
+// usable text: the pages of a scanned PDF, whose chunk text is only the page
+// images.
+func needsPageQuotes(locators types.SourceLocators) bool {
+ for _, loc := range locators {
+ if loc.Type == types.SourceLocatorPDF && loc.Page > 0 && len(loc.BBox) == 0 && weakQuote(loc.Quote) {
+ return true
+ }
+ }
+ return false
+}
+
+// scannedPagesOnly reports whether every locator is a textless whole PDF
+// page, as for a scanned PDF. An embedded figure inside a text page also
+// yields a textless page locator, but its chunk has boxed text locators too.
+func scannedPagesOnly(locators types.SourceLocators) bool {
+ if len(locators) == 0 {
+ return false
+ }
+ for _, loc := range locators {
+ if loc.Type != types.SourceLocatorPDF || loc.Page <= 0 || len(loc.BBox) != 0 || !weakQuote(loc.Quote) {
+ return false
+ }
+ }
+ return true
+}
+
+func weakQuote(quote string) bool {
+ quote = strings.TrimSpace(quote)
+ return quote == "" || strings.HasPrefix(quote, "![")
+}
+
+// withPageQuotes returns a copy of locators whose textless page locators carry
+// the recognized text of their page.
+func withPageQuotes(locators types.SourceLocators, pages map[int]string) types.SourceLocators {
+ if !needsPageQuotes(locators) {
+ return locators
+ }
+ out := make(types.SourceLocators, len(locators))
+ copy(out, locators)
+ for i, loc := range out {
+ if loc.Type != types.SourceLocatorPDF || len(loc.BBox) != 0 || !weakQuote(loc.Quote) {
continue
}
- if merged, ok := infoMap[r.ID]; ok {
- r.ImageInfo = merged
+ if quote, ok := pages[loc.Page]; ok {
+ out[i].Quote = quote
}
}
+ return out
}
// MergeImageInfoJSON combines per-chunk image_info JSON strings (from
diff --git a/internal/searchutil/imageinfo_pages_test.go b/internal/searchutil/imageinfo_pages_test.go
new file mode 100644
index 000000000..660c3d126
--- /dev/null
+++ b/internal/searchutil/imageinfo_pages_test.go
@@ -0,0 +1,66 @@
+package searchutil
+
+import (
+ "context"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+func pageChild(id, parentID string, chunkType types.ChunkType, page int, content string) *types.Chunk {
+ return &types.Chunk{
+ ID: id,
+ ParentChunkID: parentID,
+ ChunkType: chunkType,
+ IsEnabled: true,
+ Content: content,
+ SourceLocators: types.SourceLocators{{Type: types.SourceLocatorPDF, Page: page}},
+ }
+}
+
+// A scanned PDF chunk holds only page images, so its page locators have no
+// text to match a cited sentence against; the pages' OCR text supplies it.
+func TestEnrichSearchResultsGivesScannedPagesTheirOCRText(t *testing.T) {
+ repo := childrenByParentRepo{byParent: map[string][]*types.Chunk{
+ "scan": {
+ pageChild("cap-1", "scan", types.ChunkTypeImageCaption, 1, "A form about working at height"),
+ pageChild("ocr-1", "scan", types.ChunkTypeImageOCR, 1, "一、作业人员必须持证上岗。"),
+ pageChild("ocr-2", "scan", types.ChunkTypeImageOCR, 2, "五、吊篮每班作业前应进行空载试运行。"),
+ },
+ }}
+ boxed := types.SourceLocators{{Type: types.SourceLocatorPDF, Page: 3, BBox: []float64{0, 0, 1, 1}, Quote: "text"}}
+ results := []*types.SearchResult{
+ {ID: "scan", SourceLocators: types.SourceLocators{
+ {Type: types.SourceLocatorPDF, Page: 1, Quote: "![page_1.jpg](resource://a"},
+ {Type: types.SourceLocatorPDF, Page: 2},
+ }},
+ {ID: "text", SourceLocators: boxed},
+ }
+
+ EnrichSearchResultsImageInfo(context.Background(), repo, 1, results)
+
+ got := results[0].SourceLocators
+ if got[0].Quote != "一、作业人员必须持证上岗。" || got[1].Quote != "五、吊篮每班作业前应进行空载试运行。" {
+ t.Fatalf("page quotes = %q, %q", got[0].Quote, got[1].Quote)
+ }
+ if results[1].SourceLocators[0].Quote != "text" {
+ t.Fatalf("a boxed text locator must keep its quote, got %q", results[1].SourceLocators[0].Quote)
+ }
+}
+
+// A text page's chunk that already has its image info is not re-queried just
+// because an embedded figure left a textless page locator.
+func TestEnrichSearchResultsSkipsTextPagesWithImageInfo(t *testing.T) {
+ results := []*types.SearchResult{{
+ ID: "text",
+ ImageInfo: `[{"url":"local://fig.png"}]`,
+ SourceLocators: types.SourceLocators{
+ {Type: types.SourceLocatorPDF, Page: 2, BBox: []float64{0, 0, 1, 0.5}, Quote: "body text"},
+ {Type: types.SourceLocatorPDF, Page: 2},
+ },
+ }}
+ enrichSearchResultsImageInfo(results, func(parentIDs []string) ([]*types.Chunk, error) {
+ t.Fatalf("unexpected child lookup for %v", parentIDs)
+ return nil, nil
+ })
+}
diff --git a/internal/sourceloc/align.go b/internal/sourceloc/align.go
new file mode 100644
index 000000000..e95c73959
--- /dev/null
+++ b/internal/sourceloc/align.go
@@ -0,0 +1,229 @@
+package sourceloc
+
+import (
+ "sort"
+ "strings"
+ "unicode"
+ "unicode/utf8"
+
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+// Unit is one structural piece of an original file (a paragraph, a slide
+// shape, a sheet row) with the locator it should resolve to.
+type Unit struct {
+ Text string
+ Locator types.SourceLocator
+}
+
+const (
+ // alignKeyRunes is how much of a unit's normalized text is searched for.
+ alignKeyRunes = 32
+ // alignMinRunes skips units too short to place reliably.
+ alignMinRunes = 2
+ // alignWindowRunes bounds how far past the cursor a unit is searched, so
+ // a unit the parser dropped cannot drag the cursor across the document.
+ alignWindowRunes = 20000
+)
+
+// Align places units, which are in document order, into markdown and returns
+// one block per placed unit. A block runs from where its unit was found to
+// where the next placed unit starts, so the blocks tile the markdown from the
+// first placed unit on. Units are matched on letters and digits only, which
+// makes the match indifferent to Markdown syntax, whitespace and
+// punctuation the parser added or dropped.
+func Align(markdown string, units []Unit) []types.SourceBlock {
+ norm := normalizeMarkdown(markdown)
+ if norm.text == "" || len(units) == 0 {
+ return nil
+ }
+ type hit struct {
+ pos int // rune offset in markdown
+ unit int
+ }
+ var hits []hit
+ cursor := 0 // byte offset into norm.text
+ for i, u := range units {
+ key := normalizeKey(u.Text)
+ if utf8.RuneCountInString(key) < alignMinRunes {
+ continue
+ }
+ at, width := norm.find(key, cursor)
+ if at < 0 {
+ continue
+ }
+ hits = append(hits, hit{pos: norm.pos[at], unit: i})
+ cursor = at + width
+ }
+ if len(hits) == 0 {
+ return nil
+ }
+ runes := []rune(markdown)
+ prev := 0
+ for i := range hits {
+ hits[i].pos = lineLeadStart(runes, hits[i].pos, prev)
+ prev = hits[i].pos
+ }
+ total := len(runes)
+ blocks := make([]types.SourceBlock, 0, len(hits))
+ for i, h := range hits {
+ end := total
+ if i+1 < len(hits) {
+ end = hits[i+1].pos
+ }
+ if end <= h.pos {
+ continue
+ }
+ blocks = append(blocks, types.SourceBlock{Start: h.pos, End: end, Locator: units[h.unit].Locator})
+ }
+ return blocks
+}
+
+// lineLeadStart moves pos back to the start of its line when everything
+// before it on the line is markup or numbering ("## ", "| ", "1. ", "(2) "),
+// which belongs to the unit but was not part of its text. It never crosses
+// floor.
+func lineLeadStart(runes []rune, pos, floor int) int {
+ const maxLead = 16
+ start := pos
+ for start > floor && runes[start-1] != '\n' {
+ if pos-start >= maxLead || unicode.IsLetter(runes[start-1]) {
+ return pos
+ }
+ start--
+ }
+ return start
+}
+
+// normalized is the letters-and-digits projection of a text: text holds the
+// kept runes, pos[b] the rune offset in the source of the rune starting at
+// byte b of text.
+type normalized struct {
+ text string
+ pos []int
+}
+
+// find searches key (already normalized) at or after byte offset from,
+// within the search window. It tries the key's head first and falls back to
+// a slice from its middle, so a unit whose opening differs (a list number
+// the parser rendered, a leading image) still lands. It returns the byte
+// offset of the match and how far the cursor should advance.
+func (n normalized) find(key string, from int) (int, int) {
+ limit := len(n.text)
+ if w := from + alignWindowRunes*3; w < limit {
+ limit = w
+ }
+ window := n.text[from:limit]
+ head := firstRunes(key, alignKeyRunes)
+ if idx := strings.Index(window, head); idx >= 0 {
+ return from + idx, len(head)
+ }
+ runes := []rune(key)
+ if len(runes) < alignKeyRunes*2 {
+ return -1, 0
+ }
+ mid := string(runes[len(runes)/2 : len(runes)/2+alignKeyRunes])
+ if idx := strings.Index(window, mid); idx >= 0 {
+ // Aim at where the unit should begin, not at its middle.
+ back := len(string(runes[:len(runes)/2]))
+ start := from + idx - back
+ start = max(start, from)
+ for start < from+idx && !utf8.RuneStart(n.text[start]) {
+ start++
+ }
+ return start, idx + len(mid) - (start - from)
+ }
+ return -1, 0
+}
+
+// normalizeMarkdown projects markdown onto its letters and digits, skipping
+// link and image destinations and HTML tags, whose URLs and attributes are
+// not text of the document.
+func normalizeMarkdown(markdown string) normalized {
+ var b strings.Builder
+ b.Grow(len(markdown))
+ pos := make([]int, 0, len(markdown))
+ runes := []rune(markdown)
+ for i := 0; i < len(runes); i++ {
+ r := runes[i]
+ switch {
+ case r == '<' && i+1 < len(runes) && (isASCIILetter(runes[i+1]) || runes[i+1] == '/' || runes[i+1] == '!'):
+ if end := indexRune(runes, '>', i+1, 4096); end > 0 {
+ i = end
+ continue
+ }
+ case r == ']' && i+1 < len(runes) && runes[i+1] == '(':
+ if end := indexRune(runes, ')', i+2, 1<<20); end > 0 {
+ i = end
+ continue
+ }
+ }
+ if nr, ok := foldRune(r); ok {
+ start := b.Len()
+ b.WriteRune(nr)
+ for k := start; k < b.Len(); k++ {
+ pos = append(pos, i)
+ }
+ }
+ }
+ return normalized{text: b.String(), pos: pos}
+}
+
+// normalizeKey projects plain text onto its letters and digits.
+func normalizeKey(text string) string {
+ var b strings.Builder
+ b.Grow(len(text))
+ for _, r := range text {
+ if nr, ok := foldRune(r); ok {
+ b.WriteRune(nr)
+ }
+ }
+ return b.String()
+}
+
+// foldRune keeps letters and digits, lower-cased and with full-width ASCII
+// folded to its half-width form.
+func foldRune(r rune) (rune, bool) {
+ if r >= 0xFF01 && r <= 0xFF5E {
+ r -= 0xFEE0
+ }
+ if !unicode.IsLetter(r) && !unicode.IsDigit(r) {
+ return 0, false
+ }
+ return unicode.ToLower(r), true
+}
+
+func isASCIILetter(r rune) bool {
+ return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z')
+}
+
+// indexRune finds r in runes[from:], giving up after limit runes or at a
+// blank line.
+func indexRune(runes []rune, r rune, from, limit int) int {
+ end := min(len(runes), from+limit)
+ for i := from; i < end; i++ {
+ if runes[i] == r {
+ return i
+ }
+ if runes[i] == '\n' && i+1 < len(runes) && runes[i+1] == '\n' {
+ return -1
+ }
+ }
+ return -1
+}
+
+func firstRunes(s string, n int) string {
+ i := 0
+ for idx := range s {
+ if i == n {
+ return s[:idx]
+ }
+ i++
+ }
+ return s
+}
+
+// SortBlocks orders blocks by start offset.
+func SortBlocks(blocks []types.SourceBlock) {
+ sort.SliceStable(blocks, func(i, j int) bool { return blocks[i].Start < blocks[j].Start })
+}
diff --git a/internal/sourceloc/assign.go b/internal/sourceloc/assign.go
new file mode 100644
index 000000000..4dba5efe3
--- /dev/null
+++ b/internal/sourceloc/assign.go
@@ -0,0 +1,215 @@
+package sourceloc
+
+import (
+ "regexp"
+ "sort"
+ "strings"
+ "unicode/utf8"
+
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+// MaxLocatorsPerChunk caps how many locators one chunk (or merged search
+// result) carries.
+const MaxLocatorsPerChunk = 64
+
+// Index answers "which source positions does this range of the final
+// Markdown cover" for the chunker's output.
+type Index struct {
+ runes []rune
+ blocks []types.SourceBlock // sorted by Start
+ maxEnd []int // maxEnd[k] = max End over blocks[:k+1]
+}
+
+// NewIndex builds an index over blocks recorded against markdown.
+func NewIndex(markdown string, blocks []types.SourceBlock) *Index {
+ if len(blocks) == 0 {
+ return nil
+ }
+ sorted := append([]types.SourceBlock(nil), blocks...)
+ SortBlocks(sorted)
+ maxEnd := make([]int, len(sorted))
+ for k, b := range sorted {
+ maxEnd[k] = b.End
+ if k > 0 && maxEnd[k-1] > b.End {
+ maxEnd[k] = maxEnd[k-1]
+ }
+ }
+ return &Index{runes: []rune(markdown), blocks: sorted, maxEnd: maxEnd}
+}
+
+// Locators returns the locators of the blocks overlapping the rune range
+// [start, end), each quoting the overlapping text. Adjacent locators that
+// point at the same place (one page without a region, one slide, a run of
+// rows) are folded together.
+func (x *Index) Locators(start, end int) types.SourceLocators {
+ if x == nil || end <= start {
+ return nil
+ }
+ start = max(start, 0)
+ end = min(end, len(x.runes))
+ // Blocks before i all end at or before start.
+ i := sort.Search(len(x.blocks), func(k int) bool { return x.maxEnd[k] > start })
+ var out types.SourceLocators
+ for ; i < len(x.blocks) && x.blocks[i].Start < end; i++ {
+ b := x.blocks[i]
+ lo, hi := max(b.Start, start), min(b.End, end)
+ if hi <= lo {
+ continue
+ }
+ quote := CleanQuote(string(x.runes[lo:hi]))
+ if quote == "" && b.Locator.Type != types.SourceLocatorPDF {
+ continue
+ }
+ loc := b.Locator
+ loc.BBox = append([]float64(nil), loc.BBox...)
+ loc.Quote = quote
+ out = AppendLocator(out, loc)
+ }
+ if len(out) > MaxLocatorsPerChunk {
+ out = out[:MaxLocatorsPerChunk]
+ }
+ return out
+}
+
+// LocatorsAt returns the locators of the blocks containing rune offset pos,
+// without quotes. Used to place an image reference found in the markdown.
+func (x *Index) LocatorsAt(pos int) types.SourceLocators {
+ if x == nil || pos < 0 {
+ return nil
+ }
+ i := sort.Search(len(x.blocks), func(k int) bool { return x.maxEnd[k] > pos })
+ var out types.SourceLocators
+ for ; i < len(x.blocks) && x.blocks[i].Start <= pos; i++ {
+ if b := x.blocks[i]; b.End > pos {
+ loc := b.Locator
+ loc.BBox = append([]float64(nil), loc.BBox...)
+ loc.Quote = ""
+ out = append(out, loc)
+ }
+ }
+ return out
+}
+
+// AppendLocator appends loc to list, folding it into the last entry when both
+// point at the same place.
+func AppendLocator(list types.SourceLocators, loc types.SourceLocator) types.SourceLocators {
+ if n := len(list); n > 0 && foldInto(&list[n-1], loc) {
+ return list
+ }
+ return append(list, loc)
+}
+
+func foldInto(last *types.SourceLocator, loc types.SourceLocator) bool {
+ if last.Type != loc.Type {
+ return false
+ }
+ switch loc.Type {
+ case types.SourceLocatorPDF:
+ if last.Page != loc.Page || len(last.BBox) > 0 || len(loc.BBox) > 0 {
+ return false
+ }
+ case types.SourceLocatorSlide:
+ if last.Slide != loc.Slide {
+ return false
+ }
+ case types.SourceLocatorSection:
+ if last.Section != loc.Section {
+ return false
+ }
+ case types.SourceLocatorSheet:
+ if last.Sheet != loc.Sheet || loc.RowStart > last.RowEnd+1 || loc.RowEnd < last.RowStart {
+ return false
+ }
+ last.RowStart = min(last.RowStart, loc.RowStart)
+ last.RowEnd = max(last.RowEnd, loc.RowEnd)
+ case types.SourceLocatorText:
+ if loc.Start > last.End+2 || loc.End < last.Start {
+ return false
+ }
+ last.Start = min(last.Start, loc.Start)
+ last.End = max(last.End, loc.End)
+ default:
+ return false
+ }
+ last.Quote = joinQuote(last.Quote, loc.Quote)
+ return true
+}
+
+// MergeLocators unions two locator lists, dropping exact duplicates, for
+// search results that the chat pipeline merges.
+func MergeLocators(a, b types.SourceLocators) types.SourceLocators {
+ if len(b) == 0 {
+ return a
+ }
+ if len(a) == 0 {
+ return b
+ }
+ out := append(types.SourceLocators(nil), a...)
+ for _, loc := range b {
+ dup := false
+ for _, have := range out {
+ if sameLocator(have, loc) {
+ dup = true
+ break
+ }
+ }
+ if !dup {
+ out = append(out, loc)
+ }
+ }
+ if len(out) > MaxLocatorsPerChunk {
+ out = out[:MaxLocatorsPerChunk]
+ }
+ return out
+}
+
+func sameLocator(a, b types.SourceLocator) bool {
+ if a.Type != b.Type || a.Page != b.Page || a.Block != b.Block || a.Slide != b.Slide ||
+ a.Sheet != b.Sheet || a.RowStart != b.RowStart || a.RowEnd != b.RowEnd ||
+ a.Start != b.Start || a.End != b.End || a.StartMs != b.StartMs || a.EndMs != b.EndMs ||
+ a.Section != b.Section || len(a.BBox) != len(b.BBox) {
+ return false
+ }
+ for i := range a.BBox {
+ if a.BBox[i] != b.BBox[i] {
+ return false
+ }
+ }
+ return true
+}
+
+var (
+ quoteImageRe = regexp.MustCompile(`!\[[^\]\n]*\]\([^)\n]*\)`)
+ quoteLinkRe = regexp.MustCompile(`\[([^\]\n]*)\]\([^)\n]*\)`)
+ quoteTagRe = regexp.MustCompile(`</?[a-zA-Z][^>\n]*>|<!--.*?-->`)
+ quoteSpaceRe = regexp.MustCompile(`\s+`)
+)
+
+// CleanQuote strips Markdown image and link destinations and HTML tags from
+// text, collapses whitespace, and caps it at types.SourceLocatorQuoteMax
+// runes.
+func CleanQuote(text string) string {
+ text = quoteImageRe.ReplaceAllString(text, " ")
+ text = quoteLinkRe.ReplaceAllString(text, "$1")
+ text = quoteTagRe.ReplaceAllString(text, " ")
+ text = strings.TrimSpace(quoteSpaceRe.ReplaceAllString(text, " "))
+ return capRunes(text, types.SourceLocatorQuoteMax)
+}
+
+func joinQuote(a, b string) string {
+ if a == "" {
+ return b
+ }
+ if b == "" || utf8.RuneCountInString(a) >= types.SourceLocatorQuoteMax {
+ return a
+ }
+ return capRunes(a+" "+b, types.SourceLocatorQuoteMax)
+}
+
+func capRunes(s string, n int) string {
+ if utf8.RuneCountInString(s) <= n {
+ return s
+ }
+ return string([]rune(s)[:n])
+}
diff --git a/internal/sourceloc/office.go b/internal/sourceloc/office.go
new file mode 100644
index 000000000..261e1f584
--- /dev/null
+++ b/internal/sourceloc/office.go
@@ -0,0 +1,557 @@
+package sourceloc
+
+import (
+ "archive/zip"
+ "bytes"
+ "encoding/csv"
+ "encoding/xml"
+ "errors"
+ "fmt"
+ "io"
+ "path"
+ "strings"
+
+ "github.com/xuri/excelize/v2"
+ "golang.org/x/net/html"
+
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+// maxUnits bounds the structure extracted from one file.
+const maxUnits = 200000
+
+// maxZipEntryBytes bounds a single archive member read during extraction.
+const maxZipEntryBytes = 64 << 20
+
+// SupportsStructure reports whether StructureUnits can read fileType.
+func SupportsStructure(fileType string) bool {
+ switch normalizeType(fileType) {
+ case "docx", "docm", "pptx", "pptm", "xlsx", "xlsm", "csv", "epub":
+ return true
+ }
+ return false
+}
+
+// AlignStructure reads the structure of an office file and aligns the
+// parser's markdown against it. It returns nil when the file type is not
+// supported or nothing could be placed.
+func AlignStructure(fileType string, data []byte, markdown string) ([]types.SourceBlock, error) {
+ units, err := StructureUnits(fileType, data)
+ if err != nil || len(units) == 0 {
+ return nil, err
+ }
+ return Align(markdown, units), nil
+}
+
+// StructureUnits lists the structural pieces of an office file in document
+// order, each with the locator it resolves to.
+func StructureUnits(fileType string, data []byte) ([]Unit, error) {
+ switch normalizeType(fileType) {
+ case "docx", "docm":
+ return docxUnits(data)
+ case "pptx", "pptm":
+ return pptxUnits(data)
+ case "xlsx", "xlsm":
+ return xlsxUnits(data)
+ case "csv":
+ return csvUnits(data)
+ case "epub":
+ return epubUnits(data)
+ }
+ return nil, nil
+}
+
+func normalizeType(fileType string) string {
+ return strings.ToLower(strings.TrimPrefix(strings.TrimSpace(fileType), "."))
+}
+
+// ---- docx -----------------------------------------------------------------
+
+// docxUnits walks word/document.xml. Every body-level paragraph and table
+// counts as one block, in the order Word lays them out; content controls
+// (w:sdt) are transparent, as they are for the docx-preview renderer.
+// Tables contribute one unit per row so long tables still align row by row.
+func docxUnits(data []byte) ([]Unit, error) {
+ zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+ if err != nil {
+ return nil, err
+ }
+ doc, err := readZipFile(zr, "word/document.xml")
+ if err != nil {
+ return nil, err
+ }
+ dec := xml.NewDecoder(bytes.NewReader(doc))
+ var (
+ units []Unit
+ depth int // element depth below w:body; sdt wrappers do not count
+ inBody bool
+ block int
+ tblDepth int
+ text strings.Builder
+ inText bool
+ skipText int
+ )
+ flush := func() {
+ t := strings.TrimSpace(text.String())
+ text.Reset()
+ if t != "" && len(units) < maxUnits {
+ loc := types.SourceLocator{Type: types.SourceLocatorDocx, Block: block}
+ units = append(units, Unit{Text: t, Locator: loc})
+ }
+ }
+ for {
+ tok, err := dec.Token()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return units, nil
+ }
+ switch t := tok.(type) {
+ case xml.StartElement:
+ name := t.Name.Local
+ if !inBody {
+ if name == "body" {
+ inBody = true
+ }
+ continue
+ }
+ if name == "sdt" || name == "sdtContent" || name == "sdtPr" || name == "sdtEndPr" {
+ if name == "sdtPr" || name == "sdtEndPr" {
+ skipText++
+ }
+ continue
+ }
+ depth++
+ if depth == 1 && (name == "p" || name == "tbl") {
+ block++
+ text.Reset()
+ }
+ switch name {
+ case "tbl":
+ tblDepth++
+ case "t":
+ inText = true
+ case "tab":
+ text.WriteByte(' ')
+ case "br", "cr":
+ text.WriteByte(' ')
+ case "delText", "instrText":
+ skipText++
+ }
+ case xml.EndElement:
+ name := t.Name.Local
+ if !inBody {
+ continue
+ }
+ if name == "body" {
+ inBody = false
+ continue
+ }
+ if name == "sdt" || name == "sdtContent" || name == "sdtPr" || name == "sdtEndPr" {
+ if name == "sdtPr" || name == "sdtEndPr" {
+ skipText--
+ }
+ continue
+ }
+ switch name {
+ case "t":
+ inText = false
+ case "delText", "instrText":
+ skipText--
+ case "tr":
+ if tblDepth == 1 {
+ flush()
+ }
+ case "tc":
+ text.WriteByte(' ')
+ case "tbl":
+ tblDepth--
+ }
+ if depth == 1 && name == "p" && tblDepth == 0 {
+ flush()
+ }
+ depth--
+ case xml.CharData:
+ if inText && skipText == 0 {
+ text.Write(t)
+ }
+ }
+ }
+ return units, nil
+}
+
+// ---- pptx -----------------------------------------------------------------
+
+type pptxPresentation struct {
+ SlideIDs []struct {
+ RID string `xml:"http://schemas.openxmlformats.org/officeDocument/2006/relationships id,attr"`
+ } `xml:"sldIdLst>sldId"`
+}
+
+type opcRelationships struct {
+ Items []struct {
+ ID string `xml:"Id,attr"`
+ Type string `xml:"Type,attr"`
+ Target string `xml:"Target,attr"`
+ } `xml:"Relationship"`
+}
+
+// pptxUnits lists every text paragraph of every slide, and of its speaker
+// notes, in presentation order. Paragraphs rather than whole slides are the
+// units so a parser that orders shapes differently still aligns.
+func pptxUnits(data []byte) ([]Unit, error) {
+ zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+ if err != nil {
+ return nil, err
+ }
+ presXML, err := readZipFile(zr, "ppt/presentation.xml")
+ if err != nil {
+ return nil, err
+ }
+ var pres pptxPresentation
+ if err := xml.Unmarshal(presXML, &pres); err != nil {
+ return nil, err
+ }
+ rels, err := readRels(zr, "ppt/_rels/presentation.xml.rels")
+ if err != nil {
+ return nil, err
+ }
+ var units []Unit
+ for i, sld := range pres.SlideIDs {
+ target, ok := rels[sld.RID]
+ if !ok {
+ continue
+ }
+ slidePath := resolvePart("ppt", target.Target)
+ slideXML, err := readZipFile(zr, slidePath)
+ if err != nil {
+ continue
+ }
+ loc := types.SourceLocator{Type: types.SourceLocatorSlide, Slide: i + 1}
+ for _, p := range drawingParagraphs(slideXML) {
+ units = append(units, Unit{Text: p, Locator: loc})
+ }
+ // Speaker notes render after the slide in every parser we use.
+ slideRels, err := readRels(zr, relsPathFor(slidePath))
+ if err == nil {
+ for _, rel := range slideRels {
+ if strings.HasSuffix(rel.Type, "/notesSlide") {
+ notesXML, err := readZipFile(zr, resolvePart(path.Dir(slidePath), rel.Target))
+ if err == nil {
+ for _, p := range drawingParagraphs(notesXML) {
+ units = append(units, Unit{Text: p, Locator: loc})
+ }
+ }
+ }
+ }
+ }
+ if len(units) >= maxUnits {
+ break
+ }
+ }
+ return units, nil
+}
+
+// drawingParagraphs returns the text of each DrawingML paragraph (a:p).
+func drawingParagraphs(doc []byte) []string {
+ dec := xml.NewDecoder(bytes.NewReader(doc))
+ var (
+ out []string
+ text strings.Builder
+ inText bool
+ )
+ for {
+ tok, err := dec.Token()
+ if err != nil {
+ break
+ }
+ switch t := tok.(type) {
+ case xml.StartElement:
+ switch t.Name.Local {
+ case "t":
+ inText = true
+ case "br":
+ text.WriteByte(' ')
+ }
+ case xml.EndElement:
+ switch t.Name.Local {
+ case "t":
+ inText = false
+ case "p":
+ if s := strings.TrimSpace(text.String()); s != "" {
+ out = append(out, s)
+ }
+ text.Reset()
+ }
+ case xml.CharData:
+ if inText {
+ text.Write(t)
+ }
+ }
+ }
+ return out
+}
+
+// ---- xlsx / csv -----------------------------------------------------------
+
+// xlsxUnits lists every non-empty row of every sheet with its row number.
+// Rows are streamed so a large workbook is never materialized whole, and
+// reading stops once maxUnits rows are listed.
+func xlsxUnits(data []byte) ([]Unit, error) {
+ f, err := excelize.OpenReader(bytes.NewReader(data))
+ if err != nil {
+ return nil, err
+ }
+ defer func() { _ = f.Close() }()
+ var units []Unit
+ for _, sheet := range f.GetSheetList() {
+ units = appendSheetUnits(f, sheet, units)
+ if len(units) >= maxUnits {
+ break
+ }
+ }
+ return units, nil
+}
+
+// appendSheetUnits appends the non-empty rows of one sheet, up to maxUnits
+// units in total.
+func appendSheetUnits(f *excelize.File, sheet string, units []Unit) []Unit {
+ rows, err := f.Rows(sheet)
+ if err != nil {
+ return units
+ }
+ defer func() { _ = rows.Close() }()
+ // The iterator visits every row number, missing rows included.
+ for row := 1; rows.Next() && len(units) < maxUnits; row++ {
+ cells, err := rows.Columns()
+ if err != nil {
+ break
+ }
+ text := strings.TrimSpace(strings.Join(cells, " "))
+ if text == "" {
+ continue
+ }
+ units = append(units, Unit{Text: text, Locator: types.SourceLocator{
+ Type: types.SourceLocatorSheet, Sheet: sheet, RowStart: row, RowEnd: row,
+ }})
+ }
+ return units
+}
+
+// csvUnits lists every record of a CSV file as a row of its only sheet.
+func csvUnits(data []byte) ([]Unit, error) {
+ reader := csv.NewReader(bytes.NewReader(data))
+ reader.LazyQuotes = true
+ reader.TrimLeadingSpace = true
+ reader.FieldsPerRecord = -1
+ var units []Unit
+ for row := 1; ; row++ {
+ rec, err := reader.Read()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return units, nil
+ }
+ text := strings.TrimSpace(strings.Join(rec, " "))
+ if text == "" {
+ continue
+ }
+ units = append(units, Unit{Text: text, Locator: types.SourceLocator{
+ Type: types.SourceLocatorSheet, RowStart: row, RowEnd: row,
+ }})
+ if len(units) >= maxUnits {
+ break
+ }
+ }
+ return units, nil
+}
+
+// ---- epub -----------------------------------------------------------------
+
+type epubContainer struct {
+ Rootfiles []struct {
+ FullPath string `xml:"full-path,attr"`
+ } `xml:"rootfiles>rootfile"`
+}
+
+type epubPackage struct {
+ Manifest []struct {
+ ID string `xml:"id,attr"`
+ Href string `xml:"href,attr"`
+ } `xml:"manifest>item"`
+ Spine []struct {
+ IDRef string `xml:"idref,attr"`
+ } `xml:"spine>itemref"`
+}
+
+// epubUnits lists the block-level text of every spine item. The section
+// number is the item's 1-based position in the spine; the title is its first
+// heading.
+func epubUnits(data []byte) ([]Unit, error) {
+ zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+ if err != nil {
+ return nil, err
+ }
+ containerXML, err := readZipFile(zr, "META-INF/container.xml")
+ if err != nil {
+ return nil, err
+ }
+ var container epubContainer
+ if err := xml.Unmarshal(containerXML, &container); err != nil || len(container.Rootfiles) == 0 {
+ return nil, errors.New("epub: no rootfile")
+ }
+ opfPath := container.Rootfiles[0].FullPath
+ opfXML, err := readZipFile(zr, opfPath)
+ if err != nil {
+ return nil, err
+ }
+ var pkg epubPackage
+ if err := xml.Unmarshal(opfXML, &pkg); err != nil {
+ return nil, err
+ }
+ hrefs := make(map[string]string, len(pkg.Manifest))
+ for _, item := range pkg.Manifest {
+ hrefs[item.ID] = item.Href
+ }
+ var units []Unit
+ for i, ref := range pkg.Spine {
+ href, ok := hrefs[ref.IDRef]
+ if !ok {
+ continue
+ }
+ doc, err := readZipFile(zr, resolvePart(path.Dir(opfPath), stripFragment(href)))
+ if err != nil {
+ continue
+ }
+ title, blocks := htmlBlocks(doc)
+ loc := types.SourceLocator{Type: types.SourceLocatorSection, Section: i + 1, Title: title}
+ for _, b := range blocks {
+ units = append(units, Unit{Text: b, Locator: loc})
+ }
+ if len(units) >= maxUnits {
+ break
+ }
+ }
+ return units, nil
+}
+
+// htmlBlocks returns the first heading and the text of each block-level
+// element of an XHTML document.
+func htmlBlocks(doc []byte) (string, []string) {
+ z := html.NewTokenizer(bytes.NewReader(doc))
+ var (
+ title string
+ blocks []string
+ text strings.Builder
+ inHeading bool
+ skip int
+ )
+ flush := func() {
+ if s := strings.Join(strings.Fields(text.String()), " "); s != "" {
+ blocks = append(blocks, s)
+ if inHeading && title == "" {
+ title = s
+ }
+ }
+ text.Reset()
+ }
+ for {
+ tt := z.Next()
+ switch tt {
+ case html.ErrorToken:
+ flush()
+ return title, blocks
+ case html.StartTagToken, html.EndTagToken, html.SelfClosingTagToken:
+ name, _ := z.TagName()
+ tag := string(name)
+ switch tag {
+ case "script", "style", "head":
+ if tt == html.StartTagToken {
+ skip++
+ } else if tt == html.EndTagToken && skip > 0 {
+ skip--
+ }
+ continue
+ }
+ if isHTMLBlock(tag) {
+ flush()
+ if strings.HasPrefix(tag, "h") && len(tag) == 2 {
+ inHeading = tt == html.StartTagToken
+ }
+ }
+ case html.TextToken:
+ if skip == 0 {
+ text.Write(z.Text())
+ text.WriteByte(' ')
+ }
+ }
+ }
+}
+
+func isHTMLBlock(tag string) bool {
+ switch tag {
+ case "p", "div", "li", "tr", "h1", "h2", "h3", "h4", "h5", "h6",
+ "blockquote", "pre", "section", "article", "table", "dt", "dd", "br", "figcaption":
+ return true
+ }
+ return false
+}
+
+// ---- zip helpers ----------------------------------------------------------
+
+func readZipFile(zr *zip.Reader, name string) ([]byte, error) {
+ for _, f := range zr.File {
+ if f.Name == name {
+ if f.UncompressedSize64 > maxZipEntryBytes {
+ return nil, fmt.Errorf("%s: entry too large", name)
+ }
+ rc, err := f.Open()
+ if err != nil {
+ return nil, err
+ }
+ defer func() { _ = rc.Close() }()
+ return io.ReadAll(io.LimitReader(rc, maxZipEntryBytes))
+ }
+ }
+ return nil, fmt.Errorf("%s: not found", name)
+}
+
+func readRels(zr *zip.Reader, name string) (map[string]struct{ Type, Target string }, error) {
+ raw, err := readZipFile(zr, name)
+ if err != nil {
+ return nil, err
+ }
+ var rels opcRelationships
+ if err := xml.Unmarshal(raw, &rels); err != nil {
+ return nil, err
+ }
+ out := make(map[string]struct{ Type, Target string }, len(rels.Items))
+ for _, r := range rels.Items {
+ out[r.ID] = struct{ Type, Target string }{r.Type, r.Target}
+ }
+ return out, nil
+}
+
+// relsPathFor returns the relationship part of an OPC part, e.g.
+// ppt/slides/slide1.xml -> ppt/slides/_rels/slide1.xml.rels.
+func relsPathFor(part string) string {
+ return path.Join(path.Dir(part), "_rels", path.Base(part)+".rels")
+}
+
+// resolvePart resolves a relationship target against the directory of its
+// source part. Absolute targets are rooted at the package root.
+func resolvePart(dir, target string) string {
+ if strings.HasPrefix(target, "/") {
+ return strings.TrimPrefix(path.Clean(target), "/")
+ }
+ return path.Clean(path.Join(dir, target))
+}
+
+func stripFragment(href string) string {
+ if i := strings.IndexByte(href, '#'); i >= 0 {
+ return href[:i]
+ }
+ return href
+}
diff --git a/internal/sourceloc/remap.go b/internal/sourceloc/remap.go
new file mode 100644
index 000000000..d67ff41a3
--- /dev/null
+++ b/internal/sourceloc/remap.go
@@ -0,0 +1,246 @@
+// Package sourceloc maps parsed Markdown back to positions in the original
+// file: it re-aims parser-reported source blocks after the ingestion pipeline
+// rewrites the text, aligns parser output to the structure of office files,
+// and derives the per-chunk locators that citations open.
+package sourceloc
+
+import (
+ "sort"
+ "strings"
+
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+// Remapper translates rune offsets in one version of a text to the matching
+// offsets in a rewritten version.
+//
+// Ingestion rewrites parser output before chunking (line endings, HTML
+// tables, image references). Every rewrite is local, so the two texts share
+// most of their lines: the remapper pairs identical lines, maps offsets
+// inside a paired line directly, and spreads offsets proportionally across
+// the unpaired stretches between pairs. That keeps it correct for any
+// future rewrite step without each step having to report its edits.
+type Remapper struct {
+ oldStarts []int // rune offset of each old line
+ newStarts []int
+ oldLen int
+ newLen int
+ pair []int // pair[i] = new line index matched to old line i, or -1
+}
+
+// NewRemapper prepares a mapping from oldText to newText.
+func NewRemapper(oldText, newText string) *Remapper {
+ oldKeys, oldStarts, oldLen := splitLines(oldText)
+ newKeys, newStarts, newLen := splitLines(newText)
+ return &Remapper{
+ oldStarts: oldStarts,
+ newStarts: newStarts,
+ oldLen: oldLen,
+ newLen: newLen,
+ pair: matchLines(oldKeys, newKeys),
+ }
+}
+
+// Map returns the offset in the new text corresponding to pos in the old one.
+func (r *Remapper) Map(pos int) int {
+ if pos <= 0 {
+ return 0
+ }
+ if pos >= r.oldLen {
+ return r.newLen
+ }
+ i := sort.SearchInts(r.oldStarts, pos+1) - 1
+ if j := r.pair[i]; j >= 0 {
+ lineLen := r.lineLen(r.newStarts, r.newLen, j)
+ return r.newStarts[j] + min(pos-r.oldStarts[i], lineLen)
+ }
+ // Unpaired line: find the unpaired stretch around it.
+ lo := i
+ for lo > 0 && r.pair[lo-1] < 0 {
+ lo--
+ }
+ hi := i
+ for hi+1 < len(r.pair) && r.pair[hi+1] < 0 {
+ hi++
+ }
+ // A stretch rewritten line for line (image references given new URLs)
+ // has as many lines on both sides: map each line onto its counterpart,
+ // so the drift of a whole-document interpolation cannot accumulate.
+ newLo, newHi := 0, len(r.newStarts)-1
+ if lo > 0 {
+ newLo = r.pair[lo-1] + 1
+ }
+ if hi+1 < len(r.pair) {
+ newHi = r.pair[hi+1] - 1
+ }
+ if newHi-newLo == hi-lo {
+ j := newLo + (i - lo)
+ oldLine := r.lineLen(r.oldStarts, r.oldLen, i)
+ newLine := r.lineLen(r.newStarts, r.newLen, j)
+ within := pos - r.oldStarts[i]
+ if within >= oldLine {
+ return r.newStarts[j] + newLine
+ }
+ return r.newStarts[j] + within*newLine/oldLine
+ }
+ // Otherwise interpolate across the stretch.
+ oldA := r.oldStarts[lo]
+ oldB := r.oldLen
+ if hi+1 < len(r.oldStarts) {
+ oldB = r.oldStarts[hi+1]
+ }
+ newA := 0
+ if lo > 0 {
+ prev := r.pair[lo-1]
+ newA = r.newStarts[prev] + r.lineLen(r.newStarts, r.newLen, prev) + 1
+ }
+ newB := r.newLen
+ if hi+1 < len(r.pair) {
+ newB = r.newStarts[r.pair[hi+1]]
+ }
+ newA = min(newA, newB)
+ if oldB <= oldA {
+ return newA
+ }
+ return newA + (pos-oldA)*(newB-newA)/(oldB-oldA)
+}
+
+// lineLen is the rune length of line j excluding its trailing newline.
+func (r *Remapper) lineLen(starts []int, total, j int) int {
+ end := total
+ if j+1 < len(starts) {
+ end = starts[j+1] - 1
+ }
+ return max(end-starts[j], 0)
+}
+
+// RemapBlocks re-aims blocks recorded against oldText at newText. Blocks
+// that collapse to nothing are dropped.
+func RemapBlocks(blocks []types.SourceBlock, oldText, newText string) []types.SourceBlock {
+ if len(blocks) == 0 || oldText == newText {
+ return blocks
+ }
+ r := NewRemapper(oldText, newText)
+ out := make([]types.SourceBlock, 0, len(blocks))
+ for _, b := range blocks {
+ start, end := r.Map(b.Start), r.Map(b.End)
+ if end <= start {
+ continue
+ }
+ b.Start, b.End = start, end
+ out = append(out, b)
+ }
+ return out
+}
+
+// splitLines returns each line's comparison key (without its line break or a
+// trailing carriage return), the rune offset where it starts, and the total
+// rune length.
+func splitLines(text string) (keys []string, starts []int, total int) {
+ pos := 0
+ for {
+ idx := strings.IndexByte(text, '\n')
+ line := text
+ if idx >= 0 {
+ line = text[:idx]
+ }
+ keys = append(keys, strings.TrimSuffix(line, "\r"))
+ starts = append(starts, pos)
+ pos += runeCount(line)
+ if idx < 0 {
+ break
+ }
+ pos++ // the newline
+ text = text[idx+1:]
+ }
+ return keys, starts, pos
+}
+
+// matchLines pairs equal lines of a and b in order. Lines unique to both
+// texts anchor the pairing (longest increasing run of anchors), and the
+// stretches between anchors are extended greedily from both ends.
+func matchLines(a, b []string) []int {
+ pair := make([]int, len(a))
+ for i := range pair {
+ pair[i] = -1
+ }
+ countA := make(map[string]int, len(a))
+ for _, k := range a {
+ countA[k]++
+ }
+ countB := make(map[string]int, len(b))
+ indexB := make(map[string]int, len(b))
+ for j, k := range b {
+ countB[k]++
+ indexB[k] = j
+ }
+ var candA, candB []int
+ for i, k := range a {
+ if countA[k] == 1 && countB[k] == 1 {
+ candA = append(candA, i)
+ candB = append(candB, indexB[k])
+ }
+ }
+ anchors := longestIncreasing(candB)
+ prevA, prevB := 0, 0
+ for _, idx := range anchors {
+ ai, bj := candA[idx], candB[idx]
+ fillGap(a, b, pair, prevA, ai, prevB, bj)
+ pair[ai] = bj
+ prevA, prevB = ai+1, bj+1
+ }
+ fillGap(a, b, pair, prevA, len(a), prevB, len(b))
+ return pair
+}
+
+// fillGap pairs equal lines of a[a0:a1] and b[b0:b1] from the front and the
+// back until the first mismatch on each side.
+func fillGap(a, b []string, pair []int, a0, a1, b0, b1 int) {
+ for a0 < a1 && b0 < b1 && a[a0] == b[b0] {
+ pair[a0] = b0
+ a0++
+ b0++
+ }
+ for a0 < a1 && b0 < b1 && a[a1-1] == b[b1-1] {
+ pair[a1-1] = b1 - 1
+ a1--
+ b1--
+ }
+}
+
+// longestIncreasing returns the indices of a longest strictly increasing
+// subsequence of seq.
+func longestIncreasing(seq []int) []int {
+ if len(seq) == 0 {
+ return nil
+ }
+ tails := make([]int, 0, len(seq)) // index into seq of the tail of each length
+ prev := make([]int, len(seq))
+ for i, v := range seq {
+ k := sort.Search(len(tails), func(t int) bool { return seq[tails[t]] >= v })
+ if k > 0 {
+ prev[i] = tails[k-1]
+ } else {
+ prev[i] = -1
+ }
+ if k == len(tails) {
+ tails = append(tails, i)
+ } else {
+ tails[k] = i
+ }
+ }
+ out := make([]int, len(tails))
+ for i, k := len(tails)-1, tails[len(tails)-1]; i >= 0; i-- {
+ out[i] = k
+ k = prev[k]
+ }
+ return out
+}
+
+func runeCount(s string) int {
+ n := 0
+ for range s {
+ n++
+ }
+ return n
+}
diff --git a/internal/sourceloc/sourceloc_test.go b/internal/sourceloc/sourceloc_test.go
new file mode 100644
index 000000000..a5b3f06da
--- /dev/null
+++ b/internal/sourceloc/sourceloc_test.go
@@ -0,0 +1,268 @@
+package sourceloc
+
+import (
+ "archive/zip"
+ "bytes"
+ "fmt"
+ "strings"
+ "testing"
+ "unicode/utf8"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "github.com/xuri/excelize/v2"
+
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+func runeIndex(s, sub string) int {
+ i := strings.Index(s, sub)
+ if i < 0 {
+ return -1
+ }
+ return len([]rune(s[:i]))
+}
+
+func TestRemapperFollowsRewrites(t *testing.T) {
+ oldText := "第一段\r\n\r\n<table><tr><td>a</td></tr></table>\r\n\r\n![img](images/x.png)\r\n\r\n最后一段文字"
+ newText := "第一段\n\n| a |\n| --- |\n\n![img](https://cdn.example/x.png)\n\n最后一段文字"
+ r := NewRemapper(oldText, newText)
+
+ assert.Equal(t, 0, r.Map(0))
+ assert.Equal(t, runeIndex(newText, "最后"), r.Map(runeIndex(oldText, "最后")))
+ assert.Equal(t, runeIndex(newText, "段文字")+1, r.Map(runeIndex(oldText, "段文字")+1))
+ assert.Equal(t, len([]rune(newText)), r.Map(len([]rune(oldText))))
+
+ // Offsets inside the rewritten table stay inside the rewritten table.
+ tableOld := runeIndex(oldText, "<table>")
+ mapped := r.Map(tableOld + 5)
+ assert.GreaterOrEqual(t, mapped, runeIndex(newText, "| a |"))
+ assert.Less(t, mapped, runeIndex(newText, "![img]"))
+}
+
+func TestRemapBlocksDropsCollapsedBlocks(t *testing.T) {
+ blocks := []types.SourceBlock{{Start: 0, End: 3, Locator: types.SourceLocator{Type: "pdf", Page: 1}}}
+ out := RemapBlocks(blocks, "abc\ndef", "abc\ndef")
+ require.Len(t, out, 1)
+ assert.Equal(t, 3, out[0].End)
+}
+
+// A scanned PDF is only page images and blank lines, so no line survives the
+// image URL rewrite; each page must still land on its own line.
+func TestRemapBlocksKeepsRewrittenLinesOnTheirPages(t *testing.T) {
+ var oldLines, newLines []string
+ var blocks []types.SourceBlock
+ offset := 0
+ for page := 1; page <= 300; page++ {
+ line := fmt.Sprintf("![doc_page_%d.jpg](images/doc_page_%d.jpg)", page, page)
+ if page > 1 {
+ offset += 2
+ }
+ blocks = append(blocks, types.SourceBlock{
+ Start: offset, End: offset + len(line),
+ Locator: types.SourceLocator{Type: types.SourceLocatorPDF, Page: page},
+ })
+ offset += len(line)
+ oldLines = append(oldLines, line)
+ newLines = append(newLines, fmt.Sprintf("![doc_page_%d.jpg](local://1/images/0123456789abcdef.jpg)", page))
+ }
+ oldText, newText := strings.Join(oldLines, "\n\n"), strings.Join(newLines, "\n\n")
+ idx := NewIndex(newText, RemapBlocks(blocks, oldText, newText))
+ for i, line := range newLines {
+ at := strings.Index(newText, line) + strings.Index(line, "local://")
+ locs := idx.LocatorsAt(utf8.RuneCountInString(newText[:at]))
+ require.Len(t, locs, 1, "page %d", i+1)
+ assert.Equal(t, i+1, locs[0].Page)
+ }
+}
+
+func TestAlignPlacesUnitsInOrder(t *testing.T) {
+ md := "# 公路工程\n\n8.14.11 索夹与吊索施工应符合下列规定:\n\n1. 在满足施工需要的前提下,应减小猫道面层开孔面积。\n\n" +
+ "![图](data:image/png;base64,QUJDREVGR0g=)\n\n| 列 | 值 |\n| --- | --- |\n| 高度 | 12m |\n"
+ units := []Unit{
+ {Text: "公路工程", Locator: types.SourceLocator{Type: "docx", Block: 1}},
+ {Text: "8.14.11 索夹与吊索施工应符合下列规定:", Locator: types.SourceLocator{Type: "docx", Block: 2}},
+ {Text: "在满足施工需要的前提下,应减小猫道面层开孔面积。", Locator: types.SourceLocator{Type: "docx", Block: 3}},
+ {Text: "不存在的段落内容不会被放置", Locator: types.SourceLocator{Type: "docx", Block: 4}},
+ {Text: "列 值", Locator: types.SourceLocator{Type: "docx", Block: 5}},
+ {Text: "高度 12m", Locator: types.SourceLocator{Type: "docx", Block: 5}},
+ }
+ blocks := Align(md, units)
+ require.Len(t, blocks, 5)
+ assert.Equal(t, 1, blocks[0].Locator.Block)
+ assert.Equal(t, runeIndex(md, "8.14.11"), blocks[1].Start)
+ assert.Equal(t, blocks[1].End, blocks[2].Start)
+ assert.Equal(t, runeIndex(md, "1. 在满足"), blocks[2].Start, "list numbering stays with its paragraph")
+ assert.Equal(t, 5, blocks[4].Locator.Block)
+ assert.Equal(t, len([]rune(md)), blocks[4].End)
+}
+
+func TestIndexLocatorsFoldsAndQuotes(t *testing.T) {
+ md := "第一页正文。\n\n第二页正文开始\n\n第二页正文继续\n\n第三页"
+ p1 := runeIndex(md, "第二页正文开始")
+ p2 := runeIndex(md, "第二页正文继续")
+ p3 := runeIndex(md, "第三页")
+ blocks := []types.SourceBlock{
+ {Start: 0, End: p1, Locator: types.SourceLocator{Type: "pdf", Page: 1}},
+ {Start: p1, End: p2, Locator: types.SourceLocator{Type: "pdf", Page: 2}},
+ {Start: p2, End: p3, Locator: types.SourceLocator{Type: "pdf", Page: 2}},
+ {Start: p3, End: len([]rune(md)), Locator: types.SourceLocator{
+ Type: "pdf", Page: 3, BBox: []float64{0.1, 0.1, 0.9, 0.2},
+ }},
+ }
+ idx := NewIndex(md, blocks)
+ locs := idx.Locators(p1+2, len([]rune(md)))
+ require.Len(t, locs, 2)
+ assert.Equal(t, 2, locs[0].Page)
+ assert.Equal(t, "页正文开始 第二页正文继续", locs[0].Quote)
+ assert.Equal(t, 3, locs[1].Page)
+ assert.Equal(t, []float64{0.1, 0.1, 0.9, 0.2}, locs[1].BBox)
+ assert.Nil(t, idx.Locators(5, 5))
+}
+
+func TestAppendLocatorMergesSheetRows(t *testing.T) {
+ var list types.SourceLocators
+ list = AppendLocator(list, types.SourceLocator{Type: "sheet", Sheet: "S1", RowStart: 3, RowEnd: 3, Quote: "a"})
+ list = AppendLocator(list, types.SourceLocator{Type: "sheet", Sheet: "S1", RowStart: 4, RowEnd: 4, Quote: "b"})
+ list = AppendLocator(list, types.SourceLocator{Type: "sheet", Sheet: "S1", RowStart: 9, RowEnd: 9, Quote: "c"})
+ require.Len(t, list, 2)
+ assert.Equal(t, 3, list[0].RowStart)
+ assert.Equal(t, 4, list[0].RowEnd)
+ assert.Equal(t, "a b", list[0].Quote)
+}
+
+func TestMergeLocatorsDedupes(t *testing.T) {
+ a := types.SourceLocators{{Type: "slide", Slide: 1}}
+ b := types.SourceLocators{{Type: "slide", Slide: 1}, {Type: "slide", Slide: 2}}
+ assert.Len(t, MergeLocators(a, b), 2)
+}
+
+func TestCleanQuote(t *testing.T) {
+ got := CleanQuote("见 ![图1](https://x/y.png) 和 [链接](https://a.b)\n<b>加粗</b> <!-- Slide number: 3 -->")
+ assert.Equal(t, "见 和 链接 加粗", got)
+}
+
+func TestTextBlocks(t *testing.T) {
+ content := "# 标题\n\n第一段\n第一段续\n\n\n第二段"
+ blocks := TextBlocks(content)
+ require.Len(t, blocks, 3)
+ assert.Equal(t, 0, blocks[0].Start)
+ assert.Equal(t, runeIndex(content, "第一段"), blocks[1].Start)
+ assert.Equal(t, runeIndex(content, "第二段"), blocks[2].Start)
+ assert.Equal(t, len([]rune(content)), blocks[2].End)
+ assert.Equal(t, blocks[1].Start, blocks[1].Locator.Start)
+}
+
+func zipOf(t *testing.T, files map[string]string) []byte {
+ t.Helper()
+ var buf bytes.Buffer
+ zw := zip.NewWriter(&buf)
+ for name, body := range files {
+ w, err := zw.Create(name)
+ require.NoError(t, err)
+ _, err = w.Write([]byte(body))
+ require.NoError(t, err)
+ }
+ require.NoError(t, zw.Close())
+ return buf.Bytes()
+}
+
+func TestDocxUnits(t *testing.T) {
+ doc := `<?xml version="1.0"?>
+<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body>
+<w:p><w:r><w:t>标题</w:t></w:r></w:p>
+<w:sdt><w:sdtPr><w:alias w:val="x"/></w:sdtPr><w:sdtContent>
+<w:p><w:r><w:t>控件内</w:t></w:r><w:r><w:delText>删除</w:delText></w:r></w:p>
+</w:sdtContent></w:sdt>
+<w:tbl><w:tr><w:tc><w:p><w:r><w:t>A1</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>B1</w:t></w:r></w:p></w:tc></w:tr>
+<w:tr><w:tc><w:p><w:r><w:t>A2</w:t></w:r></w:p></w:tc></w:tr></w:tbl>
+<w:p/>
+<w:p><w:r><w:t xml:space="preserve">末段 </w:t></w:r><w:r><w:tab/><w:t>文字</w:t></w:r></w:p>
+<w:sectPr/></w:body></w:document>`
+ units, err := docxUnits(zipOf(t, map[string]string{"word/document.xml": doc}))
+ require.NoError(t, err)
+ var got []string
+ for _, u := range units {
+ got = append(got, u.Text+"@"+string(rune('0'+u.Locator.Block)))
+ }
+ assert.Equal(t, []string{"标题@1", "控件内@2", "A1 B1@3", "A2@3", "末段 文字@5"}, got)
+}
+
+func TestPptxUnits(t *testing.T) {
+ const (
+ pNS = `xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main"`
+ rNS = `xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"`
+ )
+ files := map[string]string{
+ "ppt/presentation.xml": `<p:presentation ` + pNS + ` ` + rNS + `><p:sldIdLst>` +
+ `<p:sldId id="256" r:id="rId3"/><p:sldId id="257" r:id="rId2"/></p:sldIdLst></p:presentation>`,
+ "ppt/_rels/presentation.xml.rels": `<Relationships>` +
+ `<Relationship Id="rId2" Type="http://x/slide" Target="slides/slide1.xml"/>` +
+ `<Relationship Id="rId3" Type="http://x/slide" Target="slides/slide2.xml"/></Relationships>`,
+ "ppt/slides/slide1.xml": `<p:sld xmlns:a="a"><a:p><a:r><a:t>第二张</a:t></a:r></a:p></p:sld>`,
+ "ppt/slides/slide2.xml": `<p:sld xmlns:a="a"><a:p><a:r><a:t>封面</a:t></a:r>` +
+ `<a:r><a:t>标题</a:t></a:r></a:p><a:p/></p:sld>`,
+ "ppt/slides/_rels/slide2.xml.rels": `<Relationships><Relationship Id="rId1" ` +
+ `Type="http://x/notesSlide" Target="../notesSlides/notesSlide1.xml"/></Relationships>`,
+ "ppt/notesSlides/notesSlide1.xml": `<p:notes xmlns:a="a"><a:p><a:r><a:t>备注</a:t></a:r></a:p></p:notes>`,
+ }
+ units, err := pptxUnits(zipOf(t, files))
+ require.NoError(t, err)
+ require.Len(t, units, 3)
+ assert.Equal(t, "封面标题", units[0].Text)
+ assert.Equal(t, 1, units[0].Locator.Slide)
+ assert.Equal(t, "备注", units[1].Text)
+ assert.Equal(t, 1, units[1].Locator.Slide)
+ assert.Equal(t, 2, units[2].Locator.Slide)
+}
+
+func TestXlsxUnits(t *testing.T) {
+ f := excelize.NewFile()
+ require.NoError(t, f.SetCellValue("Sheet1", "A1", "名称"))
+ require.NoError(t, f.SetCellValue("Sheet1", "B1", "数量"))
+ require.NoError(t, f.SetCellValue("Sheet1", "A3", "螺栓"))
+ require.NoError(t, f.SetCellValue("Sheet1", "B3", 12))
+ _, err := f.NewSheet("汇总")
+ require.NoError(t, err)
+ require.NoError(t, f.SetCellValue("汇总", "A2", "合计"))
+ buf, err := f.WriteToBuffer()
+ require.NoError(t, err)
+
+ units, err := xlsxUnits(buf.Bytes())
+ require.NoError(t, err)
+ require.Len(t, units, 3)
+ assert.Equal(t, "螺栓 12", units[1].Text)
+ assert.Equal(t, 3, units[1].Locator.RowStart)
+ assert.Equal(t, "汇总", units[2].Locator.Sheet)
+ assert.Equal(t, 2, units[2].Locator.RowStart)
+
+ md := "## Sheet1\n| 名称 | 数量 |\n| --- | --- |\n| 螺栓 | 12 |\n\n## 汇总\n| 合计 |\n"
+ blocks := Align(md, units)
+ require.Len(t, blocks, 3)
+ assert.Equal(t, runeIndex(md, "| 螺栓"), blocks[1].Start, "a table row starts at its line")
+}
+
+func TestEpubUnits(t *testing.T) {
+ files := map[string]string{
+ "META-INF/container.xml": `<container><rootfiles>` +
+ `<rootfile full-path="OEBPS/content.opf"/></rootfiles></container>`,
+ "OEBPS/content.opf": `<package><manifest><item id="c1" href="ch1.xhtml"/>` +
+ `<item id="c2" href="text/ch2.xhtml"/></manifest>` +
+ `<spine><itemref idref="c1"/><itemref idref="c2"/></spine></package>`,
+ "OEBPS/ch1.xhtml": `<html><head><title>x</title></head><body><h1>第一章</h1><p>开篇<b>正文</b></p></body></html>`,
+ "OEBPS/text/ch2.xhtml": `<html><body><h2>第二章</h2><p>后文</p><script>var a=1</script></body></html>`,
+ }
+ units, err := epubUnits(zipOf(t, files))
+ require.NoError(t, err)
+ require.Len(t, units, 4)
+ assert.Equal(t, "开篇 正文", units[1].Text)
+ assert.Equal(t, "第一章", units[1].Locator.Title)
+ assert.Equal(t, 2, units[3].Locator.Section)
+}
+
+func TestCSVUnits(t *testing.T) {
+ units, err := csvUnits([]byte("name,qty\nbolt,12\n\nnut,3\n"))
+ require.NoError(t, err)
+ require.Len(t, units, 3)
+ assert.Equal(t, 3, units[2].Locator.RowStart)
+}
diff --git a/internal/sourceloc/text.go b/internal/sourceloc/text.go
new file mode 100644
index 000000000..7a04ee560
--- /dev/null
+++ b/internal/sourceloc/text.go
@@ -0,0 +1,48 @@
+package sourceloc
+
+import "github.com/Tencent/WeKnora/internal/types"
+
+// TextBlocks splits a plain-text original (Markdown, TXT) that the parser
+// passed through unchanged into paragraph blocks whose text locators point at
+// the same rune ranges of the original file.
+func TextBlocks(content string) []types.SourceBlock {
+ var blocks []types.SourceBlock
+ start := -1 // rune offset where the current paragraph starts
+ pos := 0
+ blankRun := 0
+ emit := func(end int) {
+ if start >= 0 && end > start {
+ blocks = append(blocks, types.SourceBlock{
+ Start: start, End: end,
+ Locator: types.SourceLocator{Type: types.SourceLocatorText, Start: start, End: end},
+ })
+ }
+ start = -1
+ }
+ lineHasText := false
+ lineStart := 0
+ for _, r := range content {
+ switch r {
+ case '\n':
+ if !lineHasText {
+ blankRun++
+ if blankRun == 1 {
+ emit(lineStart)
+ }
+ } else {
+ blankRun = 0
+ }
+ lineHasText = false
+ lineStart = pos + 1
+ case ' ', '\t', '\r':
+ default:
+ if !lineHasText && start < 0 {
+ start = lineStart
+ }
+ lineHasText = true
+ }
+ pos++
+ }
+ emit(pos)
+ return blocks
+}
diff --git a/internal/types/chunk.go b/internal/types/chunk.go
index 1651aec25..cfa834cb3 100644
--- a/internal/types/chunk.go
+++ b/internal/types/chunk.go
@@ -181,6 +181,10 @@ type Chunk struct {
// ContextHeader is a Markdown heading breadcrumb prepended when indexing.
// It is persisted so a later content edit can rebuild the same index input.
ContextHeader string `json:"-" gorm:"type:text"`
+ // SourceLocators point back into the original file (page and region,
+ // slide, sheet rows, ...) so citations can open the file at this chunk.
+ // Empty when the parser reported no positions.
+ SourceLocators SourceLocators `json:"source_locators,omitempty" gorm:"type:json"`
}
// ChunkRevision is an immutable snapshot of a superseded chunk revision.
diff --git a/internal/types/docparser.go b/internal/types/docparser.go
index 7db9d5542..bd780c1cc 100644
--- a/internal/types/docparser.go
+++ b/internal/types/docparser.go
@@ -24,6 +24,10 @@ type ReadResult struct {
Error string
IsAudio bool // true when the result contains raw audio data needing ASR transcription
AudioData []byte // raw audio bytes for ASR processing
+ // SourceBlocks map rune ranges of MarkdownContent back to positions in
+ // the original file. Optional: engines that know no positions leave it
+ // empty.
+ SourceBlocks []SourceBlock
}
// ImageRef represents an image reference extracted from the document.
@@ -87,6 +91,9 @@ type ParsedChunk struct {
// >= 0 means this is a child chunk referencing the parent at this index
// in the ParentChunks slice of ProcessChunksOptions.
ParentIndex int
+
+ // SourceLocators point back into the original file.
+ SourceLocators SourceLocators
}
// EmbeddingContent returns the text that should be sent to the embedding
@@ -106,10 +113,11 @@ func (c ParsedChunk) EmbeddingContent() string {
// ParsedParentChunk represents a parent chunk in the parent-child strategy.
// Parent chunks are stored in DB for context retrieval but NOT vector-indexed.
type ParsedParentChunk struct {
- Content string
- Seq int
- Start int
- End int
+ Content string
+ Seq int
+ Start int
+ End int
+ SourceLocators SourceLocators
}
type ParsedImage struct {
diff --git a/internal/types/search.go b/internal/types/search.go
index 84962c7ea..3d3c0a9a2 100644
--- a/internal/types/search.go
+++ b/internal/types/search.go
@@ -211,6 +211,10 @@ type SearchResult struct {
// KnowledgeBaseID is the ID of the knowledge base this result belongs to
KnowledgeBaseID string `json:"knowledge_base_id,omitempty"`
+ // SourceLocators point back into the original file for citation
+ // navigation. Merged results carry the union of their chunks' locators.
+ SourceLocators SourceLocators `json:"source_locators,omitempty"`
+
// ContentRevision is the chunk edit revision at retrieval time.
// Internal only: used by the merge pipeline to decide whether source
// coordinates are still trustworthy.
diff --git a/internal/types/source_locator.go b/internal/types/source_locator.go
new file mode 100644
index 000000000..071f365ee
--- /dev/null
+++ b/internal/types/source_locator.go
@@ -0,0 +1,109 @@
+package types
+
+import (
+ "database/sql/driver"
+ "encoding/json"
+ "errors"
+)
+
+// Source locator types. A locator points from a chunk back into the original
+// uploaded file so a citation can open the file at the cited place. Every
+// ordinal (page, slide, block, section, row) is 1-based.
+const (
+ // SourceLocatorPDF is a PDF page, optionally narrowed to a region.
+ SourceLocatorPDF = "pdf"
+ // SourceLocatorDocx is the Nth body-level paragraph or table of a Word file.
+ SourceLocatorDocx = "docx"
+ // SourceLocatorSlide is a presentation slide.
+ SourceLocatorSlide = "slide"
+ // SourceLocatorSheet is a row range of a spreadsheet sheet.
+ SourceLocatorSheet = "sheet"
+ // SourceLocatorText is a rune range of a plain-text original (md, txt, json).
+ SourceLocatorText = "text"
+ // SourceLocatorTime is a time range of an audio original.
+ SourceLocatorTime = "time"
+ // SourceLocatorSection is an EPUB spine item.
+ SourceLocatorSection = "section"
+)
+
+// SourceLocatorQuoteMax caps the quote stored per locator, in runes.
+const SourceLocatorQuoteMax = 300
+
+// SourceLocator is one position in an original file. Only the fields of its
+// Type are set; zero values are omitted from JSON and read back as zero.
+type SourceLocator struct {
+ Type string `json:"type"`
+ // Page is the 1-based PDF page.
+ Page int `json:"page,omitempty"`
+ // BBox is [x0, y0, x1, y1] as fractions of the page width and height,
+ // origin at the top-left corner of the rendered page.
+ BBox []float64 `json:"bbox,omitempty"`
+ // Block is the 1-based index among a Word file's body paragraphs and tables.
+ Block int `json:"block,omitempty"`
+ // Slide is the 1-based slide number in presentation order.
+ Slide int `json:"slide,omitempty"`
+ // Sheet is the spreadsheet sheet name; RowStart/RowEnd are its 1-based
+ // row numbers as shown by spreadsheet applications.
+ Sheet string `json:"sheet,omitempty"`
+ RowStart int `json:"row_start,omitempty"`
+ RowEnd int `json:"row_end,omitempty"`
+ // Start/End are rune offsets into the original text file.
+ Start int `json:"start,omitempty"`
+ End int `json:"end,omitempty"`
+ // StartMs/EndMs bound an audio segment.
+ StartMs int64 `json:"start_ms,omitempty"`
+ EndMs int64 `json:"end_ms,omitempty"`
+ // Section is the 1-based EPUB spine index.
+ Section int `json:"section,omitempty"`
+ // Title names the section or slide when the source has one.
+ Title string `json:"title,omitempty"`
+ // Quote is the cited text as parsed, capped at SourceLocatorQuoteMax
+ // runes. Viewers match it inside the rendered original and against the
+ // answer sentence that cites the chunk.
+ Quote string `json:"quote,omitempty"`
+}
+
+// SourceBlock ties a rune range of a parser's Markdown output to the place in
+// the original file it came from.
+type SourceBlock struct {
+ Start int
+ End int
+ Locator SourceLocator
+}
+
+// SourceLocators is the per-chunk locator list, stored as a JSON column.
+type SourceLocators []SourceLocator
+
+// Scan implements sql.Scanner.
+func (s *SourceLocators) Scan(value interface{}) error {
+ if value == nil {
+ *s = nil
+ return nil
+ }
+ var raw []byte
+ switch v := value.(type) {
+ case []byte:
+ raw = v
+ case string:
+ raw = []byte(v)
+ default:
+ return errors.New("SourceLocators: type assertion to []byte or string failed")
+ }
+ if len(raw) == 0 || string(raw) == "null" {
+ *s = nil
+ return nil
+ }
+ return json.Unmarshal(raw, s)
+}
+
+// Value implements driver.Valuer. An empty list is stored as NULL.
+func (s SourceLocators) Value() (driver.Value, error) {
+ if len(s) == 0 {
+ return nil, nil
+ }
+ b, err := json.Marshal([]SourceLocator(s))
+ if err != nil {
+ return nil, err
+ }
+ return string(b), nil
+}
diff --git a/internal/types/task.go b/internal/types/task.go
index 5c36517a7..7f76dd4b0 100644
--- a/internal/types/task.go
+++ b/internal/types/task.go
@@ -540,6 +540,9 @@ type ImageMultimodalPayload struct {
// enqueue time. A task without it — one already in flight when attribute
// observation shipped — falls back to the conservative policy: run OCR.
ImageActions ImageActionsConfig `json:"image_actions,omitempty"`
+ // SourceLocators place the image in the original file; copied onto the
+ // OCR and caption chunks built from it.
+ SourceLocators SourceLocators `json:"source_locators,omitempty"`
}
// KnowledgePostProcessPayload represents the knowledge post process task payload.
diff --git a/migrations/sqlite/000033_chunk_source_locators.down.sql b/migrations/sqlite/000033_chunk_source_locators.down.sql
new file mode 100644
index 000000000..6bf4fbb17
--- /dev/null
+++ b/migrations/sqlite/000033_chunk_source_locators.down.sql
@@ -0,0 +1 @@
+ALTER TABLE chunks DROP COLUMN source_locators;
diff --git a/migrations/sqlite/000033_chunk_source_locators.up.sql b/migrations/sqlite/000033_chunk_source_locators.up.sql
new file mode 100644
index 000000000..3c69644fa
--- /dev/null
+++ b/migrations/sqlite/000033_chunk_source_locators.up.sql
@@ -0,0 +1,2 @@
+-- Positions of each chunk in its original file (see versioned/000114).
+ALTER TABLE chunks ADD COLUMN source_locators TEXT;
diff --git a/migrations/versioned/000114_chunk_source_locators.down.sql b/migrations/versioned/000114_chunk_source_locators.down.sql
new file mode 100644
index 000000000..39d6aeddf
--- /dev/null
+++ b/migrations/versioned/000114_chunk_source_locators.down.sql
@@ -0,0 +1 @@
+ALTER TABLE chunks DROP COLUMN IF EXISTS source_locators;
diff --git a/migrations/versioned/000114_chunk_source_locators.up.sql b/migrations/versioned/000114_chunk_source_locators.up.sql
new file mode 100644
index 000000000..de195f83d
--- /dev/null
+++ b/migrations/versioned/000114_chunk_source_locators.up.sql
@@ -0,0 +1,3 @@
+-- Migration 000114: positions of each chunk in its original file, used to
+-- open a cited document at the cited page, slide, rows or time.
+ALTER TABLE chunks ADD COLUMN IF NOT EXISTS source_locators JSONB;
diff --git a/website-docs/03-features/03-document-parsing.md b/website-docs/03-features/03-document-parsing.md
index 7c462cd79..d1eefcaa7 100644
--- a/website-docs/03-features/03-document-parsing.md
+++ b/website-docs/03-features/03-document-parsing.md
@@ -55,7 +55,7 @@ service DocReader {
`ReadRequest` 是统一请求:设置 `file_content`/`file_name`/`file_type` 为文件模式,设置 `url`/`title` 为 URL 模式;`config.parser_engine` 指定引擎(`builtin` / `markitdown` / `opendataloader`),`config.parser_engine_overrides` 传递引擎级覆盖参数(如 `pdf_force_scanned`、`odl_hybrid`)。
-`ReadResponse` 返回 `markdown_content` + `repeated ImageRef image_refs`(图片以 **inline bytes** 内联返回,`image_dir_path` 恒为空字符串——图片持久化完全由 Go App 负责,proto 中原来的 `image_storage` 字段 3 已 `reserved`)。
+`ReadResponse` 返回 `markdown_content` + `repeated ImageRef image_refs`(图片以 **inline bytes** 内联返回,`image_dir_path` 恒为空字符串——图片持久化完全由 Go App 负责,proto 中原来的 `image_storage` 字段 3 已 `reserved`),以及可选的 `repeated SourceBlock source_blocks`:把 `markdown_content` 的一段(Unicode 码点偏移 `start`/`end`)映射回原文件中的位置(`locator_json`),用于引用定位,见[原文位置](#source-locators)。`ReadStreamMeta` 带同样的字段。
`ReadStream` 的价值(见 `main.py::ReadStream` 与 `_iter_image_refs`):每帧独立、体积小;服务端边解码 base64 边 `images.pop(ref_path)` 释放源数据,双方都不必同时持有全部图片,解决大扫描件 PDF 的峰值内存和消息尺寸问题。Go 侧 `internal/infrastructure/docparser/grpc_parser.go` 优先调用 `ReadStream`,遇到旧版本 docreader 返回 `Unimplemented` 时自动回退 unary `Read`。
@@ -313,6 +313,22 @@ Go 侧接手后(`internal/infrastructure/docparser/image_resolver.go`):将
---
+### 原文位置(source locators) {#source-locators}
+
+每个分块入库时带一组 `source_locators`,记录它在原始文件中的位置,对话里点击引用就能打开原文并高亮(见[会话与对话体验](18-chat-experience.md#查看回答与来源))。位置信息按解析引擎分别产生,统一先表达为 Markdown 区间到原文位置的映射(source block),分块后再按区间求交得到每个分块的 locator:
+
+| 来源 | 产生方式 | locator |
+| --- | --- | --- |
+| builtin PDF(`pdf_parser.py`) | 文字页用 pdfium 字形坐标分栏成行,再按行距、列表编号、句末短行分段,每段一个框;扫描页和嵌入图只记页码 | `pdf`:`page` + `bbox` |
+| MinerU(自建 / 云端 / V1) | 读取返回的 `content_list`(`page_idx` + 0–1000 归一化 `bbox`),与 Markdown 按文本对齐;仅 PDF 与图片 | `pdf`:`page` + `bbox` |
+| PaddleOCR-VL(自建 / 云端) | 逐页结果的版面块(`prunedResult.parsing_res_list`),无版面块时按页 | `pdf`:`page`(+ `bbox`) |
+| Word / PPT / Excel / CSV / EPUB(任意引擎) | Go 侧读取原文件结构(docx 正文段落与表格、pptx 幻灯片文本、工作表行、EPUB 书脊章节),与解析器输出的 Markdown 按文本对齐,不依赖具体引擎 | `docx`:`block`;`slide`:`slide`;`sheet`:`sheet` + `row_start`/`row_end`;`section`:`section` |
+| builtin Excel(`excel_parser.py`) | 直接记录每行的工作表名与行号 | `sheet` |
+| Markdown / TXT(原样透传) | 按段落记录原文字符区间 | `text`:`start`/`end` |
+| 音频 | 语音识别返回分段时间时,转写按分段逐行输出并记录时间 | `time`:`start_ms`/`end_ms` |
+
+对齐只比较字母和数字,与 Markdown 语法、空白、标点无关;解析结果在 Go 侧还会经过换行规范化、HTML 表格转换、图片地址重写,块的区间会按行比对重新映射到最终文本。每个 locator 带 `quote`(被引用文字,最多 300 字),前端据此在渲染出的原文中精确查找,并挑出与回答句子最相符的段落。旧版本 doc/ppt/xls 与网页没有结构位置,前端回退为文本查找或打开原网页。
+
### splitter/ 分块器与 Go 侧 chunker 的关系 {#_5-splitter-分块器与-go-侧-chunker-的关系}
`docreader/splitter/splitter.py` 的 `TextSplitter` 是一个带保护模式的递归分块器:
@@ -364,6 +380,7 @@ gRPC 响应中不再返回 chunks(`ReadResponse` 没有 chunk 字段);`Exc
| `DOCREADER_PDF_FILTER_HIDDEN_TEXT` | true | 过滤不可见/页外文本(防 prompt injection) |
| `DOCREADER_PDF_SANITIZE_TEXT` / `_STRIP_CHART_DEBRIS` | true | 清理占位字符 / 图表碎屑行 |
| `DOCREADER_PDF_RENDER_VECTOR_FIGURES` | true | 将矢量图表区域渲染为 JPEG |
+| `DOCREADER_PDF_SOURCE_BOXES` | true | 记录文字页每段在页面上的框(引用高亮用);关闭后只记页码,省去一次逐字坐标读取 |
| `DOCREADER_PDF_WORD_GAP_WIDTH_RATIO` / `_MARGIN_COL_WIDTH_RATIO` / `_MIN_HEADING_LINE_CHARS` 等 | 0.4 / 0.12 / 8 | 版面重建微调参数(详见源码常量区) |
#### Go 侧解析超时
diff --git a/website-docs/03-features/18-chat-experience.md b/website-docs/03-features/18-chat-experience.md
index 3e2038f50..8811f9ef0 100644
--- a/website-docs/03-features/18-chat-experience.md
+++ b/website-docs/03-features/18-chat-experience.md
@@ -8,6 +8,21 @@
回答正文中的引用角标用于定位原文分块,右侧引用面板列出本轮使用的检索来源,包括 Wiki 工具返回的内容。关闭智能体的引用角标设置后,仍可通过引用面板查看来源。回答结束时,系统可提供追问建议,详见 [Agent 引擎](07-agent.md)。
+点击知识库引用角标,引用面板会直接打开被引用的原始文件,跳到引用所在位置并高亮:
+
+| 原文格式 | 定位与高亮 |
+| --- | --- |
+| PDF(文字型) | 跳到所在页,框出引用的段落 |
+| PDF(扫描件) | 跳到所在页;用 MinerU / PaddleOCR-VL 解析时框出段落 |
+| Word | 滚动到引用的段落或表格 |
+| PPT | 滚动到引用的幻灯片并高亮文字 |
+| Excel / CSV | 高亮引用的行 |
+| Markdown / TXT / EPUB | 滚动到引用的文字(EPUB 先定位章节) |
+| 音频 | 从引用片段的时间开始播放(语音识别模型需返回分段时间,如 whisper-1) |
+| 网页导入 | 显示引用片段,并提供按片段定位打开原网页的入口 |
+
+高亮范围按回答中被引用的那句话缩小到最相关的段落。在此功能上线前入库的文档没有位置信息,会在原文中按文字查找,找不到时提示并停留在文档开头;重新解析后即可精确定位。手动录入的知识和 FAQ 没有原始文件,点击引用仍显示引用列表。面板顶部「全部引用」返回列表,列表中文档卡片上的「查看原文」按钮也可以打开原文。嵌入渠道的访客无权读取原文件,只显示引用列表。
+
<Screenshot
src="/screenshots/chat-references-drawer.png"
caption="对话页:回答、引用角标与右侧引用面板"
diff --git a/website-docs/04-api/01-api-overview.md b/website-docs/04-api/01-api-overview.md
index 16273cbd8..531cafdb1 100644
--- a/website-docs/04-api/01-api-overview.md
+++ b/website-docs/04-api/01-api-overview.md
@@ -186,6 +186,22 @@ X-Accel-Buffering: no
| `usage` | TokenUsage | `prompt_tokens/completion_tokens/total_tokens/cache_*` |
| `finish_reason` | string | 结束原因 |
+`SearchResult`(`knowledge_references`、检索接口的结果)中的 `source_locators` 是该分块在原始文件中的位置,可据此在原文里定位引用(该功能上线前入库的文档为空):
+
+| 字段 | 适用 `type` | 说明 |
+| --- | --- | --- |
+| `type` | — | `pdf` / `docx` / `slide` / `sheet` / `text` / `time` / `section` |
+| `page`、`bbox` | `pdf` | 页码(从 1 起);`bbox` 为 `[x0,y0,x1,y1]`,相对页面宽高的比例,原点在页面左上角,可缺省 |
+| `block` | `docx` | 正文中第几个段落或表格(从 1 起) |
+| `slide` | `slide` | 第几张幻灯片(从 1 起) |
+| `sheet`、`row_start`、`row_end` | `sheet` | 工作表名(CSV 为空)与行号(从 1 起,同 Excel 行号) |
+| `start`、`end` | `text` | 原文件文本的字符区间(Unicode 码点) |
+| `start_ms`、`end_ms` | `time` | 音频时间区间(毫秒) |
+| `section`、`title` | `section` | EPUB 书脊中的第几项(从 1 起)及章节标题 |
+| `quote` | 全部 | 被引用的文字,最多 300 字 |
+
+取值为 0 的数值字段会省略。合并后的检索结果携带所合并分块位置的并集。
+
流以 `response_type:"complete"`(`done:true`)终止;出错时以 `response_type:"error"`(`done:true`)终止。工具执行失败以 `tool_result`(`data.success=false`)返回,`error` 只表示整轮失败;回答因输出上限被截断时,`answer` 事件带 `data.truncated=true`。`continue-stream` 采用重放 + 100ms 轮询追增量的续传语义(`?message_id=` 必填)。
## 文件引用形式(resource_urls)
diff --git a/website-docs/04-api/02-api-chunks.md b/website-docs/04-api/02-api-chunks.md
index b2ead73b7..7b5d9bfda 100644
--- a/website-docs/04-api/02-api-chunks.md
+++ b/website-docs/04-api/02-api-chunks.md
@@ -28,7 +28,7 @@ curl "$BASE/api/v1/chunks/k-1?page=1" -H "Authorization: Bearer $TOKEN"
用途:按 chunk ID 获取单个分块(无需 knowledge_id)。
-响应:200 `{"success":true,"data":{Chunk}}`
+响应:200 `{"success":true,"data":{Chunk}}`。`Chunk.source_locators` 是分块在原始文件中的位置,结构见 [API 概览](01-api-overview.md)中的 `source_locators` 说明。
```bash
curl $BASE/api/v1/chunks/by-id/c-1 -H "Authorization: Bearer $TOKEN"
diff --git a/website-docs/06-development/02-database-schema.md b/website-docs/06-development/02-database-schema.md
index 974ff8557..415532f4c 100644
--- a/website-docs/06-development/02-database-schema.md
+++ b/website-docs/06-development/02-database-schema.md
@@ -138,7 +138,7 @@ SQLite 版本号独立演进,不能与 PostgreSQL 数字一一对应:
| `models` | AI 模型配置(LLM/embedding/rerank 等) | `id`、`tenant_id`(FK→tenants,CASCADE)、`name`/`display_name`、`type`(`KnowledgeQA` / `Embedding` / `Rerank` / `VLLM` / `ASR`)、`source`、`parameters`(JSONB;v0.8.2 起可含可选的 `spec` 覆盖协议与能力,无需迁移)、`is_default`、`is_builtin`、`managed_by`、`status` |
| `knowledge_bases` | 知识库 | `id`(UUID)、`tenant_id`、`name`、`type`(document/faq/wiki)、`chunking_config`/`image_processing_config`/`vlm_config`/`faq_config`/`asr_config`/`wiki_config`/`indexing_strategy`/`auto_tag_config`/`profile_config`/`generated_profile`(JSONB)、`embedding_model_id`/`summary_model_id`(FK→models)、`vector_store_id`(FK→vector_stores)、`storage_backend_id`(FK→storage_backends)、`creator_id`(FK→users)、`is_temporary`、`activity_scope` |
| `knowledges` | 知识条目(文档/网页/FAQ 等) | `id`、`tenant_id`、`knowledge_base_id`(FK)、`type`、`title`、`source`(VARCHAR(2048))、`parse_status`(pending/processing/finalizing/completed/failed/cancelled/deleting)、`pending_subtasks_count`(000056,`finalizing` 阶段未完成的增强子任务数)、`enable_status`、`file_name`/`file_type`/`file_size`/`file_path`/`file_hash`、`metadata`(内部入库状态)、`custom_metadata`(JSONB,用户自填元数据,000078)、`folder_path`(目录树路径,000079)、`summary_status`、`profile`(JSONB,文档画像:gist/主题/类型/典型问题,000101)、`channel`、`processed_at`/`error_message`。**没有 `tag_id` 列**——000063 起标签走 `knowledge_tag_relations` 关联表 |
-| `chunks` | 分块(检索最小单元) | `id`、`tenant_id`、`knowledge_base_id`、`knowledge_id`(FK)、`content`、`source_content`(解析器原始输出,不可变)、`content_revision`、`index_status`(ready/processing/failed)、`last_editor_id`、`context_header`(索引用标题面包屑)、`chunk_index`、`start_at`/`end_at`、`pre_chunk_id`/`next_chunk_id`(链表)、`parent_chunk_id`(父子分块自引用)、`chunk_type`(text/image/…)、`image_info`/`video_info`、`relation_chunks`/`indirect_relation_chunks`(JSONB)、`is_enabled`、`flags`、`status`、`content_hash`、`seq_id`、`tag_id` |
+| `chunks` | 分块(检索最小单元) | `id`、`tenant_id`、`knowledge_base_id`、`knowledge_id`(FK)、`content`、`source_content`(解析器原始输出,不可变)、`content_revision`、`index_status`(ready/processing/failed)、`last_editor_id`、`context_header`(索引用标题面包屑)、`chunk_index`、`start_at`/`end_at`、`pre_chunk_id`/`next_chunk_id`(链表)、`parent_chunk_id`(父子分块自引用)、`chunk_type`(text/image/…)、`image_info`/`video_info`、`relation_chunks`/`indirect_relation_chunks`(JSONB)、`source_locators`(JSONB,分块在原始文件中的位置,000114)、`is_enabled`、`flags`、`status`、`content_hash`、`seq_id`、`tag_id` |
| `chunk_revisions` | 分块历史版本(000078) | `id`、`tenant_id`、`knowledge_base_id`、`knowledge_id`、`chunk_id`+`revision`(唯一索引)、`content`、`is_enabled`、`editor_id`、`edit_source`、`edited_at` |
| `embeddings` | 向量 + BM25 索引(Postgres/ParadeDB 检索引擎专用,受 `app.skip_embedding` 门控) | `id`、`source_id`+`source_type`(唯一,chunk/wiki 页等来源)、`chunk_id`/`knowledge_id`/`knowledge_base_id`、`content`(BM25 全文)、`dimension`、`embedding`(halfvec,HNSW 索引按 768/1024/3584 维分建)、`is_enabled`、`tag_id` |
| `knowledge_tags` | 知识标签(FAQ 分类等) | `id`、`tenant_id`、`knowledge_base_id`、`name`、`seq_id` |