Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(parse): 避免 no_split 重复创建临时目录 (#4385)

Kchenc05e5bb98766
diff --git a/openviking/parse/parsers/markdown.py b/openviking/parse/parsers/markdown.py
index a352db2d1..562cb7e60 100644
--- a/openviking/parse/parsers/markdown.py
+++ b/openviking/parse/parsers/markdown.py
@@ -414,9 +414,7 @@ class MarkdownParser(BaseParser):
doc_name = self._sanitize_for_path(doc_title)
# Preserve code source filenames as the temp document directory.
source_name = kwargs.get("source_name")
- root_name = (
- source_name if source_name and supports_code_skeleton(source_name) else doc_name
- )
+ root_name = source_name if source_name and supports_code_skeleton(source_name) else doc_name
root_dir = (
temp_uri
if kwargs.get("flatten_single_output", False)
@@ -426,8 +424,12 @@ class MarkdownParser(BaseParser):
# Find all headings
headings = self._find_headings(content)
- # The temp dir is the first thing materialized on apply.
- ops: List[_LayoutOp] = [_LayoutOp("mkdir", temp_uri)]
+ # The temp dir is the first thing materialized on apply. A flattened
+ # document uses the temp dir itself as its root, which _build_structure
+ # creates below, so do not enqueue the same mkdir twice.
+ ops: List[_LayoutOp] = []
+ if root_dir != temp_uri:
+ ops.append(_LayoutOp("mkdir", temp_uri))
await self._build_structure(
ops,
content,
@@ -782,9 +784,11 @@ class MarkdownParser(BaseParser):
for row in rows:
candidate_lines = [*current_lines, row]
candidate = "\n".join(candidate_lines)
- if current_lines and (
- len(candidate) > max_chars or self._estimate_token_count(candidate) > max_size
- ) and current_data_rows > 0:
+ if (
+ current_lines
+ and (len(candidate) > max_chars or self._estimate_token_count(candidate) > max_size)
+ and current_data_rows > 0
+ ):
flush()
current_lines = [*header, row] if repeat_header else [row]
current_data_rows = 1
diff --git a/tests/parse/test_markdown_no_split.py b/tests/parse/test_markdown_no_split.py
index 62f5d18a6..1f56afed7 100644
--- a/tests/parse/test_markdown_no_split.py
+++ b/tests/parse/test_markdown_no_split.py
@@ -24,7 +24,8 @@ class _FakeVikingFS:
return "viking://temp/pdf-no-split"
async def mkdir(self, uri: str, exist_ok: bool = False) -> None:
- del exist_ok
+ if uri in self.directories and not exist_ok:
+ raise FileExistsError(uri)
self.directories.add(uri)
async def write_file(self, uri: str, content: str) -> None:
@@ -48,15 +49,36 @@ def test_normalize_parse_mode_accepts_no_split_and_rejects_no_parse() -> None:
normalize_parse_mode("no_parse")
+async def test_flattened_no_split_markdown_creates_temp_root_once(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ source = tmp_path / "document.md"
+ content = "# Document\n\nBody"
+ source.write_text(content, encoding="utf-8")
+ fake_fs = _FakeVikingFS()
+ parser = MarkdownParser()
+ monkeypatch.setattr(parser, "_get_viking_fs", lambda: fake_fs)
+
+ result = await parser.parse(
+ source,
+ split_content=False,
+ flatten_single_output=True,
+ )
+
+ assert result.warnings == []
+ assert fake_fs.directories == {"viking://temp/pdf-no-split"}
+ assert fake_fs.files == {"viking://temp/pdf-no-split/document.md": content}
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("with_headings", [False, True])
async def test_long_markdown_no_split_plans_one_complete_file(
with_headings: bool,
) -> None:
content = _long_markdown(with_headings=with_headings)
- parser = MarkdownParser(
- config=ParserConfig(max_section_size=32, max_section_chars=128)
- )
+ parser = MarkdownParser(config=ParserConfig(max_section_size=32, max_section_chars=128))
layout = await parser._compute_layout(
content,
@@ -78,9 +100,7 @@ async def test_long_markdown_no_split_plans_one_complete_file(
@pytest.mark.asyncio
async def test_long_markdown_default_mode_still_splits() -> None:
content = _long_markdown()
- parser = MarkdownParser(
- config=ParserConfig(max_section_size=32, max_section_chars=128)
- )
+ parser = MarkdownParser(config=ParserConfig(max_section_size=32, max_section_chars=128))
layout = await parser._compute_layout(
content,
@@ -102,9 +122,7 @@ async def test_pdf_no_split_converts_to_one_complete_markdown(
source.write_bytes(b"%PDF-fixture")
content = _long_markdown(with_headings=True)
fake_fs = _FakeVikingFS()
- parser = PDFParser(
- PDFConfig(strategy="local", max_section_size=32, max_section_chars=128)
- )
+ parser = PDFParser(PDFConfig(strategy="local", max_section_size=32, max_section_chars=128))
markdown_parser = parser._get_markdown_parser()
monkeypatch.setattr(markdown_parser, "_get_viking_fs", lambda: fake_fs)
monkeypatch.setattr(
@@ -125,9 +143,6 @@ async def test_pdf_no_split_converts_to_one_complete_markdown(
assert result.parser_name == "PDFParser"
assert fake_fs.files == {
- (
- "viking://temp/pdf-no-split/社交网络中英文剧本/"
- "社交网络中英文剧本.md"
- ): content
+ ("viking://temp/pdf-no-split/社交网络中英文剧本/社交网络中英文剧本.md"): content
}
assert not any(uri.endswith(".pdf") for uri in fake_fs.files)