Skip to content

Build and run agents you can see, understand and trust.

fix(memory): extend the frontmatter read to the closing fence when the byte cap cuts it off (#3617)

konluebe0c7e8bbfce
diff --git a/src/agentscope/middleware/_longterm_memory/_agentic_memory/_middleware.py b/src/agentscope/middleware/_longterm_memory/_agentic_memory/_middleware.py
index 56e49436f..caf312dee 100644
--- a/src/agentscope/middleware/_longterm_memory/_agentic_memory/_middleware.py
+++ b/src/agentscope/middleware/_longterm_memory/_agentic_memory/_middleware.py
@@ -433,8 +433,10 @@ class AgenticMemoryMiddleware(MiddlewareBase):
default=256,
title="Retrieval Max Tokens per Frontmatter",
description=(
- "The maximum number of tokens to read from the beginning of "
- "each Markdown file when parsing frontmatter."
+ "The maximum estimated tokens of type and description per "
+ "memory file included in relevance selection. Complete "
+ "frontmatter is parsed before truncation, retaining type "
+ "before description within the shared budget."
),
)
@@ -894,6 +896,38 @@ class AgenticMemoryMiddleware(MiddlewareBase):
for fm in cls._FIELD_RE.finditer(m.group("body"))
}
+ @classmethod
+ def _frontmatter_snippet(
+ cls,
+ raw: bytes,
+ max_frontmatter_bytes: int,
+ ) -> str:
+ """Return the leading bytes to inspect for frontmatter.
+
+ Args:
+ raw (`bytes`):
+ The full file content.
+ max_frontmatter_bytes (`int`):
+ The initial byte cap for the parsing prefix.
+
+ Returns:
+ `str`:
+ The decoded prefix to parse. When the frontmatter block is
+ longer than the cap, the window extends to the closing
+ fence so its fields survive. Files without a complete
+ leading frontmatter block keep the capped prefix. The
+ metadata budget is applied separately after parsing.
+ """
+ snippet_end = min(len(raw), max_frontmatter_bytes)
+ match = re.match(
+ cls._FRONTMATTER_RE.pattern.encode("ascii"),
+ raw,
+ re.DOTALL,
+ )
+ if match:
+ snippet_end = max(snippet_end, match.end())
+ return raw[:snippet_end].decode("utf-8", errors="replace")
+
async def _list_md_files(self) -> list[_MemoryFileHeader]:
"""Scan the memory directory for individual memory files.
@@ -933,15 +967,23 @@ class AgenticMemoryMiddleware(MiddlewareBase):
try:
raw = await self._backend.read_file(full_path)
- # Only parse the leading bytes to keep this cheap.
+ # Recover complete fields before limiting manifest metadata.
max_frontmatter_bytes = _estimate_bytes(
self._parameters.retrieval_max_tokens_per_frontmatter,
)
- snippet = raw[:max_frontmatter_bytes].decode(
- "utf-8",
- errors="replace",
+ snippet = self._frontmatter_snippet(
+ raw,
+ max_frontmatter_bytes,
)
fields = self._parse_frontmatter_fields(snippet)
+ remaining_bytes = max(0, max_frontmatter_bytes)
+ for key in ("type", "description"):
+ fields[key] = (
+ fields.get(key, "")
+ .encode("utf-8")[:remaining_bytes]
+ .decode("utf-8", errors="ignore")
+ )
+ remaining_bytes -= len(fields[key].encode("utf-8"))
mtime = await self._backend.stat_mtime(full_path)
headers.append(
_MemoryFileHeader(
diff --git a/tests/middleware_filesystem_memory_test.py b/tests/middleware_filesystem_memory_test.py
index 3ca016c38..9eb1183c2 100644
--- a/tests/middleware_filesystem_memory_test.py
+++ b/tests/middleware_filesystem_memory_test.py
@@ -323,6 +323,45 @@ class AgenticMemoryMiddlewareTest(IsolatedAsyncioTestCase):
"""Remove the temporary workspace after each test."""
shutil.rmtree(self.temp_dir, ignore_errors=True)
+ async def test_discovery_caps_metadata_after_parsing(self) -> None:
+ """Recover complete fields before sharing the manifest byte budget."""
+ euro = "\u20ac"
+ mixed = f"{euro}a"
+ cases = [
+ (256, "user", "x" * 2000, "user", "x" * 1020),
+ (4, "user", euro * 1000, "user", euro * 4),
+ (5, "user", euro * 1000, "user", euro * 5),
+ (4, "user", mixed * 1000, "user", mixed * 3),
+ (4, "user", euro * 3 + "abcde", "user", euro * 3 + "abc"),
+ (4, "user", "Tea", "user", "Tea"),
+ (4, "", "x" * 2000, None, "x" * 16),
+ (1, "user", "x" * 2000, "user", None),
+ (0, "user", "x" * 2000, None, None),
+ (4, "x" * 2000, "Tea", "x" * 16, None),
+ ]
+ for cap, kind, desc, expected_kind, expected_desc in cases:
+ with self.subTest(cap=cap, kind=kind[:10], desc=desc[:10]):
+ middleware = AgenticMemoryMiddleware(
+ workdir=self.temp_dir,
+ parameters=AgenticMemoryMiddleware.Parameters(
+ retrieval_max_tokens_per_frontmatter=cap,
+ ),
+ )
+ memory_dir = os.path.join(self.temp_dir, "Memory")
+ _write_memory_file(memory_dir, "tea.md", desc, kind, "Body")
+ # pylint: disable-next=protected-access
+ headers = await middleware._list_md_files()
+ self.assertEqual(
+ [(h.filename, h.description, h.type) for h in headers],
+ [("tea.md", expected_desc, expected_kind)],
+ )
+ headers[0].mtime = None
+ tag = f"[{expected_kind}] " if expected_kind else ""
+ detail = f": {expected_desc}" if expected_desc else ""
+ # pylint: disable-next=protected-access
+ manifest = middleware._format_manifest(headers)
+ self.assertEqual(manifest, f"- {tag}tea.md (unknown){detail}")
+
async def test_retrieval_input_skips_messages_without_text_blocks(
self,
) -> None:
@@ -890,6 +929,76 @@ class AgenticMemoryMiddlewareTest(IsolatedAsyncioTestCase):
class FrontmatterParseTest(TestCase):
"""Parsing of memory-file frontmatter."""
+ def test_overlong_frontmatter_with_leading_whitespace(self) -> None:
+ """Find the closing fence after the actual opening fence."""
+ description = "Tea " * 400
+ for prefix in ("", "\n", "\n\n", "\r\n", " \n"):
+ for newline in ("\n", "\r\n"):
+ for ending in ("", newline):
+ with self.subTest(
+ prefix=prefix,
+ newline=newline,
+ ending=ending,
+ ):
+ content = (
+ prefix
+ + newline.join(
+ [
+ "---",
+ f"description: {description}",
+ "type: user",
+ "---",
+ ],
+ )
+ + ending
+ )
+ # pylint: disable-next=protected-access
+ snippet = AgenticMemoryMiddleware._frontmatter_snippet(
+ content.encode(),
+ 1024,
+ )
+ fields = (
+ # pylint: disable-next=protected-access
+ AgenticMemoryMiddleware._parse_frontmatter_fields(
+ snippet,
+ )
+ )
+ self.assertEqual(
+ fields,
+ {
+ "description": description.strip(),
+ "type": "user",
+ },
+ )
+
+ def test_fence_prefix_is_not_a_closing_fence(self) -> None:
+ """Skip lines that only start with the fence marker."""
+ raw = (
+ b"---\n---not-a-fence\ndescription: "
+ + b"x" * 2000
+ + b"\ntype: user\n---\n"
+ )
+ # pylint: disable-next=protected-access
+ snippet = AgenticMemoryMiddleware._frontmatter_snippet(raw, 1024)
+ # pylint: disable-next=protected-access
+ fields = AgenticMemoryMiddleware._parse_frontmatter_fields(snippet)
+ self.assertEqual(fields, {"description": "x" * 2000, "type": "user"})
+
+ def test_non_frontmatter_fences_keep_capped_prefix(self) -> None:
+ """Do not extend for body fences or incomplete frontmatter."""
+ cases = [
+ b"# Body\n" + b"x" * 2000 + b"\n---\n",
+ b"---\ndescription: " + b"x" * 2000 + b"\n---invalid\n",
+ ]
+ for raw in cases:
+ with self.subTest(start=raw[:20]):
+ # pylint: disable-next=protected-access
+ snippet = AgenticMemoryMiddleware._frontmatter_snippet(
+ raw,
+ 1024,
+ )
+ self.assertEqual(snippet, raw[:1024].decode())
+
def test_closing_fence_at_end_of_text_keeps_fields(self) -> None:
"""Keep fields when the closing fence is the last line.
@@ -916,3 +1025,39 @@ class FrontmatterParseTest(TestCase):
parse(ending_on_fence + "\n\nThey drink tea daily.\n"),
fields,
)
+
+ def test_overlong_frontmatter_extends_read_to_closing_fence(self) -> None:
+ """Recover fields when the frontmatter outgrows the byte cap and
+ the closing fence sits beyond it."""
+ description = "Cumulative notes about the billing migration. " * 40
+ raw = (
+ "---\n"
+ f"description: {description}\n"
+ "type: procedure\n"
+ "---\n"
+ "\n# Body\n\nirrelevant"
+ ).encode()
+ cap = 256 * 4 # default retrieval_max_tokens_per_frontmatter
+ # pylint: disable-next=protected-access
+ snippet = AgenticMemoryMiddleware._frontmatter_snippet(raw, cap)
+ self.assertIn("\n---", snippet)
+ # pylint: disable-next=protected-access
+ fields = AgenticMemoryMiddleware._parse_frontmatter_fields(snippet)
+ self.assertEqual(
+ fields,
+ {"description": description.strip(), "type": "procedure"},
+ )
+
+ def test_unclosed_frontmatter_keeps_capped_prefix(self) -> None:
+ """A file without a closing fence keeps today's capped prefix."""
+ raw = b"---\ndescription: " + b"x" * 2000
+ # pylint: disable-next=protected-access
+ snippet = AgenticMemoryMiddleware._frontmatter_snippet(raw, 1024)
+ self.assertEqual(snippet, raw[:1024].decode("utf-8", errors="replace"))
+
+ def test_short_frontmatter_unchanged(self) -> None:
+ """A frontmatter inside the cap is passed through verbatim."""
+ raw = b"---\nname: tea.md\ntype: user\n---\n\n# Body\n"
+ # pylint: disable-next=protected-access
+ snippet = AgenticMemoryMiddleware._frontmatter_snippet(raw, 1024)
+ self.assertEqual(snippet, raw.decode())