Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(memory): bound verb-pattern word runs to stop quadratic backtracking in bridge import (#4370) (#4376)

viwe-monaid232d097cf33
diff --git a/headroom/memory/bridge_parsers.py b/headroom/memory/bridge_parsers.py
index bc9f85eeb..693697390 100644
--- a/headroom/memory/bridge_parsers.py
+++ b/headroom/memory/bridge_parsers.py
@@ -185,11 +185,17 @@ def extract_relationships_from_section(
relationships.append({"source": key, "relationship": "is", "destination": value})
# Pattern: "X uses Y", "X with Y", "X via Y"
+ # Word runs start at a word boundary (`\b`), so the scanner never restarts
+ # inside a long token — without the anchor, each character of a long word
+ # (base64 blob, minified line, hash) retries a growing `\w+` prefix and the
+ # scan costs O(n^2) in the token length (#4370). The anchor keeps `\w+`
+ # unbounded, so a name of any length still binds whole instead of being
+ # cut to a fixed-size slice (review finding on the PR).
verb_patterns = [
- (r"(\w+(?:\s+\w+)?)\s+uses\s+(\w+(?:\s+\w+)?)", "uses"),
- (r"(\w+(?:\s+\w+)?)\s+requires\s+(\w+(?:\s+\w+)?)", "requires"),
- (r"(\w+(?:\s+\w+)?)\s+integrates\s+with\s+(\w+(?:\s+\w+)?)", "integrates_with"),
- (r"(\w+(?:\s+\w+)?)\s+depends\s+on\s+(\w+(?:\s+\w+)?)", "depends_on"),
+ (r"\b(\w+(?:\s+\w+)?)\s+uses\s+(\w+(?:\s+\w+)?)", "uses"),
+ (r"\b(\w+(?:\s+\w+)?)\s+requires\s+(\w+(?:\s+\w+)?)", "requires"),
+ (r"\b(\w+(?:\s+\w+)?)\s+integrates\s+with\s+(\w+(?:\s+\w+)?)", "integrates_with"),
+ (r"\b(\w+(?:\s+\w+)?)\s+depends\s+on\s+(\w+(?:\s+\w+)?)", "depends_on"),
]
for pattern, rel_type in verb_patterns:
for match in re.finditer(pattern, section.content, re.IGNORECASE):
diff --git a/headroom/memory/system.py b/headroom/memory/system.py
index b7953b19d..ab3b16d44 100644
--- a/headroom/memory/system.py
+++ b/headroom/memory/system.py
@@ -8,6 +8,7 @@ function-calling LLMs.
from __future__ import annotations
+import inspect
import logging
from typing import Any, Protocol, runtime_checkable
@@ -18,6 +19,38 @@ from headroom.memory.tools import MEMORY_TOOLS, MEMORY_TOOLS_OPTIMIZED
logger = logging.getLogger(__name__)
+def _accepts_kwarg(func: Any, name: str) -> bool:
+ """Return True if `func` accepts `name` as a keyword argument."""
+ try:
+ params = inspect.signature(func).parameters
+ except (TypeError, ValueError):
+ # Uninspectable callable: let the call itself decide.
+ return True
+ if any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values()):
+ return True
+ return name in params
+
+
+def _filter_supported_kwargs(func: Any, kwargs: dict[str, Any]) -> dict[str, Any]:
+ """Drop keyword arguments the callable's signature does not accept.
+
+ Python raises a binding ``TypeError`` before the callable runs, but from the
+ outside it is indistinguishable from a ``TypeError`` raised *inside* a
+ backend that has already persisted a record. Deciding field support by
+ calling and retrying can therefore save the memory twice, so unsupported
+ fields must be filtered out before the call. Backends that take ``**kwargs``
+ (or whose signature cannot be inspected) receive every field and decide for
+ themselves.
+ """
+ try:
+ params = inspect.signature(func).parameters
+ except (TypeError, ValueError):
+ return kwargs
+ if any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values()):
+ return kwargs
+ return {k: v for k, v in kwargs.items() if k in params}
+
+
# =============================================================================
# Memory Backend Protocol
# =============================================================================
@@ -347,7 +380,10 @@ class MemorySystem:
save_kwargs["extracted_entities"] = extracted_entities
if extracted_relationships is not None:
save_kwargs["extracted_relationships"] = extracted_relationships
- if background is not None:
+ # Only pass `background` to backends that accept it. Sending it to a
+ # backend without the parameter would raise TypeError before the save
+ # runs, losing the pre-extracted payload with it.
+ if background is not None and _accepts_kwarg(self._backend.save_memory, "background"):
save_kwargs["background"] = background
# Log optimization usage
@@ -355,21 +391,22 @@ class MemorySystem:
if has_pre_extraction:
logger.info("Using pre-extracted data for optimized storage")
- # Save the memory
- try:
- memory = await self._backend.save_memory(**save_kwargs)
- except TypeError:
- # Backend doesn't support pre-extraction fields, use basic call
- logger.debug("Backend doesn't support pre-extraction, falling back to basic save")
- memory = await self._backend.save_memory(
- content=content,
- user_id=self._user_id,
- importance=importance,
- entities=entities,
- relationships=relationships,
- session_id=self._session_id,
- metadata=metadata,
- )
+ # Save the memory. Field support is resolved from the backend's
+ # signature before the call, never by trial and error: a TypeError from
+ # argument binding happens before the backend runs, but one raised
+ # inside the backend may come after it already wrote the record, and
+ # the two cannot be told apart afterwards. A failed save is therefore
+ # never replayed — an internal TypeError propagates to the caller.
+ pre_keys = [
+ k
+ for k in ("facts", "extracted_entities", "extracted_relationships")
+ if k in save_kwargs
+ ]
+ save_kwargs = _filter_supported_kwargs(self._backend.save_memory, save_kwargs)
+ pre_extraction_persisted = bool(pre_keys) and all(k in save_kwargs for k in pre_keys)
+ if pre_keys and not pre_extraction_persisted:
+ logger.debug("Backend doesn't support pre-extraction fields, sending basic fields only")
+ memory = await self._backend.save_memory(**save_kwargs)
logger.info(f"Saved memory {memory.id}: {content[:50]}...")
@@ -383,9 +420,16 @@ class MemorySystem:
# Include optimization info if pre-extraction was used
if has_pre_extraction:
- result["optimized"] = True
- if facts:
- result["fact_count"] = len(facts)
+ if pre_extraction_persisted:
+ result["optimized"] = True
+ if facts:
+ result["fact_count"] = len(facts)
+ else:
+ # The fallback dropped the pre-extracted payload; report what
+ # was actually stored instead of the facts we were handed.
+ result["optimized"] = False
+ result["facts_stored"] = False
+ result["fact_count"] = 0
# Include async/background info if save was async
if memory.metadata and memory.metadata.get("_async"):
diff --git a/tests/test_memory_bridge.py b/tests/test_memory_bridge.py
index 789e1ced0..ef4b093dd 100644
--- a/tests/test_memory_bridge.py
+++ b/tests/test_memory_bridge.py
@@ -262,6 +262,59 @@ class TestRelationshipExtraction:
uses_rels = [r for r in rels if r["relationship"] == "uses"]
assert len(uses_rels) >= 1
+ def test_verb_patterns_still_match_with_long_words(self):
+ """Words up to the bound keep matching normally."""
+ section = ParsedSection(
+ heading="Test",
+ heading_level=2,
+ content="Aaa64 uses Bbb64",
+ facts=[],
+ )
+ rels = extract_relationships_from_section(section)
+ uses_rels = [r for r in rels if r["relationship"] == "uses"]
+ assert uses_rels and uses_rels[0]["source"] == "Aaa64"
+ assert uses_rels[0]["destination"] == "Bbb64"
+
+ def test_long_token_is_linear_not_quadratic(self):
+ """A single long token must not stall relationship extraction (#4370).
+
+ Before the word-boundary anchor, the four verb patterns each
+ backtracked O(n^2) over a long word run (an 8k-char token took
+ seconds). The anchor keeps the scan linear.
+ """
+ import time
+
+ long_token = "a" * 20000
+ section = ParsedSection(
+ heading="Test",
+ heading_level=2,
+ content=f"{long_token}\nHeadroom uses SQLite for storage",
+ facts=[],
+ )
+ start = time.monotonic()
+ rels = extract_relationships_from_section(section)
+ elapsed = time.monotonic() - start
+ assert elapsed < 2.0, f"extraction took {elapsed:.1f}s on one long token"
+ uses_rels = [r for r in rels if r["relationship"] == "uses"]
+ assert any(
+ "Headroom" in r["source"] and r["destination"].split()[0] == "SQLite" for r in uses_rels
+ )
+
+ def test_long_names_bind_whole(self):
+ """A name longer than 64 characters binds whole, not as a 64-char slice."""
+ long_src = "A" * 70
+ long_dst = "B" * 72
+ section = ParsedSection(
+ heading="Test",
+ heading_level=2,
+ content=f"{long_src} uses {long_dst}",
+ facts=[],
+ )
+ rels = extract_relationships_from_section(section)
+ uses_rels = [r for r in rels if r["relationship"] == "uses"]
+ assert uses_rels and uses_rels[0]["source"] == long_src
+ assert uses_rels[0]["destination"] == long_dst
+
# =============================================================================
# Bridge Tests (require backend)
diff --git a/tests/test_memory_system.py b/tests/test_memory_system.py
index 2409d30a9..17038f89c 100644
--- a/tests/test_memory_system.py
+++ b/tests/test_memory_system.py
@@ -1027,6 +1027,245 @@ class TestMemorySystem:
assert result["success"] is True
assert "memory_id" in result
+ @pytest.mark.asyncio
+ async def test_handle_memory_save_background_keeps_pre_extracted_payload(self):
+ """background=True must not strip facts on backends lacking the kwarg (#4359)."""
+
+ class LocalLikeBackend:
+ """Accepts pre-extraction fields but not `background` (like LocalBackend)."""
+
+ def __init__(self):
+ self.received: dict[str, Any] | None = None
+
+ async def save_memory(
+ self,
+ content: str,
+ user_id: str,
+ importance: float = 0.5,
+ entities: list[str] | None = None,
+ relationships: list[dict[str, str]] | None = None,
+ session_id: str | None = None,
+ metadata: dict[str, Any] | None = None,
+ facts: list[str] | None = None,
+ extracted_entities: list[dict[str, str]] | None = None,
+ extracted_relationships: list[dict[str, str]] | None = None,
+ ) -> Memory:
+ self.received = {"facts": facts, "extracted_entities": extracted_entities}
+ return Memory(content=content, user_id=user_id, importance=importance)
+
+ async def search_memories(self, *args: Any, **kwargs: Any) -> list[Memory]:
+ return []
+
+ async def update_memory(self, *args: Any, **kwargs: Any) -> Memory | None:
+ return None
+
+ async def delete_memory(self, *args: Any, **kwargs: Any) -> bool:
+ return False
+
+ async def get_memory(self, memory_id: str) -> Memory | None:
+ return None
+
+ backend = LocalLikeBackend()
+ system = MemorySystem(backend, user_id="alice") # type: ignore[arg-type]
+ result = await system.handle_memory_save(
+ content="Summary",
+ importance=0.5,
+ facts=["A", "B"],
+ extracted_entities=[{"entity": "A", "entity_type": "concept"}],
+ background=True,
+ )
+
+ assert result["success"] is True
+ assert backend.received is not None
+ assert backend.received["facts"] == ["A", "B"]
+ assert backend.received["extracted_entities"] == [{"entity": "A", "entity_type": "concept"}]
+ assert result["optimized"] is True
+ assert result["fact_count"] == 2
+
+ @pytest.mark.asyncio
+ async def test_handle_memory_save_fallback_reports_dropped_facts(self):
+ """A fallback that drops the payload must not report the facts as stored."""
+
+ class BasicBackend:
+ """Accepts neither pre-extraction fields nor `background`."""
+
+ async def save_memory(
+ self,
+ content: str,
+ user_id: str,
+ importance: float = 0.5,
+ entities: list[str] | None = None,
+ relationships: list[dict[str, str]] | None = None,
+ session_id: str | None = None,
+ metadata: dict[str, Any] | None = None,
+ ) -> Memory:
+ return Memory(content=content, user_id=user_id, importance=importance)
+
+ async def search_memories(self, *args: Any, **kwargs: Any) -> list[Memory]:
+ return []
+
+ async def update_memory(self, *args: Any, **kwargs: Any) -> Memory | None:
+ return None
+
+ async def delete_memory(self, *args: Any, **kwargs: Any) -> bool:
+ return False
+
+ async def get_memory(self, memory_id: str) -> Memory | None:
+ return None
+
+ system = MemorySystem(BasicBackend(), user_id="alice") # type: ignore[arg-type]
+ result = await system.handle_memory_save(
+ content="Summary",
+ importance=0.5,
+ facts=["A", "B"],
+ background=True,
+ )
+
+ assert result["success"] is True
+ assert result["optimized"] is False
+ assert result["facts_stored"] is False
+ assert result["fact_count"] == 0
+
+ @pytest.mark.asyncio
+ async def test_handle_memory_save_type_error_after_write_is_not_retried(self):
+ """A TypeError raised after the write propagates instead of saving twice.
+
+ The fallback only handles a signature mismatch; a backend that already
+ wrote the record and then raised TypeError would otherwise be called
+ again while the first payload is reported dropped (review finding)."""
+
+ class WritingBackend:
+ """Accepts the pre-extraction fields, writes, then raises TypeError."""
+
+ def __init__(self):
+ self.calls = 0
+
+ async def save_memory(self, *args: Any, **kwargs: Any) -> Memory:
+ self.calls += 1
+ raise TypeError("sqlite3.OperationalError wrapped as TypeError")
+
+ async def search_memories(self, *args: Any, **kwargs: Any) -> list[Memory]:
+ return []
+
+ async def update_memory(self, *args: Any, **kwargs: Any) -> Memory | None:
+ return None
+
+ async def delete_memory(self, *args: Any, **kwargs: Any) -> bool:
+ return False
+
+ async def get_memory(self, memory_id: str) -> Memory | None:
+ return None
+
+ backend = WritingBackend()
+ system = MemorySystem(backend, user_id="alice") # type: ignore[arg-type]
+ with pytest.raises(TypeError):
+ await system.handle_memory_save(
+ content="Summary",
+ importance=0.5,
+ facts=["A"],
+ background=True,
+ )
+ assert backend.calls == 1
+
+ @pytest.mark.asyncio
+ async def test_handle_memory_save_unsupported_fields_filtered_before_single_call(self):
+ """An older backend gets one call with only the fields it accepts."""
+
+ class BasicBackend:
+ """Accepts neither pre-extraction fields nor `background`."""
+
+ def __init__(self) -> None:
+ self.calls: list[dict[str, Any]] = []
+
+ async def save_memory(
+ self,
+ content: str,
+ user_id: str,
+ importance: float = 0.5,
+ entities: list[str] | None = None,
+ relationships: list[dict[str, str]] | None = None,
+ session_id: str | None = None,
+ metadata: dict[str, Any] | None = None,
+ ) -> Memory:
+ self.calls.append({"content": content, "metadata": metadata})
+ return Memory(content=content, user_id=user_id, importance=importance)
+
+ async def search_memories(self, *args: Any, **kwargs: Any) -> list[Memory]:
+ return []
+
+ async def update_memory(self, *args: Any, **kwargs: Any) -> Memory | None:
+ return None
+
+ async def delete_memory(self, *args: Any, **kwargs: Any) -> bool:
+ return False
+
+ async def get_memory(self, memory_id: str) -> Memory | None:
+ return None
+
+ backend = BasicBackend()
+ system = MemorySystem(backend, user_id="alice") # type: ignore[arg-type]
+ result = await system.handle_memory_save(
+ content="Summary",
+ importance=0.5,
+ facts=["A", "B"],
+ background=True,
+ )
+
+ assert result["success"] is True
+ assert len(backend.calls) == 1, (
+ "field support must be decided before the call, not by retrying"
+ )
+
+ @pytest.mark.asyncio
+ async def test_handle_memory_save_internal_unexpected_kwarg_is_not_replayed(self):
+ """An internal 'unexpected keyword argument' TypeError after the write must not re-save.
+
+ A backend that accepts ``**kwargs`` can persist the record and then call
+ a zero-argument post-write hook with a keyword — Python raises exactly
+ ``TypeError: ... unexpected keyword argument ...``, the same text a
+ binding failure produces. The save must not be replayed: that would
+ produce a duplicate record (review finding on the PR).
+ """
+
+ class PostWriteHookBackend:
+ """Writes first, then fails in an internal post-write hook."""
+
+ def __init__(self) -> None:
+ self.writes: list[str] = []
+
+ async def save_memory(self, content: str, user_id: str, **kwargs: Any) -> Memory:
+ self.writes.append(content)
+ if kwargs.get("facts") is not None:
+ # Zero-argument hook called with a keyword: the normal
+ # Python-generated binding TypeError, raised after the write.
+ def post_write_hook() -> None: # pragma: no cover - never runs
+ ...
+
+ post_write_hook(unexpected=True) # type: ignore[call-arg]
+ return Memory(content=content, user_id=user_id)
+
+ async def search_memories(self, *args: Any, **kwargs: Any) -> list[Memory]:
+ return []
+
+ async def update_memory(self, *args: Any, **kwargs: Any) -> Memory | None:
+ return None
+
+ async def delete_memory(self, *args: Any, **kwargs: Any) -> bool:
+ return False
+
+ async def get_memory(self, memory_id: str) -> Memory | None:
+ return None
+
+ backend = PostWriteHookBackend()
+ system = MemorySystem(backend, user_id="alice") # type: ignore[arg-type]
+ with pytest.raises(TypeError, match="unexpected keyword argument"):
+ await system.handle_memory_save(
+ content="Summary",
+ importance=0.5,
+ facts=["A", "B"],
+ )
+ assert backend.writes == ["Summary"], "the save must never be replayed after a write"
+
@pytest.mark.asyncio
async def test_handle_memory_save_invalid_importance_high(self, memory_system):
"""Test saving a memory with importance > 1.0."""