Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
fix(memory): escape scope and id values in LanceDB filters (#7820)
Craig Estep40bb2d3b586e
diff --git a/lib/crewai/src/crewai/memory/storage/lancedb_storage.py b/lib/crewai/src/crewai/memory/storage/lancedb_storage.pyindex 6a97900fd..23bfc354c 100644--- a/lib/crewai/src/crewai/memory/storage/lancedb_storage.py+++ b/lib/crewai/src/crewai/memory/storage/lancedb_storage.py@@ -40,6 +40,28 @@ _MAX_RETRIES = 5_RETRY_BASE_DELAY = 0.2 # seconds; doubles on each retry+def _sql_str(value: str) -> str:+ """Escape a value for use inside a single-quoted SQL string literal."""+ return value.replace("'", "''")+++def _id_in_filter(record_ids: list[str]) -> str:+ """Build an ``id IN (...)`` predicate that matches each ID literally."""+ ids_expr = ", ".join(f"'{_sql_str(str(rid))}'" for rid in record_ids)+ return f"id IN ({ids_expr})"+++def _scope_prefix_filter(prefix: str) -> str:+ r"""Build a predicate matching scopes that start with ``prefix``.++ ``\``, ``%`` and ``_`` in the prefix are escaped so they match literally;+ only the appended trailing ``%`` acts as a wildcard. Lance supports only+ ``\`` as the ``LIKE`` escape character.+ """+ pattern = prefix.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")+ return f"scope LIKE '{_sql_str(pattern)}%' ESCAPE '\\'"++class LanceDBStorage:"""LanceDB-backed storage for the unified memory system."""@@ -331,7 +353,7 @@ class LanceDBStorage:)with store_lock(self._lock_name):self._ensure_table()- safe_id = str(record.id).replace("'", "''")+ safe_id = _sql_str(str(record.id))self._do_write("delete", f"id = '{safe_id}'")row = self._record_to_row(record)if row["vector"] is None or len(row["vector"]) != self._vector_dim:@@ -352,11 +374,9 @@ class LanceDBStorage:returnwith store_lock(self._lock_name):now = datetime.utcnow().isoformat()- safe_ids = [str(rid).replace("'", "''") for rid in record_ids]- ids_expr = ", ".join(f"'{rid}'" for rid in safe_ids)self._do_write("update",- where=f"id IN ({ids_expr})",+ where=_id_in_filter(record_ids),values={"last_accessed": now},)@@ -364,7 +384,7 @@ class LanceDBStorage:"""Return a single record by ID, or None if not found."""if self._table is None:return None- safe_id = str(record_id).replace("'", "''")+ safe_id = _sql_str(str(record_id))rows = self._table.search().where(f"id = '{safe_id}'").limit(1).to_list()if not rows:return None@@ -387,9 +407,7 @@ class LanceDBStorage:)query = self._table.search(query_embedding)if scope_prefix is not None and scope_prefix.strip("/"):- prefix = scope_prefix.rstrip("/")- like_val = prefix + "%"- query = query.where(f"scope LIKE '{like_val}'")+ query = query.where(_scope_prefix_filter(scope_prefix.rstrip("/")))results = query.limit(limit * 3 if (categories or metadata_filter) else limit).to_list()@@ -423,8 +441,7 @@ class LanceDBStorage:with store_lock(self._lock_name):if record_ids and not (categories or metadata_filter):before = int(self._table.count_rows())- ids_expr = ", ".join(f"'{rid}'" for rid in record_ids)- self._do_write("delete", f"id IN ({ids_expr})")+ self._do_write("delete", _id_in_filter(record_ids))return before - int(self._table.count_rows())if categories or metadata_filter:rows = self._scan_rows(scope_prefix)@@ -445,15 +462,14 @@ class LanceDBStorage:if not to_delete:return 0before = int(self._table.count_rows())- ids_expr = ", ".join(f"'{rid}'" for rid in to_delete)- self._do_write("delete", f"id IN ({ids_expr})")+ self._do_write("delete", _id_in_filter(to_delete))return before - int(self._table.count_rows())conditions = []if scope_prefix is not None and scope_prefix.strip("/"):prefix = scope_prefix.rstrip("/")if not prefix.startswith("/"):prefix = "/" + prefix- conditions.append(f"scope LIKE '{prefix}%' OR scope = '/'")+ conditions.append(f"{_scope_prefix_filter(prefix)} OR scope = '/'")if older_than is not None:conditions.append(f"created_at < '{older_than.isoformat()}'")if not conditions:@@ -487,7 +503,7 @@ class LanceDBStorage:return []q = self._table.search()if scope_prefix is not None and scope_prefix.strip("/"):- q = q.where(f"scope LIKE '{scope_prefix.rstrip('/')}%'")+ q = q.where(_scope_prefix_filter(scope_prefix.rstrip("/")))if columns is not None:q = q.select(columns)result: list[dict[str, Any]] = q.limit(limit).to_list()@@ -613,8 +629,9 @@ class LanceDBStorage:returnprefix = scope_prefix.rstrip("/")if prefix:+ literal = _sql_str(prefix)self._do_write(- "delete", f"scope >= '{prefix}' AND scope < '{prefix}/\uffff'"+ "delete", f"scope >= '{literal}' AND scope < '{literal}/\uffff'")def optimize(self) -> None:diff --git a/lib/crewai/tests/memory/test_lancedb_filter_escaping.py b/lib/crewai/tests/memory/test_lancedb_filter_escaping.pynew file mode 100644index 000000000..acee27cf4--- /dev/null+++ b/lib/crewai/tests/memory/test_lancedb_filter_escaping.py@@ -0,0 +1,201 @@+"""Scope and record-id values must be matched literally in LanceDB filters.++LanceDB ``where()`` takes a raw SQL expression, so every caller-supplied value+has to be escaped before it is embedded. These tests run against a real+temporary LanceDB table with two tenants and check that crafted scope prefixes+and record ids cannot read, delete, or reset another tenant's records, and that+legitimate scope names (including ``-``, ``_``, ``/``, ``%``, quotes, and+backslashes) keep working.+"""++from __future__ import annotations++from collections.abc import Callable+from pathlib import Path+from unittest.mock import MagicMock++from crewai.memory.storage.lancedb_storage import LanceDBStorage+from crewai.memory.types import MemoryRecord+from crewai.memory.unified_memory import Memory+import pytest+++DIM = 4+TENANT_A = {"a1", "a2"}+TENANT_B = {"b1", "b2"}++CRAFTED_SCOPES = [+ pytest.param("/tenant-a' OR scope <> '", id="quote-breakout"),+ pytest.param("/tenant-a' OR scope LIKE '/tenant-b", id="predicate-injection"),+ pytest.param("%", id="bare-percent"),+ pytest.param("_", id="bare-underscore"),+ pytest.param("/tenant-_", id="underscore-wildcard"),+]+++def _record(+ record_id: str, scope: str, categories: list[str] | None = None+) -> MemoryRecord:+ return MemoryRecord(+ id=record_id,+ content=f"content of {record_id}",+ scope=scope,+ categories=categories or ["note"],+ embedding=[0.1] * DIM,+ )+++def _new_storage(path: Path, records: list[MemoryRecord]) -> LanceDBStorage:+ storage = LanceDBStorage(path=path, vector_dim=DIM, compact_every=0)+ storage.save(records)+ return storage+++def _all_ids(storage: LanceDBStorage) -> set[str]:+ return {r.id for r in storage.list_records(limit=1000)}+++def _search_ids(storage: LanceDBStorage, scope_prefix: str) -> set[str]:+ return {+ r.id+ for r, _ in storage.search([0.1] * DIM, scope_prefix=scope_prefix, limit=100)+ }+++def storage(tmp_path: Path) -> LanceDBStorage:+ return _new_storage(+ tmp_path / "mem",+ [+ _record("a1", "/tenant-a/agent_1"),+ _record("a2", "/tenant-a/notes"),+ _record("b1", "/tenant-b/agent_1"),+ _record("b2", "/tenant-b/secrets"),+ ],+ )+++def test_control_scope_prefix_stays_in_tenant(storage: LanceDBStorage) -> None:+ assert _search_ids(storage, "/tenant-a") == TENANT_A+ assert {r.id for r in storage.list_records("/tenant-a")} == TENANT_A+ assert storage.count("/tenant-a") == 2++[email protected]("payload", CRAFTED_SCOPES)+def test_crafted_scope_prefix_does_not_read_other_tenant(+ storage: LanceDBStorage, payload: str+) -> None:+ assert _search_ids(storage, payload) <= TENANT_A+ assert {r.id for r in storage.list_records(payload)} <= TENANT_A+ assert storage.count(payload) <= len(TENANT_A)++[email protected]("payload", CRAFTED_SCOPES)+ "remove",+ [+ pytest.param(lambda s, p: s.delete(scope_prefix=p), id="delete"),+ pytest.param(+ lambda s, p: s.delete(scope_prefix=p, categories=["note"]),+ id="delete-by-category",+ ),+ pytest.param(lambda s, p: s.reset(scope_prefix=p), id="reset"),+ ],+)+def test_crafted_scope_prefix_does_not_remove_other_tenant(+ storage: LanceDBStorage,+ payload: str,+ remove: Callable[[LanceDBStorage, str], object],+) -> None:+ remove(storage, payload)+ assert _all_ids(storage) >= TENANT_B+++def test_crafted_record_id_does_not_over_delete(storage: LanceDBStorage) -> None:+ deleted = storage.delete(record_ids=["a1') OR id <> ('"])+ assert deleted == 0+ assert _all_ids(storage) == TENANT_A | TENANT_B+++def test_record_ids_with_quotes_are_matched_literally(tmp_path: Path) -> None:+ storage = _new_storage(+ tmp_path / "mem",+ [+ _record("o'brien-1", "/people/o'brien", ["x"]),+ _record("o'brien-2", "/people/o'brien", ["x"]),+ _record("other", "/people/other", ["x"]),+ ],+ )+ deleted_by_id = storage.delete(record_ids=["o'brien-1"])+ assert deleted_by_id == 1+ deleted_by_scope = storage.delete(scope_prefix="/people/o'brien", categories=["x"])+ assert deleted_by_scope == 1+ assert _all_ids(storage) == {"other"}+++def test_memory_root_scope_confines_crafted_explicit_scope(+ storage: LanceDBStorage,+) -> None:+ embedder = MagicMock(side_effect=lambda texts: [[0.1] * DIM for _ in texts])+ memory = Memory(+ storage=storage, root_scope="/tenant-a", llm=MagicMock(), embedder=embedder+ )++ for payload in ("x' OR scope <> '", "x' OR scope LIKE '/tenant-b"):+ matches = memory.recall("anything", scope=payload, depth="shallow")+ assert {m.record.id for m in matches} <= TENANT_A+ memory.forget(scope=payload)+ assert _all_ids(storage) == TENANT_A | TENANT_B++ assert {m.record.id for m in memory.recall("anything", depth="shallow")} == TENANT_A+++def test_legitimate_scope_characters_still_match(tmp_path: Path) -> None:+ storage = _new_storage(+ tmp_path / "mem",+ [+ _record("senior", "/crew/research-crew/agent/senior_analyst"),+ _record("junior", "/crew/research-crew/agent/junior-analyst"),+ _record("underscore", "/crew/research_crew"),+ _record("lookalike", "/crew/researchXcrew"),+ ],+ )+ assert _search_ids(storage, "/crew/research-crew") == {"senior", "junior"}+ assert _search_ids(storage, "/crew/research-crew/agent/senior_analyst") == {+ "senior"+ }+ assert storage.get_scope_info("/crew/research-crew").record_count == 2+ assert storage.list_scopes("/crew/research-crew") == ["/crew/research-crew/agent"]+ assert storage.list_categories("/crew/research-crew") == {"note": 2}++ deleted = storage.delete(scope_prefix="/crew/research_crew")+ assert deleted == 1+ assert _all_ids(storage) == {"senior", "junior", "lookalike"}++ storage.reset(scope_prefix="/crew/research-crew")+ assert _all_ids(storage) == {"lookalike"}+++def test_percent_underscore_and_backslash_in_scope_names_match_literally(+ tmp_path: Path,+) -> None:+ storage = _new_storage(+ tmp_path / "mem",+ [+ _record("pct", "/metrics/100%"),+ _record("pct-child", "/metrics/100%/daily"),+ _record("digits", "/metrics/1000"),+ _record("underscore", "/a_b"),+ _record("lookalike", "/axb"),+ _record("backslash", "/share\\docs"),+ ],+ )+ assert _search_ids(storage, "/metrics/100%") == {"pct", "pct-child"}+ assert {r.id for r in storage.list_records("/metrics/100%")} == {"pct", "pct-child"}+ assert _search_ids(storage, "/a_b") == {"underscore"}+ assert storage.count("/a_b") == 1+ assert _search_ids(storage, "/share\\") == {"backslash"}++ deleted = storage.delete(scope_prefix="/a_b")+ assert deleted == 1+ assert _all_ids(storage) == {"pct", "pct-child", "digits", "lookalike", "backslash"}