Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

drop metadata requirement (#1712)

Brandon Hancock (bhancock_ai)c7c0647dd225
diff --git a/docs/concepts/knowledge.mdx b/docs/concepts/knowledge.mdx
index 06c2eb947..96200b356 100644
--- a/docs/concepts/knowledge.mdx
+++ b/docs/concepts/knowledge.mdx
@@ -48,7 +48,6 @@ from crewai.knowledge.source.string_knowledge_source import StringKnowledgeSourc
content = "Users name is John. He is 30 years old and lives in San Francisco."
string_source = StringKnowledgeSource(
content=content,
- metadata={"preference": "personal"}
)
# Create an LLM with a temperature of 0 to ensure deterministic outputs
@@ -74,10 +73,7 @@ crew = Crew(
tasks=[task],
verbose=True,
process=Process.sequential,
- knowledge={
- "sources": [string_source],
- "metadata": {"preference": "personal"}
- }, # Enable knowledge by adding the sources here. You can also add more sources to the sources list.
+ knowledge_sources=[string_source], # Enable knowledge by adding the sources here. You can also add more sources to the sources list.
)
result = crew.kickoff(inputs={"question": "What city does John live in and how old is he?"})
@@ -85,17 +81,6 @@ result = crew.kickoff(inputs={"question": "What city does John live in and how o
## Knowledge Configuration
-### Metadata and Filtering
-
-Knowledge sources support metadata for better organization and filtering. Metadata is used to filter the knowledge sources when querying the knowledge store.
-
-```python Code
-knowledge_source = StringKnowledgeSource(
- content="Users name is John. He is 30 years old and lives in San Francisco.",
- metadata={"preference": "personal"} # Metadata is used to filter the knowledge sources
-)
-```
-
### Chunking Configuration
Control how content is split for processing by setting the chunk size and overlap.
@@ -116,21 +101,28 @@ You can also configure the embedder for the knowledge store. This is useful if y
...
string_source = StringKnowledgeSource(
content="Users name is John. He is 30 years old and lives in San Francisco.",
- metadata={"preference": "personal"}
)
crew = Crew(
...
- knowledge={
- "sources": [string_source],
- "metadata": {"preference": "personal"},
- "embedder_config": {
- "provider": "openai", # Default embedder provider; can be "ollama", "gemini", e.t.c.
- "config": {"model": "text-embedding-3-small"} # Default embedder model; can be "mxbai-embed-large", "nomic-embed-tex", e.t.c.
- },
+ knowledge_sources=[string_source],
+ embedder={
+ "provider": "openai",
+ "config": {"model": "text-embedding-3-small"},
},
)
```
+## Clearing Knowledge
+
+If you need to clear the knowledge stored in CrewAI, you can use the `crewai reset-memories` command with the `--knowledge` option.
+
+```bash Command
+crewai reset-memories --knowledge
+```
+
+This is useful when you've updated your knowledge sources and want to ensure that the agents are using the most recent information.
+
+
## Custom Knowledge Sources
CrewAI allows you to create custom knowledge sources for any type of data by extending the `BaseKnowledgeSource` class. Let's create a practical example that fetches and processes space news articles.
@@ -174,12 +166,12 @@ class SpaceNewsKnowledgeSource(BaseKnowledgeSource):
formatted = "Space News Articles:\n\n"
for article in articles:
formatted += f"""
-Title: {article['title']}
-Published: {article['published_at']}
-Summary: {article['summary']}
-News Site: {article['news_site']}
-URL: {article['url']}
--------------------"""
+ Title: {article['title']}
+ Published: {article['published_at']}
+ Summary: {article['summary']}
+ News Site: {article['news_site']}
+ URL: {article['url']}
+ -------------------"""
return formatted
def add(self) -> None:
@@ -189,17 +181,12 @@ URL: {article['url']}
chunks = self._chunk_text(text)
self.chunks.extend(chunks)
- self.save_documents(metadata={
- "source": "space_news_api",
- "timestamp": datetime.now().isoformat(),
- "article_count": self.limit
- })
+ self._save_documents()
# Create knowledge source
recent_news = SpaceNewsKnowledgeSource(
api_endpoint="https://api.spaceflightnewsapi.net/v4/articles",
limit=10,
- metadata={"category": "recent_news", "source": "spaceflight_news"}
)
# Create specialized agent
@@ -265,7 +252,7 @@ The latest developments in space exploration, based on recent space news article
- Implements three key methods:
- `load_content()`: Fetches articles from the API
- `_format_articles()`: Structures the articles into readable text
- - `add()`: Processes and stores the content with metadata
+ - `add()`: Processes and stores the content
2. **Agent Configuration**:
- Specialized role as a Space News Analyst
@@ -299,14 +286,12 @@ You can customize the API query by modifying the endpoint URL:
recent_news = SpaceNewsKnowledgeSource(
api_endpoint="https://api.spaceflightnewsapi.net/v4/articles",
limit=20, # Increase the number of articles
- metadata={"category": "recent_news"}
)
# Add search parameters
recent_news = SpaceNewsKnowledgeSource(
api_endpoint="https://api.spaceflightnewsapi.net/v4/articles?search=NASA", # Search for NASA news
limit=10,
- metadata={"category": "nasa_news"}
)
```
@@ -314,16 +299,14 @@ recent_news = SpaceNewsKnowledgeSource(
<AccordionGroup>
<Accordion title="Content Organization">
- - Use descriptive metadata for better filtering
- Keep chunk sizes appropriate for your content type
- Consider content overlap for context preservation
- Organize related information into separate knowledge sources
</Accordion>
<Accordion title="Performance Tips">
- - Use metadata filtering to narrow search scope
- Adjust chunk sizes based on content complexity
- Configure appropriate embedding models
- Consider using local embedding providers for faster processing
</Accordion>
-</AccordionGroup>
\ No newline at end of file
+</AccordionGroup>
diff --git a/src/crewai/knowledge/knowledge.py b/src/crewai/knowledge/knowledge.py
index 4a1331799..f9f55a517 100644
--- a/src/crewai/knowledge/knowledge.py
+++ b/src/crewai/knowledge/knowledge.py
@@ -1,11 +1,10 @@
import os
+from typing import Any, Dict, List, Optional
-from typing import List, Optional, Dict, Any
from pydantic import BaseModel, ConfigDict, Field
from crewai.knowledge.source.base_knowledge_source import BaseKnowledgeSource
from crewai.knowledge.storage.knowledge_storage import KnowledgeStorage
-from crewai.utilities.constants import DEFAULT_SCORE_THRESHOLD
os.environ["TOKENIZERS_PARALLELISM"] = "false" # removes logging from fastembed
@@ -46,9 +45,7 @@ class Knowledge(BaseModel):
source.storage = self.storage
source.add()
- def query(
- self, query: List[str], limit: int = 3, preference: Optional[str] = None
- ) -> List[Dict[str, Any]]:
+ def query(self, query: List[str], limit: int = 3) -> List[Dict[str, Any]]:
"""
Query across all knowledge sources to find the most relevant information.
Returns the top_k most relevant chunks.
@@ -57,8 +54,6 @@ class Knowledge(BaseModel):
results = self.storage.search(
query,
limit,
- filter={"preference": preference} if preference else None,
- score_threshold=DEFAULT_SCORE_THRESHOLD,
)
return results
diff --git a/src/crewai/knowledge/source/base_file_knowledge_source.py b/src/crewai/knowledge/source/base_file_knowledge_source.py
index 9cb9ec2a2..7f078b4a4 100644
--- a/src/crewai/knowledge/source/base_file_knowledge_source.py
+++ b/src/crewai/knowledge/source/base_file_knowledge_source.py
@@ -1,13 +1,13 @@
from abc import ABC, abstractmethod
from pathlib import Path
-from typing import Union, List, Dict, Any
+from typing import Dict, List, Union
from pydantic import Field
from crewai.knowledge.source.base_knowledge_source import BaseKnowledgeSource
-from crewai.utilities.logger import Logger
from crewai.knowledge.storage.knowledge_storage import KnowledgeStorage
from crewai.utilities.constants import KNOWLEDGE_DIRECTORY
+from crewai.utilities.logger import Logger
class BaseFileKnowledgeSource(BaseKnowledgeSource, ABC):
@@ -49,10 +49,9 @@ class BaseFileKnowledgeSource(BaseKnowledgeSource, ABC):
color="red",
)
- def save_documents(self, metadata: Dict[str, Any]):
+ def _save_documents(self):
"""Save the documents to the storage."""
- chunk_metadatas = [metadata.copy() for _ in self.chunks]
- self.storage.save(self.chunks, chunk_metadatas)
+ self.storage.save(self.chunks)
def convert_to_path(self, path: Union[Path, str]) -> Path:
"""Convert a path to a Path object."""
diff --git a/src/crewai/knowledge/source/base_knowledge_source.py b/src/crewai/knowledge/source/base_knowledge_source.py
index 6be76ca40..07a9a0f25 100644
--- a/src/crewai/knowledge/source/base_knowledge_source.py
+++ b/src/crewai/knowledge/source/base_knowledge_source.py
@@ -1,5 +1,5 @@
from abc import ABC, abstractmethod
-from typing import List, Dict, Any, Optional
+from typing import Any, Dict, List, Optional
import numpy as np
from pydantic import BaseModel, ConfigDict, Field
@@ -17,7 +17,7 @@ class BaseKnowledgeSource(BaseModel, ABC):
model_config = ConfigDict(arbitrary_types_allowed=True)
storage: KnowledgeStorage = Field(default_factory=KnowledgeStorage)
- metadata: Dict[str, Any] = Field(default_factory=dict)
+ metadata: Dict[str, Any] = Field(default_factory=dict) # Currently unused
collection_name: Optional[str] = Field(default=None)
@abstractmethod
@@ -41,9 +41,9 @@ class BaseKnowledgeSource(BaseModel, ABC):
for i in range(0, len(text), self.chunk_size - self.chunk_overlap)
]
- def save_documents(self, metadata: Dict[str, Any]):
+ def _save_documents(self):
"""
Save the documents to the storage.
This method should be called after the chunks and embeddings are generated.
"""
- self.storage.save(self.chunks, metadata)
+ self.storage.save(self.chunks)
diff --git a/src/crewai/knowledge/source/csv_knowledge_source.py b/src/crewai/knowledge/source/csv_knowledge_source.py
index 117cbe32f..3bb0714d9 100644
--- a/src/crewai/knowledge/source/csv_knowledge_source.py
+++ b/src/crewai/knowledge/source/csv_knowledge_source.py
@@ -1,6 +1,6 @@
import csv
-from typing import Dict, List
from pathlib import Path
+from typing import Dict, List
from crewai.knowledge.source.base_file_knowledge_source import BaseFileKnowledgeSource
@@ -30,7 +30,7 @@ class CSVKnowledgeSource(BaseFileKnowledgeSource):
)
new_chunks = self._chunk_text(content_str)
self.chunks.extend(new_chunks)
- self.save_documents(metadata=self.metadata)
+ self._save_documents()
def _chunk_text(self, text: str) -> List[str]:
"""Utility method to split text into chunks."""
diff --git a/src/crewai/knowledge/source/excel_knowledge_source.py b/src/crewai/knowledge/source/excel_knowledge_source.py
index 5484e59d5..d379c2763 100644
--- a/src/crewai/knowledge/source/excel_knowledge_source.py
+++ b/src/crewai/knowledge/source/excel_knowledge_source.py
@@ -1,5 +1,6 @@
-from typing import Dict, List
from pathlib import Path
+from typing import Dict, List
+
from crewai.knowledge.source.base_file_knowledge_source import BaseFileKnowledgeSource
@@ -44,7 +45,7 @@ class ExcelKnowledgeSource(BaseFileKnowledgeSource):
new_chunks = self._chunk_text(content_str)
self.chunks.extend(new_chunks)
- self.save_documents(metadata=self.metadata)
+ self._save_documents()
def _chunk_text(self, text: str) -> List[str]:
"""Utility method to split text into chunks."""
diff --git a/src/crewai/knowledge/source/json_knowledge_source.py b/src/crewai/knowledge/source/json_knowledge_source.py
index ed6eec86d..b02d438e6 100644
--- a/src/crewai/knowledge/source/json_knowledge_source.py
+++ b/src/crewai/knowledge/source/json_knowledge_source.py
@@ -1,6 +1,6 @@
import json
-from typing import Any, Dict, List
from pathlib import Path
+from typing import Any, Dict, List
from crewai.knowledge.source.base_file_knowledge_source import BaseFileKnowledgeSource
@@ -42,7 +42,7 @@ class JSONKnowledgeSource(BaseFileKnowledgeSource):
)
new_chunks = self._chunk_text(content_str)
self.chunks.extend(new_chunks)
- self.save_documents(metadata=self.metadata)
+ self._save_documents()
def _chunk_text(self, text: str) -> List[str]:
"""Utility method to split text into chunks."""
diff --git a/src/crewai/knowledge/source/pdf_knowledge_source.py b/src/crewai/knowledge/source/pdf_knowledge_source.py
index e6733d78b..38cd67807 100644
--- a/src/crewai/knowledge/source/pdf_knowledge_source.py
+++ b/src/crewai/knowledge/source/pdf_knowledge_source.py
@@ -1,5 +1,5 @@
-from typing import List, Dict
from pathlib import Path
+from typing import Dict, List
from crewai.knowledge.source.base_file_knowledge_source import BaseFileKnowledgeSource
@@ -43,7 +43,7 @@ class PDFKnowledgeSource(BaseFileKnowledgeSource):
for _, text in self.content.items():
new_chunks = self._chunk_text(text)
self.chunks.extend(new_chunks)
- self.save_documents(metadata=self.metadata)
+ self._save_documents()
def _chunk_text(self, text: str) -> List[str]:
"""Utility method to split text into chunks."""
diff --git a/src/crewai/knowledge/source/string_knowledge_source.py b/src/crewai/knowledge/source/string_knowledge_source.py
index 7336fd3ea..1d40ac4b8 100644
--- a/src/crewai/knowledge/source/string_knowledge_source.py
+++ b/src/crewai/knowledge/source/string_knowledge_source.py
@@ -24,7 +24,7 @@ class StringKnowledgeSource(BaseKnowledgeSource):
"""Add string content to the knowledge source, chunk it, compute embeddings, and save them."""
new_chunks = self._chunk_text(self.content)
self.chunks.extend(new_chunks)
- self.save_documents(metadata=self.metadata)
+ self._save_documents()
def _chunk_text(self, text: str) -> List[str]:
"""Utility method to split text into chunks."""
diff --git a/src/crewai/knowledge/source/text_file_knowledge_source.py b/src/crewai/knowledge/source/text_file_knowledge_source.py
index 1ceb02843..ddb1f2516 100644
--- a/src/crewai/knowledge/source/text_file_knowledge_source.py
+++ b/src/crewai/knowledge/source/text_file_knowledge_source.py
@@ -1,5 +1,5 @@
-from typing import Dict, List
from pathlib import Path
+from typing import Dict, List
from crewai.knowledge.source.base_file_knowledge_source import BaseFileKnowledgeSource
@@ -24,7 +24,7 @@ class TextFileKnowledgeSource(BaseFileKnowledgeSource):
for _, text in self.content.items():
new_chunks = self._chunk_text(text)
self.chunks.extend(new_chunks)
- self.save_documents(metadata=self.metadata)
+ self._save_documents()
def _chunk_text(self, text: str) -> List[str]:
"""Utility method to split text into chunks."""
diff --git a/src/crewai/knowledge/storage/knowledge_storage.py b/src/crewai/knowledge/storage/knowledge_storage.py
index 79aa385a8..e0e36f4a7 100644
--- a/src/crewai/knowledge/storage/knowledge_storage.py
+++ b/src/crewai/knowledge/storage/knowledge_storage.py
@@ -1,18 +1,20 @@
import contextlib
+import hashlib
import io
import logging
-import chromadb
import os
+from typing import Any, Dict, List, Optional, Union, cast
+import chromadb
import chromadb.errors
-from crewai.utilities.paths import db_storage_path
-from typing import Optional, List, Dict, Any, Union
-from crewai.utilities import EmbeddingConfigurator
-from crewai.knowledge.storage.base_knowledge_storage import BaseKnowledgeStorage
-import hashlib
-from chromadb.config import Settings
from chromadb.api import ClientAPI
+from chromadb.api.types import OneOrMany
+from chromadb.config import Settings
+
+from crewai.knowledge.storage.base_knowledge_storage import BaseKnowledgeStorage
+from crewai.utilities import EmbeddingConfigurator
from crewai.utilities.logger import Logger
+from crewai.utilities.paths import db_storage_path
@contextlib.contextmanager
@@ -116,11 +118,16 @@ class KnowledgeStorage(BaseKnowledgeStorage):
def save(
self,
documents: List[str],
- metadata: Union[Dict[str, Any], List[Dict[str, Any]]],
+ metadata: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
):
if self.collection:
try:
- metadatas = [metadata] if isinstance(metadata, dict) else metadata
+ if metadata is None:
+ metadatas: Optional[OneOrMany[chromadb.Metadata]] = None
+ elif isinstance(metadata, list):
+ metadatas = [cast(chromadb.Metadata, m) for m in metadata]
+ else:
+ metadatas = cast(chromadb.Metadata, metadata)
ids = [
hashlib.sha256(doc.encode("utf-8")).hexdigest() for doc in documents