Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
drop metadata requirement (#1712)
Brandon Hancock (bhancock_ai)c7c0647dd225
diff --git a/docs/concepts/knowledge.mdx b/docs/concepts/knowledge.mdxindex 06c2eb947..96200b356 100644--- a/docs/concepts/knowledge.mdx+++ b/docs/concepts/knowledge.mdx@@ -48,7 +48,6 @@ from crewai.knowledge.source.string_knowledge_source import StringKnowledgeSourccontent = "Users name is John. He is 30 years old and lives in San Francisco."string_source = StringKnowledgeSource(content=content,- metadata={"preference": "personal"})# Create an LLM with a temperature of 0 to ensure deterministic outputs@@ -74,10 +73,7 @@ crew = Crew(tasks=[task],verbose=True,process=Process.sequential,- knowledge={- "sources": [string_source],- "metadata": {"preference": "personal"}- }, # Enable knowledge by adding the sources here. You can also add more sources to the sources list.+ knowledge_sources=[string_source], # Enable knowledge by adding the sources here. You can also add more sources to the sources list.)result = crew.kickoff(inputs={"question": "What city does John live in and how old is he?"})@@ -85,17 +81,6 @@ result = crew.kickoff(inputs={"question": "What city does John live in and how o## Knowledge Configuration-### Metadata and Filtering--Knowledge sources support metadata for better organization and filtering. Metadata is used to filter the knowledge sources when querying the knowledge store.--```python Code-knowledge_source = StringKnowledgeSource(- content="Users name is John. He is 30 years old and lives in San Francisco.",- metadata={"preference": "personal"} # Metadata is used to filter the knowledge sources-)-```-### Chunking ConfigurationControl how content is split for processing by setting the chunk size and overlap.@@ -116,21 +101,28 @@ You can also configure the embedder for the knowledge store. This is useful if y...string_source = StringKnowledgeSource(content="Users name is John. He is 30 years old and lives in San Francisco.",- metadata={"preference": "personal"})crew = Crew(...- knowledge={- "sources": [string_source],- "metadata": {"preference": "personal"},- "embedder_config": {- "provider": "openai", # Default embedder provider; can be "ollama", "gemini", e.t.c.- "config": {"model": "text-embedding-3-small"} # Default embedder model; can be "mxbai-embed-large", "nomic-embed-tex", e.t.c.- },+ knowledge_sources=[string_source],+ embedder={+ "provider": "openai",+ "config": {"model": "text-embedding-3-small"},},)```+## Clearing Knowledge++If you need to clear the knowledge stored in CrewAI, you can use the `crewai reset-memories` command with the `--knowledge` option.++```bash Command+crewai reset-memories --knowledge+```++This is useful when you've updated your knowledge sources and want to ensure that the agents are using the most recent information.++## Custom Knowledge SourcesCrewAI allows you to create custom knowledge sources for any type of data by extending the `BaseKnowledgeSource` class. Let's create a practical example that fetches and processes space news articles.@@ -174,12 +166,12 @@ class SpaceNewsKnowledgeSource(BaseKnowledgeSource):formatted = "Space News Articles:\n\n"for article in articles:formatted += f"""-Title: {article['title']}-Published: {article['published_at']}-Summary: {article['summary']}-News Site: {article['news_site']}-URL: {article['url']}--------------------"""+ Title: {article['title']}+ Published: {article['published_at']}+ Summary: {article['summary']}+ News Site: {article['news_site']}+ URL: {article['url']}+ -------------------"""return formatteddef add(self) -> None:@@ -189,17 +181,12 @@ URL: {article['url']}chunks = self._chunk_text(text)self.chunks.extend(chunks)- self.save_documents(metadata={- "source": "space_news_api",- "timestamp": datetime.now().isoformat(),- "article_count": self.limit- })+ self._save_documents()# Create knowledge sourcerecent_news = SpaceNewsKnowledgeSource(api_endpoint="https://api.spaceflightnewsapi.net/v4/articles",limit=10,- metadata={"category": "recent_news", "source": "spaceflight_news"})# Create specialized agent@@ -265,7 +252,7 @@ The latest developments in space exploration, based on recent space news article- Implements three key methods:- `load_content()`: Fetches articles from the API- `_format_articles()`: Structures the articles into readable text- - `add()`: Processes and stores the content with metadata+ - `add()`: Processes and stores the content2. **Agent Configuration**:- Specialized role as a Space News Analyst@@ -299,14 +286,12 @@ You can customize the API query by modifying the endpoint URL:recent_news = SpaceNewsKnowledgeSource(api_endpoint="https://api.spaceflightnewsapi.net/v4/articles",limit=20, # Increase the number of articles- metadata={"category": "recent_news"})# Add search parametersrecent_news = SpaceNewsKnowledgeSource(api_endpoint="https://api.spaceflightnewsapi.net/v4/articles?search=NASA", # Search for NASA newslimit=10,- metadata={"category": "nasa_news"})```@@ -314,16 +299,14 @@ recent_news = SpaceNewsKnowledgeSource(<AccordionGroup><Accordion title="Content Organization">- - Use descriptive metadata for better filtering- Keep chunk sizes appropriate for your content type- Consider content overlap for context preservation- Organize related information into separate knowledge sources</Accordion><Accordion title="Performance Tips">- - Use metadata filtering to narrow search scope- Adjust chunk sizes based on content complexity- Configure appropriate embedding models- Consider using local embedding providers for faster processing</Accordion>-</AccordionGroup>\ No newline at end of file+</AccordionGroup>diff --git a/src/crewai/knowledge/knowledge.py b/src/crewai/knowledge/knowledge.pyindex 4a1331799..f9f55a517 100644--- a/src/crewai/knowledge/knowledge.py+++ b/src/crewai/knowledge/knowledge.py@@ -1,11 +1,10 @@import os+from typing import Any, Dict, List, Optional-from typing import List, Optional, Dict, Anyfrom pydantic import BaseModel, ConfigDict, Fieldfrom crewai.knowledge.source.base_knowledge_source import BaseKnowledgeSourcefrom crewai.knowledge.storage.knowledge_storage import KnowledgeStorage-from crewai.utilities.constants import DEFAULT_SCORE_THRESHOLDos.environ["TOKENIZERS_PARALLELISM"] = "false" # removes logging from fastembed@@ -46,9 +45,7 @@ class Knowledge(BaseModel):source.storage = self.storagesource.add()- def query(- self, query: List[str], limit: int = 3, preference: Optional[str] = None- ) -> List[Dict[str, Any]]:+ def query(self, query: List[str], limit: int = 3) -> List[Dict[str, Any]]:"""Query across all knowledge sources to find the most relevant information.Returns the top_k most relevant chunks.@@ -57,8 +54,6 @@ class Knowledge(BaseModel):results = self.storage.search(query,limit,- filter={"preference": preference} if preference else None,- score_threshold=DEFAULT_SCORE_THRESHOLD,)return resultsdiff --git a/src/crewai/knowledge/source/base_file_knowledge_source.py b/src/crewai/knowledge/source/base_file_knowledge_source.pyindex 9cb9ec2a2..7f078b4a4 100644--- a/src/crewai/knowledge/source/base_file_knowledge_source.py+++ b/src/crewai/knowledge/source/base_file_knowledge_source.py@@ -1,13 +1,13 @@from abc import ABC, abstractmethodfrom pathlib import Path-from typing import Union, List, Dict, Any+from typing import Dict, List, Unionfrom pydantic import Fieldfrom crewai.knowledge.source.base_knowledge_source import BaseKnowledgeSource-from crewai.utilities.logger import Loggerfrom crewai.knowledge.storage.knowledge_storage import KnowledgeStoragefrom crewai.utilities.constants import KNOWLEDGE_DIRECTORY+from crewai.utilities.logger import Loggerclass BaseFileKnowledgeSource(BaseKnowledgeSource, ABC):@@ -49,10 +49,9 @@ class BaseFileKnowledgeSource(BaseKnowledgeSource, ABC):color="red",)- def save_documents(self, metadata: Dict[str, Any]):+ def _save_documents(self):"""Save the documents to the storage."""- chunk_metadatas = [metadata.copy() for _ in self.chunks]- self.storage.save(self.chunks, chunk_metadatas)+ self.storage.save(self.chunks)def convert_to_path(self, path: Union[Path, str]) -> Path:"""Convert a path to a Path object."""diff --git a/src/crewai/knowledge/source/base_knowledge_source.py b/src/crewai/knowledge/source/base_knowledge_source.pyindex 6be76ca40..07a9a0f25 100644--- a/src/crewai/knowledge/source/base_knowledge_source.py+++ b/src/crewai/knowledge/source/base_knowledge_source.py@@ -1,5 +1,5 @@from abc import ABC, abstractmethod-from typing import List, Dict, Any, Optional+from typing import Any, Dict, List, Optionalimport numpy as npfrom pydantic import BaseModel, ConfigDict, Field@@ -17,7 +17,7 @@ class BaseKnowledgeSource(BaseModel, ABC):model_config = ConfigDict(arbitrary_types_allowed=True)storage: KnowledgeStorage = Field(default_factory=KnowledgeStorage)- metadata: Dict[str, Any] = Field(default_factory=dict)+ metadata: Dict[str, Any] = Field(default_factory=dict) # Currently unusedcollection_name: Optional[str] = Field(default=None)@abstractmethod@@ -41,9 +41,9 @@ class BaseKnowledgeSource(BaseModel, ABC):for i in range(0, len(text), self.chunk_size - self.chunk_overlap)]- def save_documents(self, metadata: Dict[str, Any]):+ def _save_documents(self):"""Save the documents to the storage.This method should be called after the chunks and embeddings are generated."""- self.storage.save(self.chunks, metadata)+ self.storage.save(self.chunks)diff --git a/src/crewai/knowledge/source/csv_knowledge_source.py b/src/crewai/knowledge/source/csv_knowledge_source.pyindex 117cbe32f..3bb0714d9 100644--- a/src/crewai/knowledge/source/csv_knowledge_source.py+++ b/src/crewai/knowledge/source/csv_knowledge_source.py@@ -1,6 +1,6 @@import csv-from typing import Dict, Listfrom pathlib import Path+from typing import Dict, Listfrom crewai.knowledge.source.base_file_knowledge_source import BaseFileKnowledgeSource@@ -30,7 +30,7 @@ class CSVKnowledgeSource(BaseFileKnowledgeSource):)new_chunks = self._chunk_text(content_str)self.chunks.extend(new_chunks)- self.save_documents(metadata=self.metadata)+ self._save_documents()def _chunk_text(self, text: str) -> List[str]:"""Utility method to split text into chunks."""diff --git a/src/crewai/knowledge/source/excel_knowledge_source.py b/src/crewai/knowledge/source/excel_knowledge_source.pyindex 5484e59d5..d379c2763 100644--- a/src/crewai/knowledge/source/excel_knowledge_source.py+++ b/src/crewai/knowledge/source/excel_knowledge_source.py@@ -1,5 +1,6 @@-from typing import Dict, Listfrom pathlib import Path+from typing import Dict, List+from crewai.knowledge.source.base_file_knowledge_source import BaseFileKnowledgeSource@@ -44,7 +45,7 @@ class ExcelKnowledgeSource(BaseFileKnowledgeSource):new_chunks = self._chunk_text(content_str)self.chunks.extend(new_chunks)- self.save_documents(metadata=self.metadata)+ self._save_documents()def _chunk_text(self, text: str) -> List[str]:"""Utility method to split text into chunks."""diff --git a/src/crewai/knowledge/source/json_knowledge_source.py b/src/crewai/knowledge/source/json_knowledge_source.pyindex ed6eec86d..b02d438e6 100644--- a/src/crewai/knowledge/source/json_knowledge_source.py+++ b/src/crewai/knowledge/source/json_knowledge_source.py@@ -1,6 +1,6 @@import json-from typing import Any, Dict, Listfrom pathlib import Path+from typing import Any, Dict, Listfrom crewai.knowledge.source.base_file_knowledge_source import BaseFileKnowledgeSource@@ -42,7 +42,7 @@ class JSONKnowledgeSource(BaseFileKnowledgeSource):)new_chunks = self._chunk_text(content_str)self.chunks.extend(new_chunks)- self.save_documents(metadata=self.metadata)+ self._save_documents()def _chunk_text(self, text: str) -> List[str]:"""Utility method to split text into chunks."""diff --git a/src/crewai/knowledge/source/pdf_knowledge_source.py b/src/crewai/knowledge/source/pdf_knowledge_source.pyindex e6733d78b..38cd67807 100644--- a/src/crewai/knowledge/source/pdf_knowledge_source.py+++ b/src/crewai/knowledge/source/pdf_knowledge_source.py@@ -1,5 +1,5 @@-from typing import List, Dictfrom pathlib import Path+from typing import Dict, Listfrom crewai.knowledge.source.base_file_knowledge_source import BaseFileKnowledgeSource@@ -43,7 +43,7 @@ class PDFKnowledgeSource(BaseFileKnowledgeSource):for _, text in self.content.items():new_chunks = self._chunk_text(text)self.chunks.extend(new_chunks)- self.save_documents(metadata=self.metadata)+ self._save_documents()def _chunk_text(self, text: str) -> List[str]:"""Utility method to split text into chunks."""diff --git a/src/crewai/knowledge/source/string_knowledge_source.py b/src/crewai/knowledge/source/string_knowledge_source.pyindex 7336fd3ea..1d40ac4b8 100644--- a/src/crewai/knowledge/source/string_knowledge_source.py+++ b/src/crewai/knowledge/source/string_knowledge_source.py@@ -24,7 +24,7 @@ class StringKnowledgeSource(BaseKnowledgeSource):"""Add string content to the knowledge source, chunk it, compute embeddings, and save them."""new_chunks = self._chunk_text(self.content)self.chunks.extend(new_chunks)- self.save_documents(metadata=self.metadata)+ self._save_documents()def _chunk_text(self, text: str) -> List[str]:"""Utility method to split text into chunks."""diff --git a/src/crewai/knowledge/source/text_file_knowledge_source.py b/src/crewai/knowledge/source/text_file_knowledge_source.pyindex 1ceb02843..ddb1f2516 100644--- a/src/crewai/knowledge/source/text_file_knowledge_source.py+++ b/src/crewai/knowledge/source/text_file_knowledge_source.py@@ -1,5 +1,5 @@-from typing import Dict, Listfrom pathlib import Path+from typing import Dict, Listfrom crewai.knowledge.source.base_file_knowledge_source import BaseFileKnowledgeSource@@ -24,7 +24,7 @@ class TextFileKnowledgeSource(BaseFileKnowledgeSource):for _, text in self.content.items():new_chunks = self._chunk_text(text)self.chunks.extend(new_chunks)- self.save_documents(metadata=self.metadata)+ self._save_documents()def _chunk_text(self, text: str) -> List[str]:"""Utility method to split text into chunks."""diff --git a/src/crewai/knowledge/storage/knowledge_storage.py b/src/crewai/knowledge/storage/knowledge_storage.pyindex 79aa385a8..e0e36f4a7 100644--- a/src/crewai/knowledge/storage/knowledge_storage.py+++ b/src/crewai/knowledge/storage/knowledge_storage.py@@ -1,18 +1,20 @@import contextlib+import hashlibimport ioimport logging-import chromadbimport os+from typing import Any, Dict, List, Optional, Union, cast+import chromadbimport chromadb.errors-from crewai.utilities.paths import db_storage_path-from typing import Optional, List, Dict, Any, Union-from crewai.utilities import EmbeddingConfigurator-from crewai.knowledge.storage.base_knowledge_storage import BaseKnowledgeStorage-import hashlib-from chromadb.config import Settingsfrom chromadb.api import ClientAPI+from chromadb.api.types import OneOrMany+from chromadb.config import Settings++from crewai.knowledge.storage.base_knowledge_storage import BaseKnowledgeStorage+from crewai.utilities import EmbeddingConfiguratorfrom crewai.utilities.logger import Logger+from crewai.utilities.paths import db_storage_path@contextlib.contextmanager@@ -116,11 +118,16 @@ class KnowledgeStorage(BaseKnowledgeStorage):def save(self,documents: List[str],- metadata: Union[Dict[str, Any], List[Dict[str, Any]]],+ metadata: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,):if self.collection:try:- metadatas = [metadata] if isinstance(metadata, dict) else metadata+ if metadata is None:+ metadatas: Optional[OneOrMany[chromadb.Metadata]] = None+ elif isinstance(metadata, list):+ metadatas = [cast(chromadb.Metadata, m) for m in metadata]+ else:+ metadatas = cast(chromadb.Metadata, metadata)ids = [hashlib.sha256(doc.encode("utf-8")).hexdigest() for doc in documents