Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
Knowledge project directory standard (#1691)
Lorenze Jay1af95f514675
diff --git a/docs/concepts/knowledge.mdx b/docs/concepts/knowledge.mdxindex 69fa4e644..a00b2c2f0 100644--- a/docs/concepts/knowledge.mdx+++ b/docs/concepts/knowledge.mdx@@ -156,14 +156,35 @@ crew = Crew(agents=[agent],tasks=[task],knowledge_sources=[source],- embedder_config={- "model": "BAAI/bge-small-en-v1.5",- "normalize": True,- "max_length": 512+ embedder={+ "provider": "ollama",+ "config": {"model": "nomic-embed-text:latest"},})```+### Referencing Sources++You can reference knowledge sources by their collection name or metadata.++* Add a directory to your crew project called `knowledge`:+* File paths in knowledge can be referenced relative to the `knowledge` directory.++Example:+A file inside the `knowledge` directory called `example.txt` can be referenced as `example.txt`.++```python+source = TextFileKnowledgeSource(+ file_path="example.txt", # or /example.txt+ collection_name="example"+)+crew = Crew(+ agents=[agent],+ tasks=[task],+ knowledge_sources=[source],+)+```+## Best Practices<AccordionGroup>diff --git a/src/crewai/cli/create_crew.py b/src/crewai/cli/create_crew.pyindex 06440d74e..c658b0de1 100644--- a/src/crewai/cli/create_crew.py+++ b/src/crewai/cli/create_crew.py@@ -39,6 +39,7 @@ def create_folder_structure(name, parent_folder=None):folder_path.mkdir(parents=True)(folder_path / "tests").mkdir(exist_ok=True)+ (folder_path / "knowledge").mkdir(exist_ok=True)if not parent_folder:(folder_path / "src" / folder_name).mkdir(parents=True)(folder_path / "src" / folder_name / "tools").mkdir(parents=True)@@ -52,7 +53,14 @@ def copy_template_files(folder_path, name, class_name, parent_folder):templates_dir = package_dir / "templates" / "crew"root_template_files = (- [".gitignore", "pyproject.toml", "README.md"] if not parent_folder else []+ [+ ".gitignore",+ "pyproject.toml",+ "README.md",+ "knowledge/user_preference.txt",+ ]+ if not parent_folder+ else [])tools_template_files = ["tools/custom_tool.py", "tools/__init__.py"]config_template_files = ["config/agents.yaml", "config/tasks.yaml"]@@ -168,7 +176,9 @@ def create_crew(name, provider=None, skip_provider=False, parent_folder=None):templates_dir = package_dir / "templates" / "crew"root_template_files = (- [".gitignore", "pyproject.toml", "README.md"] if not parent_folder else []+ [".gitignore", "pyproject.toml", "README.md", "knowledge/user_preference.txt"]+ if not parent_folder+ else [])tools_template_files = ["tools/custom_tool.py", "tools/__init__.py"]config_template_files = ["config/agents.yaml", "config/tasks.yaml"]diff --git a/src/crewai/cli/templates/crew/crew.py b/src/crewai/cli/templates/crew/crew.pyindex 6f8e66c4a..0a8c7481a 100644--- a/src/crewai/cli/templates/crew/crew.py+++ b/src/crewai/cli/templates/crew/crew.py@@ -1,8 +1,9 @@from crewai import Agent, Crew, Process, Taskfrom crewai.project import CrewBase, agent, crew, task, before_kickoff, after_kickoff-# Uncomment the following line to use an example of a custom tool# from {{folder_name}}.tools.custom_tool import MyCustomTool+# Uncomment the following line to use an example of a knowledge source+# from crewai.knowledge.source.text_file_knowledge_source import TextFileKnowledgeSource# Check our tools documentations for more information on how to use them# from crewai_tools import SerperDevTool@@ -57,10 +58,20 @@ class {{crew_name}}():@crewdef crew(self) -> Crew:"""Creates the {{crew_name}} crew"""+ # You can add knowledge sources here+ # knowledge_path = "user_preference.txt"+ # sources = [+ # TextFileKnowledgeSource(+ # file_path="knowledge/user_preference.txt",+ # metadata={"preference": "personal"}+ # ),+ # ]+return Crew(agents=self.agents, # Automatically created by the @agent decoratortasks=self.tasks, # Automatically created by the @task decoratorprocess=Process.sequential,verbose=True,# process=Process.hierarchical, # In case you wanna use that instead https://docs.crewai.com/how-to/Hierarchical/+ # knowledge_sources=sources, # In the case you want to add knowledge sources)diff --git a/src/crewai/cli/templates/crew/knowledge/user_preference.txt b/src/crewai/cli/templates/crew/knowledge/user_preference.txtnew file mode 100644index 000000000..dd63a17bf--- /dev/null+++ b/src/crewai/cli/templates/crew/knowledge/user_preference.txt@@ -0,0 +1,4 @@+User name is John Doe.+User is an AI Engineer.+User is interested in AI Agents.+User is based in San Francisco, California.diff --git a/src/crewai/knowledge/source/base_file_knowledge_source.py b/src/crewai/knowledge/source/base_file_knowledge_source.pyindex b6e346534..9cb9ec2a2 100644--- a/src/crewai/knowledge/source/base_file_knowledge_source.py+++ b/src/crewai/knowledge/source/base_file_knowledge_source.py@@ -1,36 +1,72 @@+from abc import ABC, abstractmethodfrom pathlib import Path-from typing import Union, List+from typing import Union, List, Dict, Anyfrom pydantic import Fieldfrom crewai.knowledge.source.base_knowledge_source import BaseKnowledgeSource-from typing import Dict, Any+from crewai.utilities.logger import Loggerfrom crewai.knowledge.storage.knowledge_storage import KnowledgeStorage+from crewai.utilities.constants import KNOWLEDGE_DIRECTORY-class BaseFileKnowledgeSource(BaseKnowledgeSource):+class BaseFileKnowledgeSource(BaseKnowledgeSource, ABC):"""Base class for knowledge sources that load content from files."""- file_path: Union[Path, List[Path]] = Field(...)+ _logger: Logger = Logger(verbose=True)+ file_path: Union[Path, List[Path], str, List[str]] = Field(+ ..., description="The path to the file"+ )content: Dict[Path, str] = Field(init=False, default_factory=dict)storage: KnowledgeStorage = Field(default_factory=KnowledgeStorage)+ safe_file_paths: List[Path] = Field(default_factory=list)def model_post_init(self, _):"""Post-initialization method to load content."""+ self.safe_file_paths = self._process_file_paths()+ self.validate_paths()self.content = self.load_content()+ @abstractmethoddef load_content(self) -> Dict[Path, str]:- """Load and preprocess file content. Should be overridden by subclasses."""- paths = [self.file_path] if isinstance(self.file_path, Path) else self.file_path+ """Load and preprocess file content. Should be overridden by subclasses. Assume that the file path is relative to the project root in the knowledge directory."""+ pass- for path in paths:+ def validate_paths(self):+ """Validate the paths."""+ for path in self.safe_file_paths:if not path.exists():+ self._logger.log(+ "error",+ f"File not found: {path}. Try adding sources to the knowledge directory. If its inside the knowledge directory, use the relative path.",+ color="red",+ )raise FileNotFoundError(f"File not found: {path}")if not path.is_file():- raise ValueError(f"Path is not a file: {path}")- return {}+ self._logger.log(+ "error",+ f"Path is not a file: {path}",+ color="red",+ )def save_documents(self, metadata: Dict[str, Any]):"""Save the documents to the storage."""chunk_metadatas = [metadata.copy() for _ in self.chunks]self.storage.save(self.chunks, chunk_metadatas)++ def convert_to_path(self, path: Union[Path, str]) -> Path:+ """Convert a path to a Path object."""+ return Path(KNOWLEDGE_DIRECTORY + "/" + path) if isinstance(path, str) else path++ def _process_file_paths(self) -> List[Path]:+ """Convert file_path to a list of Path objects."""+ paths = (+ [self.file_path]+ if isinstance(self.file_path, (str, Path))+ else self.file_path+ )++ if not isinstance(paths, list):+ raise ValueError("file_path must be a Path, str, or a list of these types")++ return [self.convert_to_path(path) for path in paths]diff --git a/src/crewai/knowledge/source/csv_knowledge_source.py b/src/crewai/knowledge/source/csv_knowledge_source.pyindex 0946104a4..117cbe32f 100644--- a/src/crewai/knowledge/source/csv_knowledge_source.py+++ b/src/crewai/knowledge/source/csv_knowledge_source.py@@ -10,19 +10,15 @@ class CSVKnowledgeSource(BaseFileKnowledgeSource):def load_content(self) -> Dict[Path, str]:"""Load and preprocess CSV file content."""- super().load_content() # Validate the file path-- file_path = (- self.file_path[0] if isinstance(self.file_path, list) else self.file_path- )- file_path = Path(file_path) if isinstance(file_path, str) else file_path-- with open(file_path, "r", encoding="utf-8") as csvfile:- reader = csv.reader(csvfile)- content = ""- for row in reader:- content += " ".join(row) + "\n"- return {file_path: content}+ content_dict = {}+ for file_path in self.safe_file_paths:+ with open(file_path, "r", encoding="utf-8") as csvfile:+ reader = csv.reader(csvfile)+ content = ""+ for row in reader:+ content += " ".join(row) + "\n"+ content_dict[file_path] = content+ return content_dictdef add(self) -> None:"""diff --git a/src/crewai/knowledge/source/excel_knowledge_source.py b/src/crewai/knowledge/source/excel_knowledge_source.pyindex 3b5c71514..5484e59d5 100644--- a/src/crewai/knowledge/source/excel_knowledge_source.py+++ b/src/crewai/knowledge/source/excel_knowledge_source.py@@ -8,17 +8,15 @@ class ExcelKnowledgeSource(BaseFileKnowledgeSource):def load_content(self) -> Dict[Path, str]:"""Load and preprocess Excel file content."""- super().load_content() # Validate the file pathpd = self._import_dependencies()- if isinstance(self.file_path, list):- file_path = self.file_path[0]- else:- file_path = self.file_path-- df = pd.read_excel(file_path)- content = df.to_csv(index=False)- return {file_path: content}+ content_dict = {}+ for file_path in self.safe_file_paths:+ file_path = self.convert_to_path(file_path)+ df = pd.read_excel(file_path)+ content = df.to_csv(index=False)+ content_dict[file_path] = content+ return content_dictdef _import_dependencies(self):"""Dynamically import dependencies."""diff --git a/src/crewai/knowledge/source/json_knowledge_source.py b/src/crewai/knowledge/source/json_knowledge_source.pyindex 490423a00..ed6eec86d 100644--- a/src/crewai/knowledge/source/json_knowledge_source.py+++ b/src/crewai/knowledge/source/json_knowledge_source.py@@ -10,11 +10,9 @@ class JSONKnowledgeSource(BaseFileKnowledgeSource):def load_content(self) -> Dict[Path, str]:"""Load and preprocess JSON file content."""- super().load_content() # Validate the file path- paths = [self.file_path] if isinstance(self.file_path, Path) else self.file_path-content: Dict[Path, str] = {}- for path in paths:+ for path in self.safe_file_paths:+ path = self.convert_to_path(path)with open(path, "r", encoding="utf-8") as json_file:data = json.load(json_file)content[path] = self._json_to_text(data)diff --git a/src/crewai/knowledge/source/pdf_knowledge_source.py b/src/crewai/knowledge/source/pdf_knowledge_source.pyindex 623ba30a2..e6733d78b 100644--- a/src/crewai/knowledge/source/pdf_knowledge_source.py+++ b/src/crewai/knowledge/source/pdf_knowledge_source.py@@ -9,14 +9,13 @@ class PDFKnowledgeSource(BaseFileKnowledgeSource):def load_content(self) -> Dict[Path, str]:"""Load and preprocess PDF file content."""- super().load_content() # Validate the file pathspdfplumber = self._import_pdfplumber()- paths = [self.file_path] if isinstance(self.file_path, Path) else self.file_pathcontent = {}- for path in paths:+ for path in self.safe_file_paths:text = ""+ path = self.convert_to_path(path)with pdfplumber.open(path) as pdf:for page in pdf.pages:page_text = page.extract_text()diff --git a/src/crewai/knowledge/source/text_file_knowledge_source.py b/src/crewai/knowledge/source/text_file_knowledge_source.pyindex 640db4ef9..1ceb02843 100644--- a/src/crewai/knowledge/source/text_file_knowledge_source.py+++ b/src/crewai/knowledge/source/text_file_knowledge_source.py@@ -9,12 +9,11 @@ class TextFileKnowledgeSource(BaseFileKnowledgeSource):def load_content(self) -> Dict[Path, str]:"""Load and preprocess text file content."""- super().load_content()- paths = [self.file_path] if isinstance(self.file_path, Path) else self.file_pathcontent = {}- for path in paths:- with path.open("r", encoding="utf-8") as f:- content[path] = f.read() # type: ignore+ for path in self.safe_file_paths:+ path = self.convert_to_path(path)+ with open(path, "r", encoding="utf-8") as f:+ content[path] = f.read()return contentdef add(self) -> None:diff --git a/src/crewai/utilities/constants.py b/src/crewai/utilities/constants.pyindex 59f789913..97fadda48 100644--- a/src/crewai/utilities/constants.py+++ b/src/crewai/utilities/constants.py@@ -1,3 +1,4 @@TRAINING_DATA_FILE = "training_data.pkl"TRAINED_AGENTS_DATA_FILE = "trained_agents_data.pkl"DEFAULT_SCORE_THRESHOLD = 0.35+KNOWLEDGE_DIRECTORY = "knowledge"