Skip to content

Building AI agents, atomically

Add more examples

KennyVaneetvelde4bb8d6031446
diff --git a/examples/dynamic_multichoice_answer_agent.py b/examples/dynamic_multichoice_answer_agent.py
new file mode 100644
index 0000000..8dcff75
--- /dev/null
+++ b/examples/dynamic_multichoice_answer_agent.py
@@ -0,0 +1,161 @@
+# File: multiple_choice_agent.py
+
+import instructor
+import openai
+from pydantic import BaseModel, Field, create_model
+from typing import List, Dict, Any, Literal, Union
+from atomic_agents.agents.base_agent import BaseIOSchema, BaseAgent, BaseAgentConfig
+from atomic_agents.lib.components.system_prompt_generator import SystemPromptGenerator
+
+
+class Question(BaseModel):
+ question: str = Field(..., description="The multiple-choice question")
+ options: List[str] = Field(..., min_items=2, description="List of possible answer options")
+
+
+class MultipleChoiceAgentInputSchema(BaseIOSchema):
+ """This schema defines the input schema for the MultipleChoiceAgent."""
+
+ questions: List[Question] = Field(..., min_items=1, description="List of multiple-choice questions")
+ context: str = Field(..., description="Additional context or information to help answer the questions")
+
+
+def create_dynamic_output_schema(question: Question):
+ field_name = "answer"
+ options_literal = Literal[tuple(question.options)]
+ fields = {
+ # "step_by_step_reasoning": (List[str], Field(..., description="Step-by-step reasoning for the answer")),
+ field_name: (options_literal, Field(..., description=f"Answer for question: {question.question}")),
+ }
+ return create_model(f"DynamicMultipleChoiceOutput_{id(question)}", **fields)
+
+
+def run_multiple_choice_agent(questions: List[Question], context: str):
+ results = {}
+
+ for i, question in enumerate(questions):
+ # Create a dynamic output schema for each question
+ dynamic_output_schema = create_dynamic_output_schema(question)
+
+ # Update the agent's output schema with the dynamic schema for this question
+ multiple_choice_agent.output_schema = dynamic_output_schema
+
+ # Create a single-question input for the agent
+ single_question_input = MultipleChoiceAgentInputSchema(questions=[question], context=context)
+
+ # Run the agent for this question
+ result = multiple_choice_agent.run(single_question_input)
+
+ # Store the result
+ results[f"question_{i+1}"] = {"answer": result.answer, "reasoning": "result.step_by_step_reasoning"}
+
+ return results
+
+
+# Create the multiple choice agent
+multiple_choice_agent = BaseAgent(
+ BaseAgentConfig(
+ client=instructor.from_openai(openai.OpenAI()),
+ model="gpt-4o-mini", # Updated to a more recent model
+ system_prompt_generator=SystemPromptGenerator(
+ background=[
+ "You are an expert AI multiple-choice question answering system.",
+ "Your task is to provide accurate answers to multiple-choice questions based on the given context and options.",
+ ],
+ steps=[
+ "Analyze each question and its options carefully.",
+ "Consider the provided context to inform your decision.",
+ "Develop a step-by-step reasoning process for your answer.",
+ "Select the most appropriate answer for each question based on the context and options.",
+ "Provide clear and concise answers along with your reasoning.",
+ ],
+ output_instructions=[
+ "For each question, select the best answer from the provided options.",
+ "Provide a list of step-by-step reasoning points that led to your answer.",
+ "Ensure your answers and reasoning are directly relevant to the questions and context.",
+ "Format your output as requested, with both the selected answer and the step-by-step reasoning.",
+ ],
+ ),
+ input_schema=MultipleChoiceAgentInputSchema,
+ output_schema=None, # We will set this dynamically
+ )
+)
+
+
+# Example usage
+if __name__ == "__main__":
+ questions = [
+ Question(question="What is the capital of Egypt?", options=["London", "Berlin", "Paris", "Cairo", "Madrid"]),
+ Question(question="Which planet is known as the Red Planet?", options=["Venus", "Mars", "Jupiter", "Saturn"]),
+ Question(
+ question="Who wrote 'To Kill a Mockingbird'?",
+ options=["Ernest Hemingway", "Harper Lee", "F. Scott Fitzgerald", "John Steinbeck"],
+ ),
+ Question(question="Is the Earth flat?", options=["Yes", "No"]),
+ Question(
+ question="What is the largest mammal on Earth?",
+ options=["African Elephant", "Blue Whale", "Giraffe", "Hippopotamus"],
+ ),
+ Question(
+ question="Who was the first person to step on the Moon?",
+ options=["Yuri Gagarin", "Neil Armstrong", "Buzz Aldrin", "John Glenn"],
+ ),
+ # New complex questions
+ Question(
+ question="What is the capital of Burkina Faso?",
+ options=["Ouagadougou", "Bamako", "Niamey", "Accra", "Lomé"],
+ ),
+ Question(
+ question="Is JavaScript a compiled or interpreted language?",
+ options=["Compiled", "Interpreted", "Both", "Neither"],
+ ),
+ Question(
+ question="Who painted 'The Persistence of Memory'?",
+ options=["Pablo Picasso", "Salvador Dalí", "Vincent van Gogh", "Claude Monet"],
+ ),
+ Question(
+ question="What is the primary function of mitochondria in a cell?",
+ options=["Energy production", "Protein synthesis", "Cell division", "Waste removal"],
+ ),
+ Question(question="Does the moon have its own light source?", options=["Yes", "No"]),
+ Question(
+ question="Which of these is not a programming paradigm?",
+ options=["Object-Oriented", "Functional", "Imperative", "Alphabetical"],
+ ),
+ Question(
+ question="What is the capital of Canada?",
+ options=["Toronto", "Vancouver", "Montreal", "Ottawa", "Quebec City"],
+ ),
+ Question(
+ question="Who wrote '1984'?",
+ options=["George Orwell", "Aldous Huxley", "Ray Bradbury", "Philip K. Dick"],
+ ),
+ Question(
+ question="What is the largest planet in our solar system?",
+ options=["Earth", "Mars", "Jupiter", "Saturn"],
+ ),
+ Question(
+ question="Which of these is not a type of cloud?",
+ options=["Cumulus", "Stratus", "Nimbus", "Nebulus"],
+ ),
+ ]
+ context = """
+ Egypt's capital is Cairo. Mars is often called the Red Planet due to its reddish appearance.
+ Harper Lee wrote the classic novel 'To Kill a Mockingbird'. The Earth is not flat; it is an oblate spheroid.
+ The Blue Whale is the largest animal known to have ever existed. Neil Armstrong was the first person to walk on the Moon in 1969.
+ Ouagadougou is the capital of Burkina Faso. JavaScript is primarily an interpreted language, although modern engines use just-in-time compilation.
+ Salvador Dalí painted 'The Persistence of Memory'. Mitochondria are often referred to as the powerhouses of the cell due to their role in energy production.
+ The moon does not have its own light source; it reflects sunlight. Ottawa is the capital city of Canada.
+ George Orwell wrote the dystopian novel '1984'. Jupiter is the largest planet in our solar system.
+ """
+
+ answers = run_multiple_choice_agent(questions, context)
+ for i, (question_num, result) in enumerate(answers.items()):
+ print(f"\n{question_num}:")
+ print(f"\n{question_num}:")
+ print(f"Question: {questions[i].question}")
+ print(f"Options: {questions[i].options}")
+ print(f"Answer: {result['answer']}")
+ # print("Reasoning:")
+ # for step in result["reasoning"]:
+ # print(f"- {step}")
diff --git a/examples/wiki_creator_agent/agents/article_writer_agent.py b/examples/wiki_creator_agent/agents/article_writer_agent.py
new file mode 100644
index 0000000..cf64463
--- /dev/null
+++ b/examples/wiki_creator_agent/agents/article_writer_agent.py
@@ -0,0 +1,155 @@
+# File: article_writer_agent.py
+
+import instructor
+import openai
+from pydantic import BaseModel, Field
+from typing import List
+from atomic_agents.agents.base_agent import BaseAgent, BaseAgentConfig, BaseIOSchema
+from atomic_agents.lib.components.system_prompt_generator import SystemPromptGenerator
+from examples.wiki_creator_agent.agents.outline_agent import OutlineSection, Question
+
+
+class QuestionAnswer(BaseModel):
+ """Represents a question and its corresponding answer."""
+
+ question: str
+ answer: str
+
+
+class ArticleWriterAgentInputSchema(BaseIOSchema):
+ """
+ Input schema for the ArticleWriterAgent.
+
+ This schema defines the necessary input data for generating a comprehensive,
+ well-written Medium article of approximately 3000 words, including the main topic,
+ detailed outline, and extensive question-answer pairs.
+ """
+
+ topic: str = Field(..., description="The main topic of the article")
+ outline: List[OutlineSection] = Field(
+ ..., description="The detailed outline of the article, including main sections and subsections"
+ )
+ qa_pairs: List[QuestionAnswer] = Field(
+ ...,
+ description="Extensive list of question-answer pairs for each section and subsection, providing in-depth information",
+ )
+
+
+class ArticleWriterAgentOutputSchema(BaseIOSchema):
+ """
+ Output schema for the ArticleWriterAgent.
+
+ This schema defines the structure of the agent's output, which is a
+ comprehensive, well-written markdown-formatted Medium article of
+ approximately 3000 words.
+ """
+
+ article: str = Field(
+ ..., description="The comprehensive, well-written markdown-formatted Medium article of approximately 3000 words"
+ )
+
+
+article_writer_agent = BaseAgent(
+ BaseAgentConfig(
+ client=instructor.from_openai(openai.OpenAI()),
+ model="gpt-4o-mini",
+ system_prompt_generator=SystemPromptGenerator(
+ background=[
+ "You are an expert AI article writer for Medium, specializing in long-form content.",
+ "Your task is to create a comprehensive, well-written, engaging, and informative article based on the given outline and question-answer pairs.",
+ "The article MUST be approximately 3000 words long. This is a strict requirement.",
+ "Adhere to the following quantitative metrics:",
+ "- Average words per sentence: 20-25",
+ "- Syntactic complexity: Moderate to high",
+ "- Simple to complex sentence ratio: 1:2",
+ "- Type-token ratio: 0.50",
+ "- Unique words percentage: 50-55%",
+ "- Rare/complex words usage: Moderate",
+ "- Parts of Speech: Nouns 25%, Verbs 15%, Adjectives 7%, Adverbs 6%",
+ "- Punctuation: 1.4 commas per sentence, 0.04 colons per sentence, 0.05 em dashes per sentence",
+ "Qualitative measures:",
+ "- Overall tone: Informative and conversational",
+ "- Formality level: Semi-formal with frequent conversational elements",
+ "- Objectivity vs. subjectivity: Mostly objective with occasional personal insights",
+ "- Authorial presence: Slightly more present",
+ "- Narrative perspective: Primarily third-person with occasional first-person insights",
+ "- Use of rhetorical devices: Moderate, using examples, analogies, and occasional humor",
+ "- Direct address to reader: Moderate",
+ "- Use of questions: Moderate (in the title and occasionally in the text)",
+ "- Anticipation of audience needs: High, through relatable examples and explanations",
+ "Specific aspects:",
+ "- Enthusiasm: Moderate use of intensifiers and positive language",
+ "- Professionalism: Balanced use of industry-specific terms and everyday language",
+ "- Sentence structures: High variety in openings, mix of lengths, moderate use of parallel structures",
+ "- Vocabulary level: Academic/specialized vocabulary ratio of 0.30, high consistency, appropriate use of technical terms with explanations",
+ "Key characteristics:",
+ "1. Demonstrate deep understanding of the topic",
+ "2. Present information in a clear, logical manner",
+ "3. Balance technical details with accessible explanations",
+ "4. Show awareness of both theoretical and practical aspects",
+ "5. Maintain a professional and educational tone throughout",
+ "6. Provide concrete examples to illustrate complex concepts",
+ "7. Emphasize practical implications of technical decisions",
+ "8. Show understanding of challenges in modern software development",
+ "9. Tailor content for developers with varying levels of experience",
+ "10. Explain complex issues in a structured and comprehensible way",
+ "11. Showcase understanding of software architecture and its impact",
+ "12. Maintain objectivity while presenting potentially controversial issues",
+ "13. Anticipate potential questions or areas of confusion",
+ "14. Emphasize the evolving nature of development tools and practices",
+ "15. Provide critical analysis of common development practices and their limitations",
+ ],
+ steps=[
+ "Analyze the provided outline and question-answer pairs thoroughly.",
+ "Plan a detailed structure to reach the 3000-word target while maintaining coherence and depth.",
+ "Write a comprehensive introduction (about 300 words) that hooks the reader and outlines key points.",
+ "Develop each main section into 500-700 words, ensuring thorough coverage of all subsections.",
+ "For each section and subsection:",
+ " - Provide in-depth explanations and analysis",
+ " - Include relevant examples, case studies, or hypothetical scenarios",
+ " - Incorporate statistics, expert opinions, or research findings",
+ " - Address potential counterarguments or alternative viewpoints",
+ "Use paragraphs of 4-6 sentences, allowing for detailed exploration of ideas.",
+ "Create smooth transitions between sections for a cohesive narrative.",
+ "Conclude with a comprehensive summary (about 300 words) that recaps main points and offers insights.",
+ "Review the article to ensure it meets the 3000-word requirement and adheres to all specified metrics and characteristics.",
+ "Refine the content to balance accessibility for beginners with depth for advanced readers.",
+ "Double-check adherence to quantitative metrics, qualitative measures, and key characteristics.",
+ ],
+ output_instructions=[
+ "Produce a well-structured, engaging Medium article in markdown format, with a STRICT target length of 3000 words (±100 words).",
+ "Use a clear hierarchy of headers: # for main sections, ## for subsections, ### for sub-subsections.",
+ "Seamlessly incorporate information from question-answer pairs, expanding with context and insights.",
+ "Include a compelling title and subtitle reflecting the comprehensive nature of the article.",
+ "Aim for 3-5 sentences per paragraph, allowing for a mix of detailed explanations and punchier points.",
+ "Use bold for key concepts and italic for emphasis or to add a conversational tone.",
+ "Ensure natural flow between sections, maintaining a coherent narrative.",
+ "Include relevant code snippets using proper markdown syntax, followed by clear, relatable explanations.",
+ "Use lists when appropriate, balancing them with explanatory paragraphs.",
+ "Incorporate relevant subheadings within main sections to improve organization and scannability.",
+ "Use block quotes for important statements or expert opinions, providing context and analysis with a personal touch.",
+ "Conclude with a strong summary recapping main points and offering compelling insights, possibly with a call to action or thought-provoking question.",
+ "Adhere to the specified quantitative metrics for sentence structure, vocabulary, and punctuation, but allow for some flexibility to maintain a natural flow.",
+ "Maintain the required tone and style as outlined in the writer profile, leaning towards a more conversational approach.",
+ "Demonstrate deep understanding while balancing technical details with accessible explanations.",
+ "Use industry-specific jargon and technical terms appropriately, but always provide clear explanations or analogies for complex concepts.",
+ "Provide critical analysis and emphasize practical implications of technical decisions, relating them to real-world scenarios when possible.",
+ "Anticipate and address potential questions or areas of confusion proactively, using a friendly, guiding tone.",
+ "Maintain high objectivity with analytical insights throughout the article.",
+ "Maintain objectivity with analytical insights throughout the article, but don't shy away from adding occasional personal perspectives or experiences.",
+ "Use rhetorical devices moderately, including examples, analogies, and occasional humor to keep the reader engaged.",
+ "Incorporate direct address to the reader and use questions occasionally to create a more interactive feel.",
+ "Ensure high anticipation of audience needs through relatable examples and clear explanations.",
+ "Allow for moderate enthusiasm by using some intensifiers and positive language to convey excitement about the topic.",
+ "Achieve the specified academic/specialized vocabulary ratio, but prioritize clarity and accessibility.",
+ "Vary sentence openings and lengths to create a natural, flowing rhythm.",
+ "IMPORTANT: The final article MUST be approximately 3000 words. If significantly shorter, expand key points or add more examples and analysis to reach the target length.",
+ "IMPORTANT: The final article MUST flow nicely from section to section and have a single overarching narrative, with a conversational thread tying it all together.",
+ "IMPORTANT: The final article MUST have a good balance between prose and lists, with a minimum of 80% being prose, and a maximum of 20% being lists",
+ "IMPORTANT: If the article is about code, include plenty of didactic code snippets with clear, relatable explanations",
+ ],
+ ),
+ input_schema=ArticleWriterAgentInputSchema,
+ output_schema=ArticleWriterAgentOutputSchema,
+ )
+)
diff --git a/examples/wiki_creator_agent/agents/outline_agent.py b/examples/wiki_creator_agent/agents/outline_agent.py
new file mode 100644
index 0000000..5d25a1d
--- /dev/null
+++ b/examples/wiki_creator_agent/agents/outline_agent.py
@@ -0,0 +1,86 @@
+# File: outline_agent.py
+
+import instructor
+import openai
+from pydantic import BaseModel, Field
+from typing import List
+from atomic_agents.agents.base_agent import BaseIOSchema, BaseAgent, BaseAgentConfig
+from atomic_agents.lib.components.system_prompt_generator import SystemPromptGenerator
+
+
+class Question(BaseModel):
+ question: str = Field(..., description="The question to answer in this section or subsection")
+ potential_sources: List[str] = Field(
+ default_factory=list, description="URLs of potential sources or references for this question. Required. Min. 1."
+ )
+
+
+class OutlineSubSection(BaseModel):
+ title: str = Field(..., description="The title of the outline subsection")
+ questions_to_answer: List[Question] = Field(
+ default_factory=list,
+ description="A list of 2-4 hypothetical questions that the contents of this subsection should answer.",
+ )
+
+
+class OutlineSection(BaseModel):
+ title: str = Field(..., description="The title of the outline section")
+ questions_to_answer: List[Question] = Field(
+ default_factory=list, description="2-3 overarching questions to answer in this section"
+ )
+ subsections: List[OutlineSubSection] = Field(
+ default_factory=list, description="Subsections of this outline section. Each section must have 3-5 subsections."
+ )
+
+
+class OutlineAgentInputSchema(BaseIOSchema):
+ """This schema defines the input schema for the OutlineAgent."""
+
+ topic: str = Field(..., description="The main topic of the article")
+
+
+class OutlineAgentOutputSchema(BaseIOSchema):
+ """This schema defines the output schema for the OutlineAgent."""
+
+ internal_reasoning: List[str] = Field(
+ ...,
+ description="Internal reasoning about the context and what information it contains that could be useful in generating or modifying an outline.",
+ )
+ title: str = Field(..., description="The title of the article")
+ outline: List[OutlineSection] = Field(
+ ..., description="The generated or modified outline. The outline must have 5-7 main sections."
+ )
+
+
+# Create the outline agent
+outline_agent = BaseAgent(
+ BaseAgentConfig(
+ client=instructor.from_openai(openai.OpenAI()),
+ model="gpt-4o-mini",
+ system_prompt_generator=SystemPromptGenerator(
+ background=[
+ "You are an expert AI outline creator and editor, specializing in creating comprehensive and well-structured outlines.",
+ "Your task is to generate or modify article outlines based on the given topic and context, ensuring a logical flow and thorough coverage of the subject.",
+ ],
+ steps=[
+ "Analyze the given topic and any additional context provided in depth.",
+ "Create a comprehensive structure for the article with 5-7 main sections.",
+ "For each main section, develop 3-5 subsections that explore different aspects of the main section's topic.",
+ "Ensure the outline is well-organized, logical, and covers all important aspects of the topic in a balanced manner.",
+ "For each section, provide 2-3 overarching questions to answer.",
+ "For each subsection, provide 2-4 specific questions to answer, including potential sources for each question.",
+ ],
+ output_instructions=[
+ "Create a clear, concise, and comprehensive outline with 5-7 main sections.",
+ "Ensure each main section has 3-5 well-defined subsections.",
+ "Provide 1-2 overarching questions for each main section.",
+ "Include 2-4 specific questions to answer for each subsection, along with potential sources.",
+ "Use descriptive and informative titles for each section and subsection.",
+ "Ensure each section and subsection is relevant to the main topic and contributes to a cohesive narrative.",
+ "Aim for a balanced structure that covers the topic comprehensively, with appropriate depth in each section.",
+ ],
+ ),
+ input_schema=OutlineAgentInputSchema,
+ output_schema=OutlineAgentOutputSchema,
+ )
+)
diff --git a/examples/wiki_creator_agent/agents/query_agent.py b/examples/wiki_creator_agent/agents/query_agent.py
new file mode 100644
index 0000000..ef5d3f4
--- /dev/null
+++ b/examples/wiki_creator_agent/agents/query_agent.py
@@ -0,0 +1,48 @@
+# File: query_agent.py
+
+import instructor
+import openai
+from pydantic import BaseModel, Field
+from typing import List
+from atomic_agents.agents.base_agent import BaseIOSchema, BaseAgent, BaseAgentConfig
+from atomic_agents.lib.components.system_prompt_generator import SystemPromptGenerator
+
+
+class QueryAgentInputSchema(BaseIOSchema):
+ """This schema defines the input schema for the QueryAgent."""
+
+ instruction: str = Field(..., description="The user's input or question for which queries need to be generated.")
+ num_queries: int = Field(default=5, description="The number of queries to generate.")
+
+
+class QueryAgentOutputSchema(BaseIOSchema):
+ """This schema defines the output schema for the QueryAgent."""
+
+ queries: List[str] = Field(..., description="A list of generated queries based on the user's input.")
+
+
+# Create the query agent
+query_agent = BaseAgent(
+ BaseAgentConfig(
+ client=instructor.from_openai(openai.OpenAI()),
+ model="gpt-4o-mini",
+ system_prompt_generator=SystemPromptGenerator(
+ background=[
+ "You are an expert google query generator.",
+ "Your task is to generate relevant and diverse google search queries based on the user's input or question.",
+ ],
+ steps=[
+ "Analyze the user's input or question.",
+ "Generate a specified number of relevant google search queries.",
+ "Ensure the queries cover different aspects of the user's input.",
+ ],
+ output_instructions=[
+ "Generate clear and concise google queries.",
+ "Ensure each google query is directly relevant to the user's input.",
+ "Avoid repetitive queries and aim for diversity in the generated set.",
+ ],
+ ),
+ input_schema=QueryAgentInputSchema,
+ output_schema=QueryAgentOutputSchema,
+ )
+)
diff --git a/examples/wiki_creator_agent/agents/question_answering_agent.py b/examples/wiki_creator_agent/agents/question_answering_agent.py
new file mode 100644
index 0000000..1c9a485
--- /dev/null
+++ b/examples/wiki_creator_agent/agents/question_answering_agent.py
@@ -0,0 +1,42 @@
+import instructor
+import openai
+from pydantic import BaseModel, Field
+from atomic_agents.agents.base_agent import BaseAgent, BaseAgentConfig
+from atomic_agents.lib.components.system_prompt_generator import SystemPromptGenerator
+
+
+class QuestionAnsweringAgentInputSchema(BaseModel):
+ question: str = Field(..., description="The question to be answered")
+
+
+class QuestionAnsweringAgentOutputSchema(BaseModel):
+ answer: str = Field(..., description="The answer to the provided question")
+
+
+# Create the question answering agent
+question_answering_agent = BaseAgent(
+ BaseAgentConfig(
+ client=instructor.from_openai(openai.OpenAI()),
+ model="gpt-4o-mini",
+ system_prompt_generator=SystemPromptGenerator(
+ background=[
+ "You are an expert AI question answering system.",
+ "Your task is to provide accurate and concise answers to questions based on the given context.",
+ ],
+ steps=[
+ "Analyze the provided question and context.",
+ "Search the context for relevant information.",
+ "Generate a clear and concise answer based on the context.",
+ "Identify the sources used for the answer.",
+ ],
+ output_instructions=[
+ "Provide an accurate answer to the question.",
+ "Ensure the answer is directly relevant to the question.",
+ "Include sources for the answer when available.",
+ "Keep the answer concise but informative.",
+ ],
+ ),
+ input_schema=QuestionAnsweringAgentInputSchema,
+ output_schema=QuestionAnsweringAgentOutputSchema,
+ )
+)
diff --git a/examples/wiki_creator_agent/main.py b/examples/wiki_creator_agent/main.py
new file mode 100644
index 0000000..bd0ad5f
--- /dev/null
+++ b/examples/wiki_creator_agent/main.py
@@ -0,0 +1,229 @@
+import os
+import asyncio
+from typing import List
+import rich
+from atomic_agents.lib.tools.search.searxng_tool import SearxNGTool, SearxNGToolConfig
+from atomic_agents.lib.components.system_prompt_generator import SystemPromptContextProviderBase
+from examples.wiki_creator_agent.agents.query_agent import query_agent, QueryAgentInputSchema
+from examples.wiki_creator_agent.agents.outline_agent import outline_agent, OutlineAgentInputSchema
+from examples.wiki_creator_agent.agents.question_answering_agent import (
+ question_answering_agent,
+ QuestionAnsweringAgentInputSchema,
+)
+from examples.wiki_creator_agent.utils.scraper import ContentScraper
+from examples.wiki_creator_agent.utils.vector_db import InMemFaiss
+from examples.wiki_creator_agent.agents.article_writer_agent import (
+ article_writer_agent,
+ ArticleWriterAgentInputSchema,
+ QuestionAnswer,
+)
+
+rich_console = rich.console.Console()
+
+
+###################
+# CONTEXT PROVIDERS #
+###################
+class SearchResultsProvider(SystemPromptContextProviderBase):
+ def __init__(self, title="Search Results"):
+ super().__init__(title)
+ self.search_results = None
+
+ def get_info(self) -> str:
+ if self.search_results is None:
+ return "No search results available."
+
+ results_str = "Search Results:\n"
+ for i, result in enumerate(self.search_results.results, 1):
+ results_str += f"{i}. Title: {result.title}\n URL: {result.url}\n Content: {result.content[:100]}...\n\n"
+ return results_str
+
+
+class VectorDBResultsProvider(SystemPromptContextProviderBase):
+ def __init__(self, title="Vector Database Results"):
+ super().__init__(title)
+ self.vector_db_results = None
+
+ def get_info(self) -> str:
+ if self.vector_db_results is None:
+ return "No vector database results available."
+
+ results_str = "Vector Database Results:\n\n"
+ for i, chunk in enumerate(self.vector_db_results, 1):
+ results_str += f"## RESULT {i}\n"
+ results_str += f"{chunk}\n"
+ results_str += "=" * 50 + "\n\n"
+
+ return results_str
+
+
+class OutlineProvider(SystemPromptContextProviderBase):
+ def __init__(self, title="Article Outline"):
+ super().__init__(title)
+ self.outline = None
+
+ def get_info(self) -> str:
+ if self.outline is None:
+ return "No outline available."
+
+ outline_str = "Article Outline:\n"
+ for section in self.outline:
+ outline_str += f"Section: {section.title}\n"
+ for subsection in section.subsections:
+ outline_str += f" Subsection: {subsection.title}\n"
+ return outline_str
+
+
+#############
+# PIPELINES #
+#############
+class QueryAndSearchPipeline:
+ @staticmethod
+ def generate_queries_and_search(user_input: str, num_queries: int = 3):
+ # Generate queries
+ query_instruction = f"Generate research queries for an article about {user_input}"
+ query_input = QueryAgentInputSchema(instruction=query_instruction, num_queries=num_queries)
+ query_output = query_agent.run(query_input)
+ queries = query_output.queries
+
+ # Perform search
+ search_tool_instance = SearxNGTool(config=SearxNGToolConfig(base_url=os.getenv("SEARXNG_BASE_URL"), max_results=30))
+ search_input = SearxNGTool.input_schema(queries=queries)
+ search_output = search_tool_instance.run(search_input)
+
+ return queries, search_output
+
+
+class OutlineGenerationPipeline:
+ @staticmethod
+ def generate_outline(user_input: str, search_results_provider):
+ outline_agent.register_context_provider("search_results", search_results_provider)
+ outline_input = OutlineAgentInputSchema(topic=user_input)
+ outline_output = outline_agent.run(outline_input)
+ return outline_output.outline
+
+ @staticmethod
+ def extract_urls_from_outline(outline):
+ urls = set()
+ for section in outline:
+ urls.update(url for point in section.questions_to_answer for url in point.potential_sources)
+ for subsection in section.subsections:
+ urls.update(url for point in subsection.questions_to_answer for url in point.potential_sources)
+ return sorted(list(urls))
+
+
+class ContentScrapingPipeline:
+ @staticmethod
+ async def scrape_and_ingest_urls(urls: List[str], batch_size: int = 5):
+ faiss_db = InMemFaiss(openai_api_key=os.getenv("OPENAI_API_KEY"))
+ batches = [urls[i : i + batch_size] for i in range(0, len(urls), batch_size)]
+
+ for batch_num, batch in enumerate(batches, 1):
+ rich_console.print(f"[bold magenta]Scraping and ingesting batch {batch_num} of {len(batches)}[/bold magenta]")
+ scraped_content = await ContentScraper.scrape_urls(batch)
+ await faiss_db.ingest_documents(scraped_content)
+
+ for url in scraped_content.keys():
+ rich_console.print(f"[green]Ingested content from {url}[/green]")
+
+ return faiss_db
+
+
+class QuestionAnsweringPipeline:
+ @staticmethod
+ async def answer_questions(outline, faiss_db, faiss_db_context_provider, outline_provider):
+ question_answering_agent.register_context_provider("vector_db_results", faiss_db_context_provider)
+ question_answering_agent.register_context_provider("outline", outline_provider)
+
+ qa_pairs = []
+
+ for section in outline:
+ for subsection in section.subsections:
+ questions_to_answer = "\n".join([item.question for item in subsection.questions_to_answer])
+ context = faiss_db.retrieve_chunks(questions_to_answer)
+ faiss_db_context_provider.vector_db_results = context
+
+ for question_item in subsection.questions_to_answer:
+ question_input = QuestionAnsweringAgentInputSchema(question=question_item.question)
+ answer_output = question_answering_agent.run(question_input)
+
+ rich_console.print(f"[bold cyan]Question:[/bold cyan] {question_item.question}")
+ rich_console.print(f"[bold green]Answer:[/bold green] {answer_output.answer}\n")
+
+ qa_pairs.append(QuestionAnswer(question=question_item.question, answer=answer_output.answer))
+
+ return qa_pairs
+
+
+class ArticleWritingPipeline:
+ @staticmethod
+ def write_article(topic: str, outline, qa_pairs):
+ input_data = ArticleWriterAgentInputSchema(topic=topic, outline=outline, qa_pairs=qa_pairs)
+ output = article_writer_agent.run(input_data)
+ return output.article
+
+
+###################
+# MAIN APPLICATION #
+###################
+class WikiCreatorApplication:
+ def __init__(self):
+ self.rich_console = rich.console.Console()
+ self.search_results_provider = SearchResultsProvider()
+ self.vector_db_results_provider = VectorDBResultsProvider()
+ self.outline_provider = OutlineProvider()
+ self.qa_pairs = []
+
+ async def run(self):
+ self.rich_console.print("[bold blue]Script execution started[/bold blue]")
+ user_input = input("I want to write an article about ")
+
+ queries, search_output = QueryAndSearchPipeline.generate_queries_and_search(user_input, num_queries=5)
+ self.rich_console.print("[bold yellow]Generated Queries:[/bold yellow]")
+ for i, query in enumerate(queries, 1):
+ self.rich_console.print(f"[yellow]{i}. {query}[/yellow]")
+ self.rich_console.print()
+
+ self.search_results_provider.search_results = search_output
+
+ outline = OutlineGenerationPipeline.generate_outline(user_input, self.search_results_provider)
+ self.rich_console.print("[bold green]Generated Outline:[/bold green]")
+ self.rich_console.print(outline)
+
+ urls = OutlineGenerationPipeline.extract_urls_from_outline(outline)
+ self.rich_console.print("\n[bold cyan]Extracted Potential Source URLs:[/bold cyan]")
+ for i, url in enumerate(urls, 1):
+ self.rich_console.print(f"[cyan]{i}. {url}[/cyan]")
+
+ batch_size = 10 # You can change this value or make it configurable
+ faiss_db = await ContentScrapingPipeline.scrape_and_ingest_urls(urls, batch_size)
+
+ self.outline_provider.outline = outline
+ self.qa_pairs = await QuestionAnsweringPipeline.answer_questions(
+ outline, faiss_db, self.vector_db_results_provider, self.outline_provider
+ )
+
+ # New code for article writing
+ self.rich_console.print("[bold magenta]Writing the article...[/bold magenta]")
+ article = ArticleWritingPipeline.write_article(user_input, outline, self.qa_pairs)
+
+ self.rich_console.print("[bold green]Article generated successfully![/bold green]")
+ self.rich_console.print("\n[bold cyan]Generated Article:[/bold cyan]")
+ self.rich_console.print(article)
+
+ # Optionally, save the article to a file
+ with open("generated_article.md", "w", encoding="utf-8") as f:
+ f.write(article)
+ self.rich_console.print("\n[bold green]Article saved to 'generated_article.md'[/bold green]")
+
+
+####################
+# SCRIPT EXECUTION #
+####################
+async def main():
+ app = WikiCreatorApplication()
+ await app.run()
+
+
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/examples/wiki_creator_agent/utils/scraper.py b/examples/wiki_creator_agent/utils/scraper.py
new file mode 100644
index 0000000..aa62404
--- /dev/null
+++ b/examples/wiki_creator_agent/utils/scraper.py
@@ -0,0 +1,45 @@
+import aiohttp
+import asyncio
+from bs4 import BeautifulSoup
+from readability import Document
+import html2text
+from typing import List, Dict
+import trafilatura
+
+
+class ContentScraper:
+ @staticmethod
+ async def scrape_urls(urls: List[str]) -> Dict[str, str]:
+ async def fetch_url(session: aiohttp.ClientSession, url: str) -> tuple:
+ try:
+ async with session.get(url, timeout=30) as response: # Added timeout
+ html_content = await response.text()
+
+ # Try trafilatura first
+ extracted_content = trafilatura.extract(html_content)
+
+ if not extracted_content:
+ # If trafilatura fails, fall back to readability
+ doc = Document(html_content)
+ extracted_content = doc.summary()
+
+ # Clean up the extracted content
+ soup = BeautifulSoup(extracted_content, "html.parser")
+ for script in soup(["script", "style"]):
+ script.decompose()
+ cleaned_content = soup.get_text()
+
+ # Convert to markdown
+ h = html2text.HTML2Text()
+ h.ignore_links = False
+ markdown_content = h.handle(cleaned_content)
+
+ return url, markdown_content.strip()
+ except Exception as e:
+ return url, f"Error scraping {url}: {str(e)}"
+
+ async with aiohttp.ClientSession() as session:
+ tasks = [fetch_url(session, url) for url in urls]
+ results = await asyncio.gather(*tasks)
+
+ return dict(results)
diff --git a/examples/wiki_creator_agent/utils/vector_db.py b/examples/wiki_creator_agent/utils/vector_db.py
new file mode 100644
index 0000000..3214261
--- /dev/null
+++ b/examples/wiki_creator_agent/utils/vector_db.py
@@ -0,0 +1,57 @@
+import openai
+import faiss
+import numpy as np
+from typing import List, Dict
+from tqdm import tqdm
+
+
+class InMemFaiss:
+ def __init__(self, openai_api_key):
+ openai.api_key = openai_api_key
+ self.index = faiss.IndexFlatL2(1536)
+ self.texts = []
+ self.metadata = []
+
+ def split_text(self, text, chunk_size=1750, overlap=250):
+ chunks = []
+ for i in range(0, len(text), chunk_size - overlap):
+ chunk = text[i : i + chunk_size].strip()
+ chunks.append(chunk)
+ return chunks
+
+ def generate_embeddings(self, texts):
+ embeddings = []
+ batch_size = 1000 # Adjust this based on your needs and API limits
+
+ for i in tqdm(range(0, len(texts), batch_size)):
+ batch = texts[i : i + batch_size]
+ try:
+ response = openai.embeddings.create(input=batch, model="text-embedding-3-small")
+ batch_embeddings = [item.embedding for item in response.data]
+ embeddings.extend(batch_embeddings)
+ except Exception as e:
+ print(f"Error generating embeddings for batch {i//batch_size}: {e}")
+ # Handle the error appropriately, e.g., retry or skip
+
+ return np.array(embeddings).astype("float32")
+
+ async def ingest_documents(self, documents: Dict[str, str]):
+ all_chunks = []
+ all_metadata = []
+
+ for url, content in documents.items():
+ if len(content) == 0:
+ continue
+ chunks = self.split_text(content)
+ all_chunks.extend(chunks)
+ all_metadata.extend([{"url": url}] * len(chunks))
+
+ embeddings = self.generate_embeddings(all_chunks)
+ self.index.add(embeddings)
+ self.texts.extend(all_chunks)
+ self.metadata.extend(all_metadata)
+
+ def retrieve_chunks(self, query, top_k=5):
+ query_embedding = self.generate_embeddings([query])
+ distances, indices = self.index.search(query_embedding, top_k)
+ return [self.texts[idx] for idx in indices[0]]