Building AI agents, atomically
Add more examples
KennyVaneetvelde4bb8d6031446
diff --git a/examples/dynamic_multichoice_answer_agent.py b/examples/dynamic_multichoice_answer_agent.pynew file mode 100644index 0000000..8dcff75--- /dev/null+++ b/examples/dynamic_multichoice_answer_agent.py@@ -0,0 +1,161 @@+# File: multiple_choice_agent.py++import instructor+import openai+from pydantic import BaseModel, Field, create_model+from typing import List, Dict, Any, Literal, Union+from atomic_agents.agents.base_agent import BaseIOSchema, BaseAgent, BaseAgentConfig+from atomic_agents.lib.components.system_prompt_generator import SystemPromptGenerator+++class Question(BaseModel):+ question: str = Field(..., description="The multiple-choice question")+ options: List[str] = Field(..., min_items=2, description="List of possible answer options")+++class MultipleChoiceAgentInputSchema(BaseIOSchema):+ """This schema defines the input schema for the MultipleChoiceAgent."""++ questions: List[Question] = Field(..., min_items=1, description="List of multiple-choice questions")+ context: str = Field(..., description="Additional context or information to help answer the questions")+++def create_dynamic_output_schema(question: Question):+ field_name = "answer"+ options_literal = Literal[tuple(question.options)]+ fields = {+ # "step_by_step_reasoning": (List[str], Field(..., description="Step-by-step reasoning for the answer")),+ field_name: (options_literal, Field(..., description=f"Answer for question: {question.question}")),+ }+ return create_model(f"DynamicMultipleChoiceOutput_{id(question)}", **fields)+++def run_multiple_choice_agent(questions: List[Question], context: str):+ results = {}++ for i, question in enumerate(questions):+ # Create a dynamic output schema for each question+ dynamic_output_schema = create_dynamic_output_schema(question)++ # Update the agent's output schema with the dynamic schema for this question+ multiple_choice_agent.output_schema = dynamic_output_schema++ # Create a single-question input for the agent+ single_question_input = MultipleChoiceAgentInputSchema(questions=[question], context=context)++ # Run the agent for this question+ result = multiple_choice_agent.run(single_question_input)++ # Store the result+ results[f"question_{i+1}"] = {"answer": result.answer, "reasoning": "result.step_by_step_reasoning"}++ return results+++# Create the multiple choice agent+multiple_choice_agent = BaseAgent(+ BaseAgentConfig(+ client=instructor.from_openai(openai.OpenAI()),+ model="gpt-4o-mini", # Updated to a more recent model+ system_prompt_generator=SystemPromptGenerator(+ background=[+ "You are an expert AI multiple-choice question answering system.",+ "Your task is to provide accurate answers to multiple-choice questions based on the given context and options.",+ ],+ steps=[+ "Analyze each question and its options carefully.",+ "Consider the provided context to inform your decision.",+ "Develop a step-by-step reasoning process for your answer.",+ "Select the most appropriate answer for each question based on the context and options.",+ "Provide clear and concise answers along with your reasoning.",+ ],+ output_instructions=[+ "For each question, select the best answer from the provided options.",+ "Provide a list of step-by-step reasoning points that led to your answer.",+ "Ensure your answers and reasoning are directly relevant to the questions and context.",+ "Format your output as requested, with both the selected answer and the step-by-step reasoning.",+ ],+ ),+ input_schema=MultipleChoiceAgentInputSchema,+ output_schema=None, # We will set this dynamically+ )+)+++# Example usage+if __name__ == "__main__":+ questions = [+ Question(question="What is the capital of Egypt?", options=["London", "Berlin", "Paris", "Cairo", "Madrid"]),+ Question(question="Which planet is known as the Red Planet?", options=["Venus", "Mars", "Jupiter", "Saturn"]),+ Question(+ question="Who wrote 'To Kill a Mockingbird'?",+ options=["Ernest Hemingway", "Harper Lee", "F. Scott Fitzgerald", "John Steinbeck"],+ ),+ Question(question="Is the Earth flat?", options=["Yes", "No"]),+ Question(+ question="What is the largest mammal on Earth?",+ options=["African Elephant", "Blue Whale", "Giraffe", "Hippopotamus"],+ ),+ Question(+ question="Who was the first person to step on the Moon?",+ options=["Yuri Gagarin", "Neil Armstrong", "Buzz Aldrin", "John Glenn"],+ ),+ # New complex questions+ Question(+ question="What is the capital of Burkina Faso?",+ options=["Ouagadougou", "Bamako", "Niamey", "Accra", "Lomé"],+ ),+ Question(+ question="Is JavaScript a compiled or interpreted language?",+ options=["Compiled", "Interpreted", "Both", "Neither"],+ ),+ Question(+ question="Who painted 'The Persistence of Memory'?",+ options=["Pablo Picasso", "Salvador Dalí", "Vincent van Gogh", "Claude Monet"],+ ),+ Question(+ question="What is the primary function of mitochondria in a cell?",+ options=["Energy production", "Protein synthesis", "Cell division", "Waste removal"],+ ),+ Question(question="Does the moon have its own light source?", options=["Yes", "No"]),+ Question(+ question="Which of these is not a programming paradigm?",+ options=["Object-Oriented", "Functional", "Imperative", "Alphabetical"],+ ),+ Question(+ question="What is the capital of Canada?",+ options=["Toronto", "Vancouver", "Montreal", "Ottawa", "Quebec City"],+ ),+ Question(+ question="Who wrote '1984'?",+ options=["George Orwell", "Aldous Huxley", "Ray Bradbury", "Philip K. Dick"],+ ),+ Question(+ question="What is the largest planet in our solar system?",+ options=["Earth", "Mars", "Jupiter", "Saturn"],+ ),+ Question(+ question="Which of these is not a type of cloud?",+ options=["Cumulus", "Stratus", "Nimbus", "Nebulus"],+ ),+ ]+ context = """+ Egypt's capital is Cairo. Mars is often called the Red Planet due to its reddish appearance.+ Harper Lee wrote the classic novel 'To Kill a Mockingbird'. The Earth is not flat; it is an oblate spheroid.+ The Blue Whale is the largest animal known to have ever existed. Neil Armstrong was the first person to walk on the Moon in 1969.+ Ouagadougou is the capital of Burkina Faso. JavaScript is primarily an interpreted language, although modern engines use just-in-time compilation.+ Salvador Dalí painted 'The Persistence of Memory'. Mitochondria are often referred to as the powerhouses of the cell due to their role in energy production.+ The moon does not have its own light source; it reflects sunlight. Ottawa is the capital city of Canada.+ George Orwell wrote the dystopian novel '1984'. Jupiter is the largest planet in our solar system.+ """++ answers = run_multiple_choice_agent(questions, context)+ for i, (question_num, result) in enumerate(answers.items()):+ print(f"\n{question_num}:")+ print(f"\n{question_num}:")+ print(f"Question: {questions[i].question}")+ print(f"Options: {questions[i].options}")+ print(f"Answer: {result['answer']}")+ # print("Reasoning:")+ # for step in result["reasoning"]:+ # print(f"- {step}")diff --git a/examples/wiki_creator_agent/agents/article_writer_agent.py b/examples/wiki_creator_agent/agents/article_writer_agent.pynew file mode 100644index 0000000..cf64463--- /dev/null+++ b/examples/wiki_creator_agent/agents/article_writer_agent.py@@ -0,0 +1,155 @@+# File: article_writer_agent.py++import instructor+import openai+from pydantic import BaseModel, Field+from typing import List+from atomic_agents.agents.base_agent import BaseAgent, BaseAgentConfig, BaseIOSchema+from atomic_agents.lib.components.system_prompt_generator import SystemPromptGenerator+from examples.wiki_creator_agent.agents.outline_agent import OutlineSection, Question+++class QuestionAnswer(BaseModel):+ """Represents a question and its corresponding answer."""++ question: str+ answer: str+++class ArticleWriterAgentInputSchema(BaseIOSchema):+ """+ Input schema for the ArticleWriterAgent.++ This schema defines the necessary input data for generating a comprehensive,+ well-written Medium article of approximately 3000 words, including the main topic,+ detailed outline, and extensive question-answer pairs.+ """++ topic: str = Field(..., description="The main topic of the article")+ outline: List[OutlineSection] = Field(+ ..., description="The detailed outline of the article, including main sections and subsections"+ )+ qa_pairs: List[QuestionAnswer] = Field(+ ...,+ description="Extensive list of question-answer pairs for each section and subsection, providing in-depth information",+ )+++class ArticleWriterAgentOutputSchema(BaseIOSchema):+ """+ Output schema for the ArticleWriterAgent.++ This schema defines the structure of the agent's output, which is a+ comprehensive, well-written markdown-formatted Medium article of+ approximately 3000 words.+ """++ article: str = Field(+ ..., description="The comprehensive, well-written markdown-formatted Medium article of approximately 3000 words"+ )+++article_writer_agent = BaseAgent(+ BaseAgentConfig(+ client=instructor.from_openai(openai.OpenAI()),+ model="gpt-4o-mini",+ system_prompt_generator=SystemPromptGenerator(+ background=[+ "You are an expert AI article writer for Medium, specializing in long-form content.",+ "Your task is to create a comprehensive, well-written, engaging, and informative article based on the given outline and question-answer pairs.",+ "The article MUST be approximately 3000 words long. This is a strict requirement.",+ "Adhere to the following quantitative metrics:",+ "- Average words per sentence: 20-25",+ "- Syntactic complexity: Moderate to high",+ "- Simple to complex sentence ratio: 1:2",+ "- Type-token ratio: 0.50",+ "- Unique words percentage: 50-55%",+ "- Rare/complex words usage: Moderate",+ "- Parts of Speech: Nouns 25%, Verbs 15%, Adjectives 7%, Adverbs 6%",+ "- Punctuation: 1.4 commas per sentence, 0.04 colons per sentence, 0.05 em dashes per sentence",+ "Qualitative measures:",+ "- Overall tone: Informative and conversational",+ "- Formality level: Semi-formal with frequent conversational elements",+ "- Objectivity vs. subjectivity: Mostly objective with occasional personal insights",+ "- Authorial presence: Slightly more present",+ "- Narrative perspective: Primarily third-person with occasional first-person insights",+ "- Use of rhetorical devices: Moderate, using examples, analogies, and occasional humor",+ "- Direct address to reader: Moderate",+ "- Use of questions: Moderate (in the title and occasionally in the text)",+ "- Anticipation of audience needs: High, through relatable examples and explanations",+ "Specific aspects:",+ "- Enthusiasm: Moderate use of intensifiers and positive language",+ "- Professionalism: Balanced use of industry-specific terms and everyday language",+ "- Sentence structures: High variety in openings, mix of lengths, moderate use of parallel structures",+ "- Vocabulary level: Academic/specialized vocabulary ratio of 0.30, high consistency, appropriate use of technical terms with explanations",+ "Key characteristics:",+ "1. Demonstrate deep understanding of the topic",+ "2. Present information in a clear, logical manner",+ "3. Balance technical details with accessible explanations",+ "4. Show awareness of both theoretical and practical aspects",+ "5. Maintain a professional and educational tone throughout",+ "6. Provide concrete examples to illustrate complex concepts",+ "7. Emphasize practical implications of technical decisions",+ "8. Show understanding of challenges in modern software development",+ "9. Tailor content for developers with varying levels of experience",+ "10. Explain complex issues in a structured and comprehensible way",+ "11. Showcase understanding of software architecture and its impact",+ "12. Maintain objectivity while presenting potentially controversial issues",+ "13. Anticipate potential questions or areas of confusion",+ "14. Emphasize the evolving nature of development tools and practices",+ "15. Provide critical analysis of common development practices and their limitations",+ ],+ steps=[+ "Analyze the provided outline and question-answer pairs thoroughly.",+ "Plan a detailed structure to reach the 3000-word target while maintaining coherence and depth.",+ "Write a comprehensive introduction (about 300 words) that hooks the reader and outlines key points.",+ "Develop each main section into 500-700 words, ensuring thorough coverage of all subsections.",+ "For each section and subsection:",+ " - Provide in-depth explanations and analysis",+ " - Include relevant examples, case studies, or hypothetical scenarios",+ " - Incorporate statistics, expert opinions, or research findings",+ " - Address potential counterarguments or alternative viewpoints",+ "Use paragraphs of 4-6 sentences, allowing for detailed exploration of ideas.",+ "Create smooth transitions between sections for a cohesive narrative.",+ "Conclude with a comprehensive summary (about 300 words) that recaps main points and offers insights.",+ "Review the article to ensure it meets the 3000-word requirement and adheres to all specified metrics and characteristics.",+ "Refine the content to balance accessibility for beginners with depth for advanced readers.",+ "Double-check adherence to quantitative metrics, qualitative measures, and key characteristics.",+ ],+ output_instructions=[+ "Produce a well-structured, engaging Medium article in markdown format, with a STRICT target length of 3000 words (±100 words).",+ "Use a clear hierarchy of headers: # for main sections, ## for subsections, ### for sub-subsections.",+ "Seamlessly incorporate information from question-answer pairs, expanding with context and insights.",+ "Include a compelling title and subtitle reflecting the comprehensive nature of the article.",+ "Aim for 3-5 sentences per paragraph, allowing for a mix of detailed explanations and punchier points.",+ "Use bold for key concepts and italic for emphasis or to add a conversational tone.",+ "Ensure natural flow between sections, maintaining a coherent narrative.",+ "Include relevant code snippets using proper markdown syntax, followed by clear, relatable explanations.",+ "Use lists when appropriate, balancing them with explanatory paragraphs.",+ "Incorporate relevant subheadings within main sections to improve organization and scannability.",+ "Use block quotes for important statements or expert opinions, providing context and analysis with a personal touch.",+ "Conclude with a strong summary recapping main points and offering compelling insights, possibly with a call to action or thought-provoking question.",+ "Adhere to the specified quantitative metrics for sentence structure, vocabulary, and punctuation, but allow for some flexibility to maintain a natural flow.",+ "Maintain the required tone and style as outlined in the writer profile, leaning towards a more conversational approach.",+ "Demonstrate deep understanding while balancing technical details with accessible explanations.",+ "Use industry-specific jargon and technical terms appropriately, but always provide clear explanations or analogies for complex concepts.",+ "Provide critical analysis and emphasize practical implications of technical decisions, relating them to real-world scenarios when possible.",+ "Anticipate and address potential questions or areas of confusion proactively, using a friendly, guiding tone.",+ "Maintain high objectivity with analytical insights throughout the article.",+ "Maintain objectivity with analytical insights throughout the article, but don't shy away from adding occasional personal perspectives or experiences.",+ "Use rhetorical devices moderately, including examples, analogies, and occasional humor to keep the reader engaged.",+ "Incorporate direct address to the reader and use questions occasionally to create a more interactive feel.",+ "Ensure high anticipation of audience needs through relatable examples and clear explanations.",+ "Allow for moderate enthusiasm by using some intensifiers and positive language to convey excitement about the topic.",+ "Achieve the specified academic/specialized vocabulary ratio, but prioritize clarity and accessibility.",+ "Vary sentence openings and lengths to create a natural, flowing rhythm.",+ "IMPORTANT: The final article MUST be approximately 3000 words. If significantly shorter, expand key points or add more examples and analysis to reach the target length.",+ "IMPORTANT: The final article MUST flow nicely from section to section and have a single overarching narrative, with a conversational thread tying it all together.",+ "IMPORTANT: The final article MUST have a good balance between prose and lists, with a minimum of 80% being prose, and a maximum of 20% being lists",+ "IMPORTANT: If the article is about code, include plenty of didactic code snippets with clear, relatable explanations",+ ],+ ),+ input_schema=ArticleWriterAgentInputSchema,+ output_schema=ArticleWriterAgentOutputSchema,+ )+)diff --git a/examples/wiki_creator_agent/agents/outline_agent.py b/examples/wiki_creator_agent/agents/outline_agent.pynew file mode 100644index 0000000..5d25a1d--- /dev/null+++ b/examples/wiki_creator_agent/agents/outline_agent.py@@ -0,0 +1,86 @@+# File: outline_agent.py++import instructor+import openai+from pydantic import BaseModel, Field+from typing import List+from atomic_agents.agents.base_agent import BaseIOSchema, BaseAgent, BaseAgentConfig+from atomic_agents.lib.components.system_prompt_generator import SystemPromptGenerator+++class Question(BaseModel):+ question: str = Field(..., description="The question to answer in this section or subsection")+ potential_sources: List[str] = Field(+ default_factory=list, description="URLs of potential sources or references for this question. Required. Min. 1."+ )+++class OutlineSubSection(BaseModel):+ title: str = Field(..., description="The title of the outline subsection")+ questions_to_answer: List[Question] = Field(+ default_factory=list,+ description="A list of 2-4 hypothetical questions that the contents of this subsection should answer.",+ )+++class OutlineSection(BaseModel):+ title: str = Field(..., description="The title of the outline section")+ questions_to_answer: List[Question] = Field(+ default_factory=list, description="2-3 overarching questions to answer in this section"+ )+ subsections: List[OutlineSubSection] = Field(+ default_factory=list, description="Subsections of this outline section. Each section must have 3-5 subsections."+ )+++class OutlineAgentInputSchema(BaseIOSchema):+ """This schema defines the input schema for the OutlineAgent."""++ topic: str = Field(..., description="The main topic of the article")+++class OutlineAgentOutputSchema(BaseIOSchema):+ """This schema defines the output schema for the OutlineAgent."""++ internal_reasoning: List[str] = Field(+ ...,+ description="Internal reasoning about the context and what information it contains that could be useful in generating or modifying an outline.",+ )+ title: str = Field(..., description="The title of the article")+ outline: List[OutlineSection] = Field(+ ..., description="The generated or modified outline. The outline must have 5-7 main sections."+ )+++# Create the outline agent+outline_agent = BaseAgent(+ BaseAgentConfig(+ client=instructor.from_openai(openai.OpenAI()),+ model="gpt-4o-mini",+ system_prompt_generator=SystemPromptGenerator(+ background=[+ "You are an expert AI outline creator and editor, specializing in creating comprehensive and well-structured outlines.",+ "Your task is to generate or modify article outlines based on the given topic and context, ensuring a logical flow and thorough coverage of the subject.",+ ],+ steps=[+ "Analyze the given topic and any additional context provided in depth.",+ "Create a comprehensive structure for the article with 5-7 main sections.",+ "For each main section, develop 3-5 subsections that explore different aspects of the main section's topic.",+ "Ensure the outline is well-organized, logical, and covers all important aspects of the topic in a balanced manner.",+ "For each section, provide 2-3 overarching questions to answer.",+ "For each subsection, provide 2-4 specific questions to answer, including potential sources for each question.",+ ],+ output_instructions=[+ "Create a clear, concise, and comprehensive outline with 5-7 main sections.",+ "Ensure each main section has 3-5 well-defined subsections.",+ "Provide 1-2 overarching questions for each main section.",+ "Include 2-4 specific questions to answer for each subsection, along with potential sources.",+ "Use descriptive and informative titles for each section and subsection.",+ "Ensure each section and subsection is relevant to the main topic and contributes to a cohesive narrative.",+ "Aim for a balanced structure that covers the topic comprehensively, with appropriate depth in each section.",+ ],+ ),+ input_schema=OutlineAgentInputSchema,+ output_schema=OutlineAgentOutputSchema,+ )+)diff --git a/examples/wiki_creator_agent/agents/query_agent.py b/examples/wiki_creator_agent/agents/query_agent.pynew file mode 100644index 0000000..ef5d3f4--- /dev/null+++ b/examples/wiki_creator_agent/agents/query_agent.py@@ -0,0 +1,48 @@+# File: query_agent.py++import instructor+import openai+from pydantic import BaseModel, Field+from typing import List+from atomic_agents.agents.base_agent import BaseIOSchema, BaseAgent, BaseAgentConfig+from atomic_agents.lib.components.system_prompt_generator import SystemPromptGenerator+++class QueryAgentInputSchema(BaseIOSchema):+ """This schema defines the input schema for the QueryAgent."""++ instruction: str = Field(..., description="The user's input or question for which queries need to be generated.")+ num_queries: int = Field(default=5, description="The number of queries to generate.")+++class QueryAgentOutputSchema(BaseIOSchema):+ """This schema defines the output schema for the QueryAgent."""++ queries: List[str] = Field(..., description="A list of generated queries based on the user's input.")+++# Create the query agent+query_agent = BaseAgent(+ BaseAgentConfig(+ client=instructor.from_openai(openai.OpenAI()),+ model="gpt-4o-mini",+ system_prompt_generator=SystemPromptGenerator(+ background=[+ "You are an expert google query generator.",+ "Your task is to generate relevant and diverse google search queries based on the user's input or question.",+ ],+ steps=[+ "Analyze the user's input or question.",+ "Generate a specified number of relevant google search queries.",+ "Ensure the queries cover different aspects of the user's input.",+ ],+ output_instructions=[+ "Generate clear and concise google queries.",+ "Ensure each google query is directly relevant to the user's input.",+ "Avoid repetitive queries and aim for diversity in the generated set.",+ ],+ ),+ input_schema=QueryAgentInputSchema,+ output_schema=QueryAgentOutputSchema,+ )+)diff --git a/examples/wiki_creator_agent/agents/question_answering_agent.py b/examples/wiki_creator_agent/agents/question_answering_agent.pynew file mode 100644index 0000000..1c9a485--- /dev/null+++ b/examples/wiki_creator_agent/agents/question_answering_agent.py@@ -0,0 +1,42 @@+import instructor+import openai+from pydantic import BaseModel, Field+from atomic_agents.agents.base_agent import BaseAgent, BaseAgentConfig+from atomic_agents.lib.components.system_prompt_generator import SystemPromptGenerator+++class QuestionAnsweringAgentInputSchema(BaseModel):+ question: str = Field(..., description="The question to be answered")+++class QuestionAnsweringAgentOutputSchema(BaseModel):+ answer: str = Field(..., description="The answer to the provided question")+++# Create the question answering agent+question_answering_agent = BaseAgent(+ BaseAgentConfig(+ client=instructor.from_openai(openai.OpenAI()),+ model="gpt-4o-mini",+ system_prompt_generator=SystemPromptGenerator(+ background=[+ "You are an expert AI question answering system.",+ "Your task is to provide accurate and concise answers to questions based on the given context.",+ ],+ steps=[+ "Analyze the provided question and context.",+ "Search the context for relevant information.",+ "Generate a clear and concise answer based on the context.",+ "Identify the sources used for the answer.",+ ],+ output_instructions=[+ "Provide an accurate answer to the question.",+ "Ensure the answer is directly relevant to the question.",+ "Include sources for the answer when available.",+ "Keep the answer concise but informative.",+ ],+ ),+ input_schema=QuestionAnsweringAgentInputSchema,+ output_schema=QuestionAnsweringAgentOutputSchema,+ )+)diff --git a/examples/wiki_creator_agent/main.py b/examples/wiki_creator_agent/main.pynew file mode 100644index 0000000..bd0ad5f--- /dev/null+++ b/examples/wiki_creator_agent/main.py@@ -0,0 +1,229 @@+import os+import asyncio+from typing import List+import rich+from atomic_agents.lib.tools.search.searxng_tool import SearxNGTool, SearxNGToolConfig+from atomic_agents.lib.components.system_prompt_generator import SystemPromptContextProviderBase+from examples.wiki_creator_agent.agents.query_agent import query_agent, QueryAgentInputSchema+from examples.wiki_creator_agent.agents.outline_agent import outline_agent, OutlineAgentInputSchema+from examples.wiki_creator_agent.agents.question_answering_agent import (+ question_answering_agent,+ QuestionAnsweringAgentInputSchema,+)+from examples.wiki_creator_agent.utils.scraper import ContentScraper+from examples.wiki_creator_agent.utils.vector_db import InMemFaiss+from examples.wiki_creator_agent.agents.article_writer_agent import (+ article_writer_agent,+ ArticleWriterAgentInputSchema,+ QuestionAnswer,+)++rich_console = rich.console.Console()+++###################+# CONTEXT PROVIDERS #+###################+class SearchResultsProvider(SystemPromptContextProviderBase):+ def __init__(self, title="Search Results"):+ super().__init__(title)+ self.search_results = None++ def get_info(self) -> str:+ if self.search_results is None:+ return "No search results available."++ results_str = "Search Results:\n"+ for i, result in enumerate(self.search_results.results, 1):+ results_str += f"{i}. Title: {result.title}\n URL: {result.url}\n Content: {result.content[:100]}...\n\n"+ return results_str+++class VectorDBResultsProvider(SystemPromptContextProviderBase):+ def __init__(self, title="Vector Database Results"):+ super().__init__(title)+ self.vector_db_results = None++ def get_info(self) -> str:+ if self.vector_db_results is None:+ return "No vector database results available."++ results_str = "Vector Database Results:\n\n"+ for i, chunk in enumerate(self.vector_db_results, 1):+ results_str += f"## RESULT {i}\n"+ results_str += f"{chunk}\n"+ results_str += "=" * 50 + "\n\n"++ return results_str+++class OutlineProvider(SystemPromptContextProviderBase):+ def __init__(self, title="Article Outline"):+ super().__init__(title)+ self.outline = None++ def get_info(self) -> str:+ if self.outline is None:+ return "No outline available."++ outline_str = "Article Outline:\n"+ for section in self.outline:+ outline_str += f"Section: {section.title}\n"+ for subsection in section.subsections:+ outline_str += f" Subsection: {subsection.title}\n"+ return outline_str+++#############+# PIPELINES #+#############+class QueryAndSearchPipeline:+ @staticmethod+ def generate_queries_and_search(user_input: str, num_queries: int = 3):+ # Generate queries+ query_instruction = f"Generate research queries for an article about {user_input}"+ query_input = QueryAgentInputSchema(instruction=query_instruction, num_queries=num_queries)+ query_output = query_agent.run(query_input)+ queries = query_output.queries++ # Perform search+ search_tool_instance = SearxNGTool(config=SearxNGToolConfig(base_url=os.getenv("SEARXNG_BASE_URL"), max_results=30))+ search_input = SearxNGTool.input_schema(queries=queries)+ search_output = search_tool_instance.run(search_input)++ return queries, search_output+++class OutlineGenerationPipeline:+ @staticmethod+ def generate_outline(user_input: str, search_results_provider):+ outline_agent.register_context_provider("search_results", search_results_provider)+ outline_input = OutlineAgentInputSchema(topic=user_input)+ outline_output = outline_agent.run(outline_input)+ return outline_output.outline++ @staticmethod+ def extract_urls_from_outline(outline):+ urls = set()+ for section in outline:+ urls.update(url for point in section.questions_to_answer for url in point.potential_sources)+ for subsection in section.subsections:+ urls.update(url for point in subsection.questions_to_answer for url in point.potential_sources)+ return sorted(list(urls))+++class ContentScrapingPipeline:+ @staticmethod+ async def scrape_and_ingest_urls(urls: List[str], batch_size: int = 5):+ faiss_db = InMemFaiss(openai_api_key=os.getenv("OPENAI_API_KEY"))+ batches = [urls[i : i + batch_size] for i in range(0, len(urls), batch_size)]++ for batch_num, batch in enumerate(batches, 1):+ rich_console.print(f"[bold magenta]Scraping and ingesting batch {batch_num} of {len(batches)}[/bold magenta]")+ scraped_content = await ContentScraper.scrape_urls(batch)+ await faiss_db.ingest_documents(scraped_content)++ for url in scraped_content.keys():+ rich_console.print(f"[green]Ingested content from {url}[/green]")++ return faiss_db+++class QuestionAnsweringPipeline:+ @staticmethod+ async def answer_questions(outline, faiss_db, faiss_db_context_provider, outline_provider):+ question_answering_agent.register_context_provider("vector_db_results", faiss_db_context_provider)+ question_answering_agent.register_context_provider("outline", outline_provider)++ qa_pairs = []++ for section in outline:+ for subsection in section.subsections:+ questions_to_answer = "\n".join([item.question for item in subsection.questions_to_answer])+ context = faiss_db.retrieve_chunks(questions_to_answer)+ faiss_db_context_provider.vector_db_results = context++ for question_item in subsection.questions_to_answer:+ question_input = QuestionAnsweringAgentInputSchema(question=question_item.question)+ answer_output = question_answering_agent.run(question_input)++ rich_console.print(f"[bold cyan]Question:[/bold cyan] {question_item.question}")+ rich_console.print(f"[bold green]Answer:[/bold green] {answer_output.answer}\n")++ qa_pairs.append(QuestionAnswer(question=question_item.question, answer=answer_output.answer))++ return qa_pairs+++class ArticleWritingPipeline:+ @staticmethod+ def write_article(topic: str, outline, qa_pairs):+ input_data = ArticleWriterAgentInputSchema(topic=topic, outline=outline, qa_pairs=qa_pairs)+ output = article_writer_agent.run(input_data)+ return output.article+++###################+# MAIN APPLICATION #+###################+class WikiCreatorApplication:+ def __init__(self):+ self.rich_console = rich.console.Console()+ self.search_results_provider = SearchResultsProvider()+ self.vector_db_results_provider = VectorDBResultsProvider()+ self.outline_provider = OutlineProvider()+ self.qa_pairs = []++ async def run(self):+ self.rich_console.print("[bold blue]Script execution started[/bold blue]")+ user_input = input("I want to write an article about ")++ queries, search_output = QueryAndSearchPipeline.generate_queries_and_search(user_input, num_queries=5)+ self.rich_console.print("[bold yellow]Generated Queries:[/bold yellow]")+ for i, query in enumerate(queries, 1):+ self.rich_console.print(f"[yellow]{i}. {query}[/yellow]")+ self.rich_console.print()++ self.search_results_provider.search_results = search_output++ outline = OutlineGenerationPipeline.generate_outline(user_input, self.search_results_provider)+ self.rich_console.print("[bold green]Generated Outline:[/bold green]")+ self.rich_console.print(outline)++ urls = OutlineGenerationPipeline.extract_urls_from_outline(outline)+ self.rich_console.print("\n[bold cyan]Extracted Potential Source URLs:[/bold cyan]")+ for i, url in enumerate(urls, 1):+ self.rich_console.print(f"[cyan]{i}. {url}[/cyan]")++ batch_size = 10 # You can change this value or make it configurable+ faiss_db = await ContentScrapingPipeline.scrape_and_ingest_urls(urls, batch_size)++ self.outline_provider.outline = outline+ self.qa_pairs = await QuestionAnsweringPipeline.answer_questions(+ outline, faiss_db, self.vector_db_results_provider, self.outline_provider+ )++ # New code for article writing+ self.rich_console.print("[bold magenta]Writing the article...[/bold magenta]")+ article = ArticleWritingPipeline.write_article(user_input, outline, self.qa_pairs)++ self.rich_console.print("[bold green]Article generated successfully![/bold green]")+ self.rich_console.print("\n[bold cyan]Generated Article:[/bold cyan]")+ self.rich_console.print(article)++ # Optionally, save the article to a file+ with open("generated_article.md", "w", encoding="utf-8") as f:+ f.write(article)+ self.rich_console.print("\n[bold green]Article saved to 'generated_article.md'[/bold green]")+++####################+# SCRIPT EXECUTION #+####################+async def main():+ app = WikiCreatorApplication()+ await app.run()+++if __name__ == "__main__":+ asyncio.run(main())diff --git a/examples/wiki_creator_agent/utils/scraper.py b/examples/wiki_creator_agent/utils/scraper.pynew file mode 100644index 0000000..aa62404--- /dev/null+++ b/examples/wiki_creator_agent/utils/scraper.py@@ -0,0 +1,45 @@+import aiohttp+import asyncio+from bs4 import BeautifulSoup+from readability import Document+import html2text+from typing import List, Dict+import trafilatura+++class ContentScraper:+ @staticmethod+ async def scrape_urls(urls: List[str]) -> Dict[str, str]:+ async def fetch_url(session: aiohttp.ClientSession, url: str) -> tuple:+ try:+ async with session.get(url, timeout=30) as response: # Added timeout+ html_content = await response.text()++ # Try trafilatura first+ extracted_content = trafilatura.extract(html_content)++ if not extracted_content:+ # If trafilatura fails, fall back to readability+ doc = Document(html_content)+ extracted_content = doc.summary()++ # Clean up the extracted content+ soup = BeautifulSoup(extracted_content, "html.parser")+ for script in soup(["script", "style"]):+ script.decompose()+ cleaned_content = soup.get_text()++ # Convert to markdown+ h = html2text.HTML2Text()+ h.ignore_links = False+ markdown_content = h.handle(cleaned_content)++ return url, markdown_content.strip()+ except Exception as e:+ return url, f"Error scraping {url}: {str(e)}"++ async with aiohttp.ClientSession() as session:+ tasks = [fetch_url(session, url) for url in urls]+ results = await asyncio.gather(*tasks)++ return dict(results)diff --git a/examples/wiki_creator_agent/utils/vector_db.py b/examples/wiki_creator_agent/utils/vector_db.pynew file mode 100644index 0000000..3214261--- /dev/null+++ b/examples/wiki_creator_agent/utils/vector_db.py@@ -0,0 +1,57 @@+import openai+import faiss+import numpy as np+from typing import List, Dict+from tqdm import tqdm+++class InMemFaiss:+ def __init__(self, openai_api_key):+ openai.api_key = openai_api_key+ self.index = faiss.IndexFlatL2(1536)+ self.texts = []+ self.metadata = []++ def split_text(self, text, chunk_size=1750, overlap=250):+ chunks = []+ for i in range(0, len(text), chunk_size - overlap):+ chunk = text[i : i + chunk_size].strip()+ chunks.append(chunk)+ return chunks++ def generate_embeddings(self, texts):+ embeddings = []+ batch_size = 1000 # Adjust this based on your needs and API limits++ for i in tqdm(range(0, len(texts), batch_size)):+ batch = texts[i : i + batch_size]+ try:+ response = openai.embeddings.create(input=batch, model="text-embedding-3-small")+ batch_embeddings = [item.embedding for item in response.data]+ embeddings.extend(batch_embeddings)+ except Exception as e:+ print(f"Error generating embeddings for batch {i//batch_size}: {e}")+ # Handle the error appropriately, e.g., retry or skip++ return np.array(embeddings).astype("float32")++ async def ingest_documents(self, documents: Dict[str, str]):+ all_chunks = []+ all_metadata = []++ for url, content in documents.items():+ if len(content) == 0:+ continue+ chunks = self.split_text(content)+ all_chunks.extend(chunks)+ all_metadata.extend([{"url": url}] * len(chunks))++ embeddings = self.generate_embeddings(all_chunks)+ self.index.add(embeddings)+ self.texts.extend(all_chunks)+ self.metadata.extend(all_metadata)++ def retrieve_chunks(self, query, top_k=5):+ query_embedding = self.generate_embeddings([query])+ distances, indices = self.index.search(query_embedding, top_k)+ return [self.texts[idx] for idx in indices[0]]