🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: Source2synth (#1289)
CaelumF30dd8a7f02cd
diff --git a/camel/agents/multi_hop_generator_agent.py b/camel/agents/multi_hop_generator_agent.pynew file mode 100644index 000000000..a232fce84--- /dev/null+++ b/camel/agents/multi_hop_generator_agent.py@@ -0,0 +1,85 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import textwrap+from typing import Any++from pydantic import ConfigDict++from camel.agents.programmed_agent_instruction import (+ ProgrammableChatAgent,+ ProgrammedAgentInstructionResult,+ programmable_capability,+)+from camel.messages import BaseMessage+from camel.synthetic_datagen.source2synth.models import (+ ContextPrompt,+ MultiHopQA,+)+++class MultiHopGeneratorAgent(ProgrammableChatAgent):+ model_config = ConfigDict(arbitrary_types_allowed=True)++ def __init__(self, **kwargs: Any):+ super().__init__(**kwargs)++ system_text: str = textwrap.dedent(+ """\+ You are an expert at generating+ multi-hop question-answer pairs.+ For each context, you should:+ 1. Identify multiple related facts or pieces of information+ 2. Create questions that require reasoning across these multiple pieces+ 3. Ensure the reasoning chain is clear and logical+ 4. Generate questions that require at least 2-3 steps of reasoning+ 5. Include the reasoning steps in the answer++ Give your response with this information:+ Question: [Complex question requiring multiple reasoning steps]+ Reasoning Steps:+ 1. [First reasoning step]+ 2. [Second reasoning step]+ 3. [Final reasoning step]+ Answer: [Final answer]+ Supporting Facts: [List of relevant text segments used]+ """ # noqa: E501+ )+ self.system_message = BaseMessage.make_assistant_message(+ role_name='Assistant', content=system_text+ )++ @programmable_capability+ def generate_multi_hop_qa(+ self, context: str+ ) -> ProgrammedAgentInstructionResult[MultiHopQA]:+ context_prompt = ContextPrompt(+ main_context=context, related_contexts=None+ )++ user_message = BaseMessage.make_user_message(+ content=context_prompt.model_dump_json(), role_name="User"+ )+ response = self.step(+ input_message=user_message, response_format=MultiHopQA+ )+ value = MultiHopQA.model_validate_json(response.msgs[0].content)++ if response.msgs:+ return ProgrammedAgentInstructionResult(+ user_message=user_message,+ agent_message=response.msgs[0],+ value=value,+ )+ raise RuntimeError("No response from agent")diff --git a/camel/agents/programmed_agent_instruction.py b/camel/agents/programmed_agent_instruction.pynew file mode 100644index 000000000..708d5997b--- /dev/null+++ b/camel/agents/programmed_agent_instruction.py@@ -0,0 +1,148 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import abc+import threading+from enum import Enum+from functools import wraps+from typing import Any, Callable, Generic, Optional, TypeVar++from pydantic import BaseModel, ConfigDict++from camel.agents import ChatAgent+from camel.messages import BaseMessage++T = TypeVar('T')+++class ProgrammableAgentRequirement(Enum):+ LAST_MESSAGE_NOT_USER = "LAST_MESSAGE_NOT_USER"+++class ProgrammedAgentInstructionResult(BaseModel, Generic[T]):+ r"""Result of a programmable agent instruction execution.++ Contains the messages exchanged during execution and the computed value.+ The value type is specified by the generic type parameter T.+ """++ user_message: BaseMessage+ agent_message: BaseMessage+ value: T++ model_config = ConfigDict(arbitrary_types_allowed=True)+++class AbstractProgrammableAgent(abc.ABC):+ r"""Abstract class for a programmable agent.++ A programmable agent is an agent that can be programmed to perform a+ specific function or task. This class defines the interface for a+ programmable+ agent.++ These methods should be implemented in order to ensure the agent supports+ the necessary guarantees to enable a programming interface while+ maintaining compatibility in a multi-agent system.++ A programmable agent is responsible for providing and maintaining a+ programming interface for its functionality.+ """++ @abc.abstractmethod+ def run_atomic(+ self, callback: Callable[[], ProgrammedAgentInstructionResult[T]]+ ) -> ProgrammedAgentInstructionResult[T]:+ r"""Run an atomic operation on the agent.++ An atomic operation is an operation that is guaranteed to+ be executed without interruption by any other operation.++ If the operation fails or times out the agents state should be+ unchanged.++ If an operation is already in progress, this method should throw an+ exception. (It is up to the caller to do any queuing)++ If the agent is in a state where it can perform the operation,+ it must leave the agent in a state where it can perform the+ operation again. Though if state changes in successful operation+ improve its ability to perform the operation, it should keep them.+ """+ raise NotImplementedError++ @abc.abstractmethod+ def repair_state(self, requirement: ProgrammableAgentRequirement) -> None:+ r"""Repair the state of the agent.++ Agents may have other non-atomic interfaces, such as a user interface,+ or chat between other agents.++ This method should restore the agent to a state where it can perform+ operations according to the specified requirement.+ """+ raise NotImplementedError+++def programmable_capability(+ func: Callable[..., ProgrammedAgentInstructionResult[T]],+) -> Callable[..., ProgrammedAgentInstructionResult[T]]:+ r"""Decorator for programmable agent capabilities.++ Wraps a method to ensure it is executed atomically via the agent's+ run_atomic interface.+ The decorated method must return a ProgrammedAgentInstructionResult with+ appropriate type parameter.+ """++ @wraps(func)+ def wrapper(+ self, *args: Any, **kwargs: Any+ ) -> ProgrammedAgentInstructionResult[T]:+ return self.run_atomic(lambda: func(self, *args, **kwargs))++ return wrapper+++class ProgrammableChatAgent(ChatAgent, AbstractProgrammableAgent):+ r"""A chat agent that can be programmed to perform specific tasks.++ Provides a default implementation of atomic execution using threading locks+ and basic state tracking for message roles. Implementing classes need to+ provide specific repair logic for their use cases.+ """++ def __init__(self, **kwargs: Any):+ super().__init__(**kwargs)+ self._operation_lock = threading.Lock()+ self._last_message_role: Optional[str] = None++ def run_atomic(+ self, callback: Callable[[], ProgrammedAgentInstructionResult[T]]+ ) -> ProgrammedAgentInstructionResult[T]:+ if not self._operation_lock.acquire(blocking=False):+ raise RuntimeError("Operation already in progress")++ try:+ result = callback()+ self._last_message_role = result.agent_message.role_name+ return result+ finally:+ self._operation_lock.release()++ def repair_state(self, requirement: ProgrammableAgentRequirement) -> None:+ if requirement == ProgrammableAgentRequirement.LAST_MESSAGE_NOT_USER:+ if self._last_message_role == "user":+ raise NotImplementedError(+ "Must implement repair for LAST_MESSAGE_NOT_USER"+ )diff --git a/camel/synthetic_datagen/source2synth/data_processor.py b/camel/synthetic_datagen/source2synth/data_processor.pynew file mode 100644index 000000000..9780663c4--- /dev/null+++ b/camel/synthetic_datagen/source2synth/data_processor.py@@ -0,0 +1,373 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import random+from typing import Any, Dict, List, Optional, Sequence++import numpy as np+from tqdm import tqdm++from camel.agents.multi_hop_generator_agent import MultiHopGeneratorAgent+from camel.logger import get_logger+from camel.synthetic_datagen.source2synth.user_data_processor_config import (+ ProcessorConfig,+)++logger = get_logger(__name__)+++class UserDataProcessor:+ r"""User Data Processor."""++ def __init__(self, config: Optional[ProcessorConfig] = None):+ self.config = config or ProcessorConfig()+ random.seed(self.config.seed)+ np.random.seed(self.config.seed)+ self.multi_hop_agent = (+ MultiHopGeneratorAgent() if self.config.use_ai_model else None+ )++ def process_text(+ self, text: str, source: str = "user_input"+ ) -> List[Dict[str, Any]]:+ r"""Process a single text."""+ # Convert text to standard format+ raw_data = [+ {+ 'text': text,+ 'source': source,+ }+ ]++ # Construct examples+ constructor = ExampleConstructor(self.config, self.multi_hop_agent)+ examples = constructor.construct_examples(raw_data)++ # Manage data+ curator = DataCurator(self.config)+ final_dataset = curator.curate_dataset(examples)++ return final_dataset++ def process_batch(+ self, texts: List[str], sources: Optional[List[str]] = None+ ) -> List[Dict[str, Any]]:+ r"""Process multiple texts in batch."""+ if sources is None:+ sources = ["user_input"] * len(texts)+ elif len(sources) != len(texts):+ raise ValueError("Length of sources must match length of texts")++ raw_data = [+ {+ 'text': text,+ 'source': source,+ }+ for text, source in zip(texts, sources)+ ]++ # Construct examples+ constructor = ExampleConstructor(self.config, self.multi_hop_agent)+ examples = constructor.construct_examples(raw_data)++ # Manage data+ curator = DataCurator(self.config)+ final_dataset = curator.curate_dataset(examples)++ return final_dataset+++class ExampleConstructor:+ r"""Example Constructor."""++ def __init__(+ self,+ config: ProcessorConfig,+ multi_hop_agent: Optional[MultiHopGeneratorAgent] = None,+ ):+ self.config = config+ self.multi_hop_agent = multi_hop_agent++ def construct_examples(+ self, raw_data: List[Dict[str, Any]]+ ) -> List[Dict[str, Any]]:+ r"""Construct training examples."""+ logger.info("Starting to construct training examples...")+ examples = []++ for data in tqdm(raw_data, desc="Constructing examples"):+ # 1. Text preprocessing+ processed_text = self._preprocess_text(data.get('text', ''))+ if not processed_text:+ continue++ # 2. Generate key information pairs+ info_pairs = self._extract_info_pairs(processed_text)++ # 3. Construct question-answer pairs+ qa_pairs = self._generate_qa_pairs(info_pairs)++ # 4. Add metadata+ example = {+ 'text': processed_text,+ 'qa_pairs': qa_pairs,+ 'metadata': {+ 'source': data.get('source', 'unknown'),+ 'timestamp': data.get('timestamp', ''),+ 'complexity': self._calculate_complexity(qa_pairs),+ },+ }++ examples.append(example)++ logger.info(f"Successfully constructed {len(examples)} examples")+ return examples++ def _preprocess_text(self, text: str) -> str:+ r"""Text preprocessing."""+ if not isinstance(text, str):+ return ''++ # 1. Basic cleaning+ text = text.strip()++ # 2. Length check+ if (+ len(text) < self.config.min_length+ or len(text) > self.config.max_length+ ):+ return ''++ # 3. Quality check+ if not self._check_text_quality(text):+ return ''++ return text++ def _check_text_quality(self, text: str) -> bool:+ r"""Check text quality."""+ # 1. Basic quality check+ if text.count('.') < 2: # Must have at least 2 sentences+ return False++ # 2. Special character ratio check+ special_char_ratio = len(+ [c for c in text if not c.isalnum() and not c.isspace()]+ ) / len(text)+ if special_char_ratio > 0.3: # No more than 30% special characters+ return False++ return True++ def _extract_info_pairs(self, text: str) -> List[Dict[str, Sequence[str]]]:+ r"""Extract information pairs and relationships."""+ # Split into sentences+ sentences = [s.strip() for s in text.split('.') if s.strip()]+ info_pairs = []++ # Extract combinations of multiple related sentences+ for i in range(len(sentences) - 2):+ if len(sentences[i]) > 10 and len(sentences[i + 1]) > 10:+ info_pairs.append(+ {+ 'premise': sentences[i],+ 'intermediate': sentences[i + 1],+ 'conclusion': sentences[i + 2]+ if i + 2 < len(sentences)+ else '',+ 'related_contexts': [+ s+ for j, s in enumerate(sentences)+ if j != i and j != i + 1 and len(s) > 10+ ][:2],+ # Limit to 2 additional related contexts+ }+ )++ return info_pairs++ def _generate_qa_pairs(+ self, info_pairs: List[Dict[str, Sequence[str]]]+ ) -> List[Dict[str, str]]:+ r"""Generate multi-hop question-answer pairs."""+ qa_pairs = []++ for pair in info_pairs:+ # 1. Generate multi-hop question-answer pair using AI+ if self.multi_hop_agent:+ # Construct full context+ context = (+ f"{pair['premise']}. {pair['intermediate']}."+ f" {pair['conclusion']}"+ )+ response = self.multi_hop_agent.generate_multi_hop_qa(context)+ if response:+ qa_pairs.append(response.value.dict())+ continue++ return qa_pairs++ def _calculate_complexity(self, qa_pairs: List[Dict[str, Any]]) -> float:+ r"""Calculate complexity of QA pairs."""+ if not qa_pairs:+ return 0.0++ # Calculate complexity based on multiple factors+ complexities = []+ for qa in qa_pairs:+ # 1. Number of reasoning steps+ reasoning_steps_count = len(qa.get('reasoning_steps', []))++ # 2. Number of supporting facts+ supporting_facts_count = len(qa.get('supporting_facts', []))++ # 3. Question length+ question_length = len(qa['question'].split())++ # 4. Answer length+ answer_length = len(qa['answer'].split())++ # Calculate complexity of a single QA pair+ qa_complexity = (+ min(reasoning_steps_count / 3, 1.0)+ * 0.4 # Weight for reasoning steps+ + min(supporting_facts_count / 3, 1.0)+ * 0.3 # Weight for supporting facts+ + min(question_length / 20, 1.0)+ * 0.15 # Weight for question length+ + min(answer_length / 50, 1.0) * 0.15+ # Weight for answer length+ )++ complexities.append(qa_complexity)++ return sum(complexities) / len(complexities)+++class DataCurator:+ r"""Data Manager."""++ def __init__(self, config: ProcessorConfig):+ self.config = config++ def curate_dataset(+ self, examples: List[Dict[str, Any]]+ ) -> List[Dict[str, Any]]:+ r"""Dataset management."""+ logger.info("Starting dataset management...")++ # 1. Quality filtering+ quality_filtered = self._quality_filter(examples)+ logger.info(+ f"Remaining examples after quality filtering:"+ f" {len(quality_filtered)}"+ )++ # 2. Complexity filtering+ complexity_filtered = self._complexity_filter(quality_filtered)+ logger.info(+ f"Remaining examples after complexity filtering:"+ f" {len(complexity_filtered)}"+ )++ # 3. Deduplication+ deduplicated = self._remove_duplicates(complexity_filtered)+ logger.info(+ f"Remaining examples after deduplication: {len(deduplicated)}"+ )++ # 4. Sample to target size+ final_dataset = self._sample_dataset(deduplicated)+ logger.info(f"Final dataset size: {len(final_dataset)}")++ return final_dataset++ def _quality_filter(+ self, examples: List[Dict[str, Any]]+ ) -> List[Dict[str, Any]]:+ r"""Quality filtering."""+ filtered = []++ for example in examples:+ # 1. Check QA pair quality+ qa_quality = self._check_qa_quality(example.get('qa_pairs', []))++ # 2. Check text quality+ text_quality = (+ len(example.get('text', '').split()) >= 20+ ) # At least 20 words++ if qa_quality and text_quality:+ filtered.append(example)++ return filtered++ def _check_qa_quality(self, qa_pairs: List[Dict[str, str]]) -> bool:+ r"""Check quality of QA pairs."""+ if not qa_pairs:+ return False++ for qa in qa_pairs:+ # 1. Length check+ if (+ len(qa.get('question', '')) < 10+ or len(qa.get('answer', '')) < 5+ ):+ return False++ # 2. QA pair duplication check+ if qa.get('question', '') == qa.get('answer', ''):+ return False++ return True++ def _complexity_filter(+ self, examples: List[Dict[str, Any]]+ ) -> List[Dict[str, Any]]:+ r"""Complexity filtering."""+ return [+ example+ for example in examples+ if example.get('metadata', {}).get('complexity', 0)+ >= self.config.complexity_threshold+ ]++ def _remove_duplicates(+ self, examples: List[Dict[str, Any]]+ ) -> List[Dict[str, Any]]:+ r"""Remove duplicates."""+ seen = set()+ unique_examples = []++ for example in examples:+ # Use text and QA pair combination as unique identifier+ text = example.get('text', '')+ qa_str = str(example.get('qa_pairs', []))++ identifier = hash(text + qa_str)++ if identifier not in seen:+ seen.add(identifier)+ unique_examples.append(example)++ return unique_examples++ def _sample_dataset(+ self, examples: List[Dict[str, Any]]+ ) -> List[Dict[str, Any]]:+ r"""Sample to target dataset size."""+ if len(examples) <= self.config.dataset_size:+ return examples++ return random.sample(examples, self.config.dataset_size)diff --git a/camel/synthetic_datagen/source2synth/models.py b/camel/synthetic_datagen/source2synth/models.pynew file mode 100644index 000000000..568581411--- /dev/null+++ b/camel/synthetic_datagen/source2synth/models.py@@ -0,0 +1,68 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from typing import Any, ClassVar, Dict, List, Optional++from pydantic import BaseModel, Field+++class ReasoningStep(BaseModel):+ step: str = Field(+ ..., description="A single step in the reasoning process."+ )+++class MultiHopQA(BaseModel):+ question: str = Field(+ ..., description="The question that requires multi-hop reasoning."+ )+ reasoning_steps: List[ReasoningStep] = Field(+ ...,+ description="The steps involved in reasoning to answer the question.",+ )+ answer: str = Field(+ ..., description="The answer to the multi-hop question."+ )+ supporting_facts: List[str] = Field(+ ..., description="Facts that support the reasoning and answer."+ )+ type: str = Field(description="The type of question-answer pair.")++ class Config:+ json_schema_extra: ClassVar[Dict[str, Any]] = {+ "example": {+ "question": "What is the capital of France?",+ "reasoning_steps": [+ {"step": "Identify the country France."},+ {"step": "Find the capital city of France."},+ ],+ "answer": "Paris",+ "supporting_facts": [+ "France is a country in Europe.",+ "Paris is the capital city of France.",+ ],+ "type": "multi_hop_qa",+ }+ }+++class ContextPrompt(BaseModel):+ main_context: str = Field(+ ...,+ description="The main context for generating"+ " the question-answer pair.",+ )+ related_contexts: Optional[List[str]] = Field(+ default=None,+ description="Additional contexts related to the main context.",+ )diff --git a/camel/synthetic_datagen/source2synth/user_data_processor_config.py b/camel/synthetic_datagen/source2synth/user_data_processor_config.pynew file mode 100644index 000000000..9b99b9b83--- /dev/null+++ b/camel/synthetic_datagen/source2synth/user_data_processor_config.py@@ -0,0 +1,73 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import random++from pydantic import BaseModel, ConfigDict, Field++from camel.agents.multi_hop_generator_agent import MultiHopGeneratorAgent+++class ProcessorConfig(BaseModel):+ r"""Data processing configuration class"""++ def __repr__(self):+ return "MultiHopGeneratorAgent()"++ model_config = ConfigDict(+ validate_assignment=True,+ frozen=False,+ protected_namespaces=(),+ arbitrary_types_allowed=True,+ )++ seed: int = Field( # Generate a random seed for reproducibility+ default_factory=lambda: random.randint(0, 1000),+ description="Random seed for reproducibility",+ )++ min_length: int = Field(+ default=50, description="Minimum text length", ge=0+ )++ max_length: int = Field(+ default=512, description="Maximum text length", gt=0+ )++ quality_threshold: float = Field(+ default=0.7,+ description="Quality threshold for processing",+ ge=0.0,+ le=1.0,+ )++ complexity_threshold: float = Field(+ default=0.5,+ description="Complexity threshold for processing",+ ge=0.0,+ le=1.0,+ )++ dataset_size: int = Field(+ default=1000, description="Target size of the dataset", gt=0+ )++ use_ai_model: bool = Field(+ default=True, description="Whether to use AI model in processing"+ )++ hop_generating_agent: MultiHopGeneratorAgent = Field(+ default_factory=lambda: MultiHopGeneratorAgent(),+ description="Agent for generating multi-hop text",+ )diff --git a/examples/source2synth.py b/examples/source2synth.pynew file mode 100644index 000000000..ba2549aed--- /dev/null+++ b/examples/source2synth.py@@ -0,0 +1,173 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import json+import logging++from camel.synthetic_datagen.source2synth.data_processor import (+ UserDataProcessor,+)+from camel.synthetic_datagen.source2synth.user_data_processor_config import (+ ProcessorConfig,+)++# Configure logging+logging.basicConfig(+ level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s'+)+logger = logging.getLogger(__name__)+++def save_results(results, output_file: str):+ r"""Save results to a JSON file."""+ with open(output_file, 'w', encoding='utf-8') as f:+ json.dump(results, f, ensure_ascii=False, indent=2)+ logger.info(f"Results saved to: {output_file}")+++def main():+ r"""Example usage."""+ # 1. Create configuration+ config = ProcessorConfig(+ seed=42,+ min_length=50,+ max_length=1000,+ quality_threshold=0.7,+ complexity_threshold=0.5,+ dataset_size=10,+ use_ai_model=True,+ )++ # 2. Create the processor+ processor = UserDataProcessor(config)++ # 3. Prepare test data - texts containing multiple related information+ test_texts = [+ # Chain of technological developments+ """+ The invention of transistors revolutionized electronics in the 1950s.+ These tiny semiconductor devices enabled the development of smaller and more+ efficient computers. The miniaturization of computers led to the creation of+ personal computers in the 1980s, which transformed how people work and communicate.+ This digital revolution eventually gave rise to the internet, connecting billions+ of people worldwide. Today, this interconnected network powers artificial+ intelligence systems that are reshaping various industries.+ """, # noqa: E501+ # Environmental changes causation chain+ """+ Industrial activities have significantly increased carbon dioxide emissions since+ the Industrial Revolution. These elevated CO2 levels have enhanced the greenhouse+ effect, trapping more heat in Earth's atmosphere. The rising global temperatures+ have accelerated the melting of polar ice caps, which has led to rising sea levels.+ Coastal communities are now facing increased flooding risks, forcing many to+ consider relocation. This migration pattern is creating new challenges for urban+ planning and resource management.+ """, # noqa: E501+ # Biological evolution chain+ """+ The discovery of antibiotics began with Alexander Fleming's observation of+ penicillin in 1928. The widespread use of these medications has saved countless+ lives from bacterial infections. However, the extensive use of antibiotics has+ led to the evolution of resistant bacteria strains. These superbugs now pose+ a significant challenge to modern medicine, requiring the development of new+ treatment approaches. Scientists are exploring alternative solutions like+ bacteriophage therapy to combat antibiotic resistance.+ """, # noqa: E501+ ]++ # 4. Process a single text+ logger.info("Processing a single text example...")+ single_result = processor.process_text(+ test_texts[0], source="technology_evolution"+ )++ # Save the single text processing result+ save_results(single_result, "single_text_results.json")++ # 5. Batch process texts+ logger.info("Batch processing texts...")+ batch_results = processor.process_batch(+ test_texts,+ sources=["tech_evolution", "climate_change", "medical_evolution"],+ )++ # Save the batch processing results+ save_results(batch_results, "batch_results.json")++ # 6. Print example results+ print("\n=== Single Text Processing Example ===")+ if single_result:+ for i, result in enumerate(single_result, 1):+ print(f"\nText {i}:")+ print(f"Source: {result['metadata']['source']}")+ print(f"Complexity: {result['metadata']['complexity']:.2f}")+ print("\nQ&A Pairs:")+ for j, qa in enumerate(result['qa_pairs'], 1):+ print(f"\nQ&A Pair {j}:")+ print(f"Type: {qa['type']}")+ print(f"Question: {qa['question']}")+ print("Reasoning Steps:")+ for step_num, step in enumerate(+ qa.get('reasoning_steps', []), 1+ ):+ print(f"{step_num}. {step}")+ print(f"Answer: {qa['answer']}")+ print("Supporting Facts:")+ for fact_num, fact in enumerate(+ qa.get('supporting_facts', []), 1+ ):+ print(f"{fact_num}. {fact}")++ print("\n=== Batch Processing Statistics ===")+ print(f"Total texts processed: {len(test_texts)}")+ pairs_generated = sum(len(result['qa_pairs']) for result in batch_results)+ print(f"Total Q&A pairs generated: {pairs_generated}")++ # 7. Analyze results+ multi_hop_qa = sum(+ 1+ for result in batch_results+ for qa in result['qa_pairs']+ if qa['type'] == 'multi_hop_qa'+ )+ template_generated = sum(+ 1+ for result in batch_results+ for qa in result['qa_pairs']+ if qa['type'] == 'template_generated_multi_hop'+ )++ print("\n=== Generation Statistics ===")+ print(f"AI-generated multi-hop Q&A count: {multi_hop_qa}")+ print(f"Template-generated multi-hop Q&A count: {template_generated}")++ # 8. Analyze reasoning steps+ avg_steps = sum(+ len(qa.get('reasoning_steps', []))+ for result in batch_results+ for qa in result['qa_pairs']+ ) / sum(len(result['qa_pairs']) for result in batch_results)++ print(f"\nAverage reasoning steps: {avg_steps:.2f}")++ # 9. Calculate average complexity+ avg_complexity = sum(+ result['metadata']['complexity'] for result in batch_results+ ) / len(batch_results)++ print(f"Average complexity score: {avg_complexity:.2f}")+++if __name__ == "__main__":+ main()