Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

45.5 KB
Raw Blame
# ========= Copyright 2023-2026 @ CAMEL-AI.org. All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2023-2026 @ CAMEL-AI.org. All Rights Reserved. =========
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# ruff: noqa: E501

#!/usr/bin/env python3
import argparse
import ast
import glob
import importlib
import json
import os
import re
import time
from collections import defaultdict
from pathlib import Path

# Module name to display name mapping
MODULE_NAME_DISPLAY = {
    "agents": "Agents",
    "configs": "Configs",
    "datagen": "Data Generation",
    "datasets": "Datasets",
    "data_collector": "Data Collector",
    "embeddings": "Embeddings",
    "environments": "Environments",
    "interpreters": "Interpreters",
    "loaders": "Loaders",
    "memories": "Memory",
    "messages": "Messages",
    "models": "Models",
    "prompts": "Prompts",
    "responses": "Responses",
    "retrievers": "Retrievers",
    "runtime": "Runtime",
    "schemas": "Schemas",
    "societies": "Societies",
    "storages": "Storage",
    "tasks": "Tasks",
    "terminators": "Terminators",
    "toolkits": "Toolkits",
    "types": "Types",
    "utils": "Utilities",
    "verifiers": "Verifiers",
    "benchmarks": "Benchmarks",
    "bots": "Bots",
    "personas": "Personas",
    "extractors": "Extractors",
}

# Custom order, this determines the display order of top-level modules
MODULE_ORDER = [
    "agents",
    "configs",
    "datagen",
    "datasets",
    "embeddings",
    "models",
    "interpreters",
    "memories",
    "messages",
    "prompts",
    "responses",
    "retrievers",
    "societies",
    "storages",
    "tasks",
    "terminators",
    "toolkits",
    "types",
    "verifiers",
    "bots",
    "runtime",
    "utils",
    "environments",
    "extractors",
    "personas",
]


def get_module_display_name(module_name):
    """Get the beautiful display name for a module"""
    if module_name in MODULE_NAME_DISPLAY:
        return MODULE_NAME_DISPLAY[module_name]
    # If no mapping exists, convert snake_case to title case
    return module_name.replace('_', ' ').title()


def get_all_modules(package_name="camel", recursive=True):
    """Get all modules in the package"""
    modules = []

    try:
        # Import main package
        package = importlib.import_module(package_name)
        modules.append(package_name)

        # Get package path
        package_path = os.path.dirname(package.__file__)

        # Traverse all Python files in the package
        for root, dirs, files in os.walk(package_path):
            if not recursive and root != package_path:
                continue

            for file in files:
                if file.endswith(".py") and file != "__init__.py":
                    # Calculate relative path of the module
                    rel_path = os.path.relpath(
                        os.path.join(root, file), os.path.dirname(package_path)
                    )
                    # Convert to module name
                    module_name = os.path.splitext(rel_path)[0].replace(
                        os.sep, "."
                    )
                    modules.append(module_name)

            # Handle subpackages
            for dir_name in dirs:
                if os.path.isfile(os.path.join(root, dir_name, "__init__.py")):
                    # Calculate relative path of the subpackage
                    rel_path = os.path.relpath(
                        os.path.join(root, dir_name),
                        os.path.dirname(package_path),
                    )
                    # Convert to package name
                    subpackage_name = rel_path.replace(os.sep, ".")
                    modules.append(subpackage_name)

    except ImportError as e:
        print(f"Error importing {package_name}: {e}")

    return sorted(modules)


def get_changed_modules(package_name="camel", since_hours=24):
    """Get recently modified modules (for incremental updates)"""
    changed_modules = []

    try:
        # Import main package
        package = importlib.import_module(package_name)
        package_path = os.path.dirname(package.__file__)

        # Calculate time threshold
        time_threshold = time.time() - (since_hours * 3600)

        # Traverse all Python files in the package
        for root, _dirs, files in os.walk(package_path):
            for file in files:
                if file.endswith(".py"):
                    file_path = os.path.join(root, file)

                    # Check file modification time
                    if os.path.getmtime(file_path) > time_threshold:
                        if file == "__init__.py":
                            # Handle package
                            rel_path = os.path.relpath(
                                root, os.path.dirname(package_path)
                            )
                            module_name = rel_path.replace(os.sep, ".")
                            changed_modules.append(module_name)
                        else:
                            # Handle module
                            rel_path = os.path.relpath(
                                file_path, os.path.dirname(package_path)
                            )
                            module_name = os.path.splitext(rel_path)[
                                0
                            ].replace(os.sep, ".")
                            changed_modules.append(module_name)

    except ImportError as e:
        print(f"Error importing {package_name}: {e}")

    return sorted(set(changed_modules))


def is_content_substantial(content):
    """Check if content is substantial enough to avoid generating empty documentation"""
    if not content.strip():
        return False

    # Remove empty lines and common useless content
    lines = [line.strip() for line in content.split('\n') if line.strip()]

    # Filter out cases with only headers
    substantial_lines = []
    for line in lines:
        # Skip header lines
        if line.startswith('#'):
            continue
        # Skip lines with only module IDs
        if line.startswith('<a id='):
            continue
        # Skip empty code blocks
        if line in ['```', '```python']:
            continue
        substantial_lines.append(line)

    # If substantial content is less than 3 lines, consider it insufficient
    return len(substantial_lines) >= 3


def build_module_tree(mdx_files):
    """Build module tree based on MDX file names"""

    # Factory function to create new submodule dictionary
    def new_module_dict():
        return {"pages": [], "submodules": defaultdict(new_module_dict)}

    # Build tree structure using nested defaultdict
    module_tree = new_module_dict()

    for file in mdx_files:
        # Get module path from file name
        module_path = file.stem  # Remove .mdx suffix

        # Usually we expect file names in "camel.xxx.yyy" format
        if not module_path.startswith("camel."):
            # If not a camel module, skip
            continue

        # Split module path
        parts = module_path.split('.')

        # Build reference path
        reference_path = f"reference/{module_path}"

        # If it's just the camel module itself
        if len(parts) == 1:
            module_tree["pages"].append(reference_path)
            continue

        # Handle submodules
        current = module_tree
        for i, part in enumerate(
            parts[:-1]
        ):  # Exclude the last part, which is the file name
            if i == 0:  # camel root module
                continue

            # Navigate to submodule
            current = current["submodules"][part]

        # Ensure pages list exists
        if "pages" not in current:
            current["pages"] = []

        # Ensure directory pages (like camel.agents/camel.agents.mdx) appear before their submodules
        if parts[-1] == parts[-2]:
            current["pages"].insert(0, reference_path)
        else:
            current["pages"].append(reference_path)

    return module_tree


def convert_tree_to_navigation(module_tree):
    """Convert module tree to mint.json navigation format"""
    navigation = []

    # Add submodules in custom order
    for module_name in MODULE_ORDER:
        if module_name in module_tree["submodules"]:
            submodule = module_tree["submodules"][module_name]

            # Get pretty display name for submodule
            display_name = get_module_display_name(module_name)

            # Create navigation group
            nav_group = {"group": display_name, "pages": []}

            # Add direct pages for this module
            if "pages" in submodule:
                nav_group["pages"].extend(sorted(submodule["pages"]))

            # Recursively handle submodules
            if submodule.get("submodules"):
                for _sub_name, sub_data in sorted(
                    submodule["submodules"].items()
                ):
                    if sub_data.get("pages"):
                        # Directly flatten submodule pages
                        nav_group["pages"].extend(sorted(sub_data["pages"]))

            # Only add group if it has pages
            if nav_group["pages"]:
                navigation.append(nav_group)

    return navigation


def update_docs_json(docs_json_path, navigation):
    """Update the API Reference navigation section in docs.json file"""
    if not Path(docs_json_path).exists():
        print(f"Error: {docs_json_path} not found")
        return False

    with open(docs_json_path, "r", encoding="utf-8") as f:
        docs_data = json.load(f)

    # Find API Reference tab and update its groups
    tabs = docs_data.get("navigation", {}).get("tabs", [])
    for tab in tabs:
        if tab.get("tab") == "API Reference":
            # Preserve existing Overview group if it exists
            existing_groups = tab.get("groups", [])
            overview_group = None

            # Look for existing Overview group
            for group in existing_groups:
                if group.get("group") == "Overview":
                    overview_group = group
                    break

            # Start with Overview group if it exists, otherwise create a default one
            if overview_group:
                new_groups = [overview_group]
            else:
                new_groups = [
                    {"group": "Overview", "pages": ["reference/index"]}
                ]

            # Add the generated navigation groups
            new_groups.extend(navigation)

            # Update the tab groups
            tab["groups"] = new_groups
            break

    # Save updated docs.json
    with open(docs_json_path, "w", encoding="utf-8") as f:
        json.dump(docs_data, f, indent=2, ensure_ascii=False)

    return True


def parse_docstring(docstring):
    """Parse a docstring and extract structured information"""
    if not docstring:
        return {
            'description': '',
            'args': [],
            'returns': '',
            'raises': [],
            'examples': [],
            'notes': [],
        }

    lines = docstring.strip().split('\n')
    result = {
        'description': '',
        'args': [],
        'returns': '',
        'raises': [],
        'examples': [],
        'notes': [],
    }

    current_section = 'description'
    current_lines = []

    for line in lines:
        line = line.strip()

        # Check for section headers
        if line.lower().startswith(
            ('args:', 'arguments:', 'parameters:', 'param:', 'attributes:')
        ):
            if current_lines and current_section == 'description':
                result['description'] = '\n'.join(current_lines).strip()
            current_section = 'args'
            current_lines = []
        elif line.lower().startswith(('returns:', 'return:')):
            if current_lines and current_section == 'args':
                result['args'].extend(parse_args_section(current_lines))
            current_section = 'returns'
            current_lines = []
        elif line.lower().startswith(
            ('raises:', 'raise:', 'except:', 'exceptions:')
        ):
            if current_lines:
                if current_section == 'args':
                    result['args'].extend(parse_args_section(current_lines))
                elif current_section == 'returns':
                    result['returns'] = '\n'.join(current_lines).strip()
            current_section = 'raises'
            current_lines = []
        elif line.lower().startswith(('examples:', 'example:')):
            if current_lines:
                if current_section == 'args':
                    result['args'].extend(parse_args_section(current_lines))
                elif current_section == 'returns':
                    result['returns'] = '\n'.join(current_lines).strip()
                elif current_section == 'raises':
                    result['raises'].extend(
                        parse_raises_section(current_lines)
                    )
            current_section = 'examples'
            current_lines = []
        elif line.lower().startswith(
            ('note:', 'notes:', 'warning:', 'warnings:')
        ):
            if current_lines:
                if current_section == 'args':
                    result['args'].extend(parse_args_section(current_lines))
                elif current_section == 'returns':
                    result['returns'] = '\n'.join(current_lines).strip()
                elif current_section == 'raises':
                    result['raises'].extend(
                        parse_raises_section(current_lines)
                    )
                elif current_section == 'examples':
                    result['examples'] = current_lines
                elif current_section == 'description':
                    result['description'] = '\n'.join(current_lines).strip()
            current_section = 'notes'
            current_lines = []
        else:
            current_lines.append(line)

    # Process final section
    if current_lines:
        if current_section == 'description':
            result['description'] = '\n'.join(current_lines).strip()
        elif current_section == 'args':
            result['args'].extend(parse_args_section(current_lines))
        elif current_section == 'returns':
            result['returns'] = '\n'.join(current_lines).strip()
        elif current_section == 'raises':
            result['raises'].extend(parse_raises_section(current_lines))
        elif current_section == 'examples':
            result['examples'] = current_lines
        elif current_section == 'notes':
            result['notes'] = current_lines

    return result


def parse_args_section(lines):
    """Parse arguments section and extract parameter info"""
    args = []
    current_arg = None

    for line in lines:
        line = line.strip()
        if not line:
            continue

        # Check if this line starts a new parameter (has parameter name followed by type and colon)
        if re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*\s*(\([^)]+\))?\s*:', line):
            # Save previous arg
            if current_arg:
                # Clean up description
                current_arg['description'] = re.sub(
                    r'\s+', ' ', current_arg['description']
                ).strip()
                args.append(current_arg)

            # Parse new arg
            parts = line.split(':', 1)
            if len(parts) == 2:
                name_type = parts[0].strip()
                description = parts[1].strip()

                # Extract name and type
                type_match = re.match(
                    r'^([a-zA-Z_][a-zA-Z0-9_]*)\s*\(([^)]+)\)', name_type
                )
                if type_match:
                    name = type_match.group(1).strip()
                    type_info = type_match.group(2).strip()
                else:
                    name = name_type.strip()
                    type_info = ''

                current_arg = {
                    'name': name,
                    'type': type_info,
                    'description': description,
                    'optional': 'optional' in description.lower(),
                    'default': extract_default_value(description),
                }
        else:
            # Continuation of current arg description
            if current_arg:
                current_arg['description'] += ' ' + line

    # Add final arg
    if current_arg:
        # Clean up final description
        current_arg['description'] = re.sub(
            r'\s+', ' ', current_arg['description']
        ).strip()
        args.append(current_arg)

    return args


def parse_raises_section(lines):
    """Parse raises section"""
    raises = []
    for line in lines:
        line = line.strip()
        if ':' in line:
            parts = line.split(':', 1)
            if len(parts) == 2:
                exception = parts[0].strip()
                description = parts[1].strip()
                raises.append(
                    {'exception': exception, 'description': description}
                )
    return raises


def extract_default_value(description):
    """Extract default value from parameter description"""
    # Look for patterns like (default: value) or (default: :obj:`value`)

    patterns = [
        r'\(default:\s*:obj:`([^`]+)`\)',
        r'\(default:\s*([^)]+)\)',
        r'defaults?\s+to\s+([^.,:;)]+)',
        r'\(default:\s*([^)]*)\)',  # More flexible pattern
    ]

    for pattern in patterns:
        match = re.search(pattern, description, re.IGNORECASE)
        if match:
            default_val = match.group(1).strip()
            # Clean up common artifacts
            default_val = default_val.replace(':obj:', '').replace('`', '')
            # Remove trailing punctuation
            default_val = re.sub(r'[.,:;]+$', '', default_val)
            if default_val and default_val not in ['None', 'obj']:
                return default_val

    return None


def escape_mdx_content(text):
    """Escape special characters in text content for MDX compatibility"""
    if not text:
        return text

    # Store code blocks and inline code to protect them from escaping
    code_blocks = []
    inline_codes = []

    # Extract and temporarily replace code blocks (```)
    def extract_code_block(match):
        code_blocks.append(match.group(0))
        return f"__CODE_BLOCK_{len(code_blocks) - 1}__"

    text = re.sub(r'```[\s\S]*?```', extract_code_block, text)

    # Extract and temporarily replace inline code (`) - including our newly created ones
    def extract_inline_code(match):
        inline_codes.append(match.group(0))
        return f"__INLINE_CODE_{len(inline_codes) - 1}__"

    # First extract existing inline code
    text = re.sub(r'`[^`\n]+`', extract_inline_code, text)

    # Handle reStructuredText-style links: `text <url>`_ -> [text](url)
    def convert_rst_links(text):
        """Convert reStructuredText links to Markdown format"""
        # Pattern for `text <url>`_ format
        rst_link_pattern = r'`([^`]+)\s+<([^>]+)>`_'

        def replace_rst_link(match):
            text_part = match.group(1).strip()
            url_part = match.group(2).strip()
            return f"[{text_part}]({url_part})"

        return re.sub(rst_link_pattern, replace_rst_link, text)

    # Apply reStructuredText link conversion
    text = convert_rst_links(text)

    # Find and wrap JSON-like structures with backticks
    def find_and_wrap_json(text):
        """Find JSON objects in text and wrap them with backticks"""
        result = ""
        i = 0
        while i < len(text):
            if text[i] == '{':
                # Found potential JSON start, try to find the matching closing brace
                brace_count = 1
                json_start = i
                i += 1
                has_quotes = False
                has_colon = False

                while i < len(text) and brace_count > 0:
                    if text[i] == '{':
                        brace_count += 1
                    elif text[i] == '}':
                        brace_count -= 1
                    elif text[i] in ['"', "'"]:
                        has_quotes = True
                    elif text[i] == ':':
                        has_colon = True
                    i += 1

                if brace_count == 0 and has_quotes and has_colon:
                    # This looks like a complete JSON object
                    json_content = text[json_start:i]
                    result += f"`{json_content}`"
                else:
                    # Not a complete JSON, add the opening brace and continue
                    result += text[json_start]
                    i = json_start + 1
            else:
                result += text[i]
                i += 1

        return result

    # Apply JSON wrapping
    text = find_and_wrap_json(text)

    # Find and wrap content with >= and <= operators that should be in code format
    # Do this BEFORE angle bracket wrapping to avoid conflicts
    def find_and_wrap_comparison_operators(text):
        """Find comparison operators and wrap them with backticks when appropriate"""
        # Pattern for length constraints like "length >= 1 and <= 100"
        # Match the full expression and wrap it once
        text = re.sub(r'(?<![`\w])(length\s*>=\s*\d+\s*and\s*<=\s*\d+)(?![`\w])', r'`\1`', text)
        text = re.sub(r'(?<![`\w])(>=\s*\d+\s*and\s*<=\s*\d+)(?![`\w])', r'`\1`', text)

        # Extract newly created inline codes to protect them from further processing
        protected = []

        def protect_inline(match):
            protected.append(match.group(0))
            return f"__PROTECTED_{len(protected) - 1}__"

        text = re.sub(r'`[^`\n]+`', protect_inline, text)

        # Now handle standalone comparison operators
        # Pattern for standalone >= or <= with numbers (not inside already wrapped content)
        text = re.sub(r'(?<![`\w])(>=\s*\d+)(?![`\w])', r'`\1`', text)
        text = re.sub(r'(?<![`\w])(<=\s*\d+)(?![`\w])', r'`\1`', text)

        # Restore protected content
        for i, code in enumerate(protected):
            text = text.replace(f"__PROTECTED_{i}__", code)

        return text

    # Apply comparison operator wrapping
    text = find_and_wrap_comparison_operators(text)

    # Extract inline codes created by comparison operator wrapping before angle bracket processing
    text = re.sub(r'`[^`\n]+`', extract_inline_code, text)

    # Find and wrap angle bracket content that should be in code format
    def find_and_wrap_angle_brackets(text):
        """Find angle bracket content and wrap with backticks when appropriate"""

        # Use regex to find all angle bracket patterns and wrap them
        def replace_angle_brackets(match):
            full_match = match.group(0)

            # Skip if it's already a proper HTML anchor tag like <a id="...">
            if re.match(r'^<a\s+id=', full_match):
                return full_match

            # For all other angle bracket content, wrap it in backticks
            return f"`{full_match}`"

        # Pattern to match angle brackets with content inside
        # This will match <anything> where anything doesn't contain < or >
        pattern = r'<[^<>]+>'

        return re.sub(pattern, replace_angle_brackets, text)

    # Apply angle bracket wrapping
    text = find_and_wrap_angle_brackets(text)

    # Now extract ALL newly created inline code sections (including angle brackets and operators)
    text = re.sub(r'`[^`\n]+`', extract_inline_code, text)

    # Escape remaining curly braces that would be interpreted as JavaScript expressions
    # This will only affect braces that are NOT inside backticks now
    text = re.sub(r'(?<!\\)\{', r'\\{', text)
    text = re.sub(r'(?<!\\)\}', r'\\}', text)

    # Escape remaining angle brackets that could be interpreted as JSX elements
    # But be careful not to escape legitimate comparison operators or HTML tags
    text = re.sub(r'(?<![\w\s=!<>])<(?![\w\s=/])', r'&lt;', text)
    text = re.sub(r'(?<![\w\s=!<>])>(?![\w\s=])', r'&gt;', text)

    # Restore inline code
    for i, code in enumerate(inline_codes):
        text = text.replace(f"__INLINE_CODE_{i}__", code)

    # Restore code blocks
    for i, block in enumerate(code_blocks):
        text = text.replace(f"__CODE_BLOCK_{i}__", block)

    return text


def format_code_content(content):
    """Format content as inline code if it contains special characters"""
    if not content:
        return content

    # Check if content looks like JSON/dict/object structure
    # This includes patterns like {"key": "value"} or complex nested structures
    json_like_patterns = [
        r'\{.*:.*\}',  # Basic key-value structure
        r'\{.*\{.*\}.*\}',  # Nested structures
        r'^\{.*\}$',  # Entire string is a JSON-like object
        r'\[.*\{.*\}.*\]',  # Array containing objects
    ]

    for pattern in json_like_patterns:
        if re.search(pattern, content):
            # Remove existing backticks to avoid double-escaping
            content = content.replace('`', '')
            return f"`{content}`"

    # Check if content contains other special characters that should be in code format
    special_chars = ['<', '>', '"', "'"]
    if any(char in content for char in special_chars):
        # Remove existing backticks to avoid double-escaping
        content = content.replace('`', '')
        return f"`{content}`"

    return content


def is_class_substantial(class_node):
    """Check if a class has substantial content worth documenting"""
    # Get class docstring
    class_doc = ast.get_docstring(class_node)

    # Count meaningful methods (excluding __init__ and other special methods)
    meaningful_methods = []

    for node in class_node.body:
        if isinstance(node, ast.FunctionDef):
            if node.name == '__init__':
                # Check if __init__ has substantial documentation
                init_doc = ast.get_docstring(node)
                if (
                    init_doc and len(init_doc.strip()) > 50
                ):  # Substantial docstring
                    meaningful_methods.append(node.name)
            elif not node.name.startswith('_'):  # Public methods
                meaningful_methods.append(node.name)
            elif node.name in [
                '__str__',
                '__repr__',
                '__call__',
            ]:  # Important special methods
                meaningful_methods.append(node.name)

    # A class is substantial if:
    # 1. It has a class docstring, OR
    # 2. It has public methods beyond __init__, OR
    # 3. It has a documented __init__ method, OR
    # 4. It has important special methods
    return (class_doc and len(class_doc.strip()) > 20) or len(
        meaningful_methods
    ) > 0


def generate_ast_docs(module_name, output_dir):
    """Generate documentation by parsing Python source code directly using AST"""
    try:
        # Import the module to get the file path
        module = importlib.import_module(module_name)
        if not hasattr(module, '__file__') or not module.__file__:
            return None

        module_file = module.__file__
        if module_file.endswith('.pyc'):
            module_file = module_file[:-1]  # Remove 'c' to get .py file

        if not os.path.exists(module_file):
            return None

        # Parse the source file
        with open(module_file, 'r', encoding='utf-8') as f:
            source_code = f.read()

        tree = ast.parse(source_code)

        # Extract module-level docstring
        module_doc = ast.get_docstring(tree) or ""

        # Generate markdown content
        markdown_lines = []

        # Add module anchor point
        markdown_lines.append(f'<a id="{module_name}"></a>')
        markdown_lines.append("")

        if module_doc:
            escaped_module_doc = escape_mdx_content(module_doc)
            markdown_lines.append(escaped_module_doc)
            markdown_lines.append("")

        # Process classes and functions
        for node in ast.walk(tree):
            if isinstance(node, ast.ClassDef):
                if is_class_substantial(node):
                    class_md = generate_class_docs(node, module_name)
                    markdown_lines.extend(class_md)
            elif (
                isinstance(node, ast.FunctionDef) and node.col_offset == 0
            ):  # Top-level functions only
                func_md = generate_function_docs(node, module_name)
                markdown_lines.extend(func_md)

        # Write output
        if not is_content_substantial('\n'.join(markdown_lines)):
            return None

        output_file = os.path.join(output_dir, f"{module_name}.mdx")
        with open(output_file, 'w', encoding='utf-8') as f:
            f.write('\n'.join(markdown_lines))

        return output_file

    except Exception as e:
        print(f"Error generating docs for {module_name}: {e}")
        return None


def generate_class_docs(class_node, module_name):
    """Generate documentation for a class"""
    lines = []

    # Class anchor point
    class_name = class_node.name
    lines.append(f'<a id="{module_name}.{class_name}"></a>')
    lines.append("")

    # Class header without backticks
    lines.append(f"## {class_name}")
    lines.append("")

    # Class signature in Python code block
    # Get class signature with base classes
    bases = []
    for base in class_node.bases:
        if isinstance(base, ast.Name):
            bases.append(base.id)
        elif isinstance(base, ast.Attribute):
            bases.append(f"{base.attr}")

    signature = f"class {class_name}"
    if bases:
        signature += f"({', '.join(bases)})"
    signature += ":"

    lines.append("```python")
    lines.append(signature)
    lines.append("```")
    lines.append("")

    # Class docstring
    class_doc = ast.get_docstring(class_node)
    if class_doc:
        doc_info = parse_docstring(class_doc)

        if doc_info['description']:
            escaped_description = escape_mdx_content(doc_info['description'])
            lines.append(escaped_description)
            lines.append("")

        if doc_info['args']:
            lines.append("**Parameters:**")
            lines.append("")
            for arg in doc_info['args']:
                type_str = (
                    f" ({format_code_content(arg['type'])})"
                    if arg['type']
                    else ""
                )
                default_str = (
                    f" (default: {format_code_content(arg['default'])})"
                    if arg['default']
                    else ""
                )
                escaped_description = escape_mdx_content(arg['description'])
                lines.append(
                    f"- **{arg['name']}**{type_str}: {escaped_description}{default_str}"
                )
            lines.append("")

        if doc_info.get('notes'):
            lines.append('**Note:**')
            lines.append("")
            lines.extend(
                [escape_mdx_content(line) for line in doc_info['notes']]
            )
            lines.append("")

    # Process methods
    for node in class_node.body:
        if isinstance(node, ast.FunctionDef):
            method_lines = generate_method_docs(node, class_name, module_name)
            lines.extend(method_lines)

    return lines


def generate_function_docs(func_node, module_name):
    """Generate documentation for a function"""
    lines = []

    # Function anchor point
    function_name = func_node.name
    lines.append(f'<a id="{module_name}.{function_name}"></a>')
    lines.append("")

    # Function header without backticks
    lines.append(f"## {function_name}")
    lines.append("")

    # Function signature in Python code block
    signature = generate_function_signature(func_node, multiline=True)

    lines.append("```python")
    lines.append(f"def {signature}:")
    lines.append("```")
    lines.append("")

    # Function docstring
    func_doc = ast.get_docstring(func_node)
    if func_doc:
        doc_info = parse_docstring(func_doc)

        if doc_info['description']:
            escaped_description = escape_mdx_content(doc_info['description'])
            lines.append(escaped_description)
            lines.append("")

        if doc_info['args']:
            lines.append("**Parameters:**")
            lines.append("")
            for arg in doc_info['args']:
                type_str = (
                    f" ({format_code_content(arg['type'])})"
                    if arg['type']
                    else ""
                )
                default_str = (
                    f" (default: {format_code_content(arg['default'])})"
                    if arg['default']
                    else ""
                )
                escaped_description = escape_mdx_content(arg['description'])
                lines.append(
                    f"- **{arg['name']}**{type_str}: {escaped_description}{default_str}"
                )
            lines.append("")

        if doc_info['returns']:
            lines.append("**Returns:**")
            lines.append("")
            escaped_returns = escape_mdx_content(doc_info['returns'])
            lines.append(f"  {escaped_returns}")
            lines.append("")

        if doc_info['raises']:
            lines.append("**Raises:**")
            lines.append("")
            for exc in doc_info['raises']:
                escaped_description = escape_mdx_content(exc['description'])
                lines.append(
                    f"- **{format_code_content(exc['exception'])}**: {escaped_description}"
                )
            lines.append("")

        if doc_info.get('notes'):
            lines.append('**Note:**')
            lines.append("")
            lines.extend(
                [escape_mdx_content(line) for line in doc_info['notes']]
            )
            lines.append("")

    return lines


def generate_method_docs(method_node, class_name, module_name):
    """Generate documentation for a class method"""
    lines = []

    # Method anchor point
    method_name = method_node.name
    lines.append(f'<a id="{module_name}.{class_name}.{method_name}"></a>')
    lines.append("")

    # Method header without backticks
    lines.append(f"### {method_name}")
    lines.append("")

    # Method signature in Python code block
    signature = generate_function_signature(method_node, multiline=True)

    lines.append("```python")
    lines.append(f"def {signature}:")
    lines.append("```")
    lines.append("")

    # Method docstring
    method_doc = ast.get_docstring(method_node)
    if method_doc:
        doc_info = parse_docstring(method_doc)

        if doc_info['description']:
            escaped_description = escape_mdx_content(doc_info['description'])
            lines.append(escaped_description)
            lines.append("")

        if doc_info['args']:
            lines.append("**Parameters:**")
            lines.append("")
            for arg in doc_info['args']:
                # Skip 'self' parameter
                if arg['name'] == 'self':
                    continue
                type_str = (
                    f" ({format_code_content(arg['type'])})"
                    if arg['type']
                    else ""
                )
                default_str = (
                    f" (default: {format_code_content(arg['default'])})"
                    if arg['default']
                    else ""
                )
                escaped_description = escape_mdx_content(arg['description'])
                lines.append(
                    f"- **{arg['name']}**{type_str}: {escaped_description}{default_str}"
                )
            lines.append("")

        if doc_info['returns']:
            lines.append("**Returns:**")
            lines.append("")
            escaped_returns = escape_mdx_content(doc_info['returns'])
            lines.append(f"  {escaped_returns}")
            lines.append("")

        if doc_info.get('notes'):
            lines.append('**Note:**')
            lines.append("")
            lines.extend(
                [escape_mdx_content(line) for line in doc_info['notes']]
            )
            lines.append("")

    return lines


def generate_function_signature(func_node, multiline=False):
    """Generate function signature from AST node"""
    args = []

    # Regular arguments
    for arg in func_node.args.args:
        arg_str = arg.arg
        if arg.annotation:
            arg_str += f": {ast.unparse(arg.annotation)}"
        args.append(arg_str)

    # Default values
    num_defaults = len(func_node.args.defaults)
    if num_defaults > 0:
        for i, default in enumerate(func_node.args.defaults):
            arg_index = len(func_node.args.args) - num_defaults + i
            args[arg_index] += f" = {ast.unparse(default)}"

    # *args
    if func_node.args.vararg:
        vararg_str = f"*{func_node.args.vararg.arg}"
        if func_node.args.vararg.annotation:
            vararg_str += f": {ast.unparse(func_node.args.vararg.annotation)}"
        args.append(vararg_str)

    # **kwargs
    if func_node.args.kwarg:
        kwarg_str = f"**{func_node.args.kwarg.arg}"
        if func_node.args.kwarg.annotation:
            kwarg_str += f": {ast.unparse(func_node.args.kwarg.annotation)}"
        args.append(kwarg_str)

    # Format signature based on length and multiline preference
    if multiline and (len(args) > 3 or sum(len(arg) for arg in args) > 60):
        # Multi-line format
        if args:
            formatted_args = ',\n    '.join(args)
            return f"{func_node.name}(\n    {formatted_args}\n)"
        else:
            return f"{func_node.name}()"
    else:
        # Single-line format
        return f"{func_node.name}({', '.join(args)})"


def generate_custom_docs(modules, output_dir, package_name="camel"):
    """Generate documentation using custom AST parser"""
    os.makedirs(output_dir, exist_ok=True)

    generated_count = 0
    skipped_count = 0

    for i, module in enumerate(modules):
        print(f"  [{i+1}/{len(modules)}] Processing {module}...")

        output_file = generate_ast_docs(module, output_dir)
        if output_file:
            print(f"    Generated {os.path.basename(output_file)}")
            generated_count += 1
        else:
            print(f"    Skipped {module} (insufficient content)")
            skipped_count += 1

    return generated_count, skipped_count


def discover_module_structure(package_name="camel"):
    """Dynamically discover the module structure from the actual package"""
    try:
        package = importlib.import_module(package_name)
        package_path = os.path.dirname(package.__file__)

        module_structure = {}

        # Walk through the package directory
        for root, dirs, files in os.walk(package_path):
            # Skip __pycache__ and hidden directories
            dirs[:] = [
                d
                for d in dirs
                if not d.startswith('__pycache__') and not d.startswith('.')
            ]

            # Get relative path from package root
            rel_path = os.path.relpath(root, package_path)
            if rel_path == '.':
                current_module = package_name
            else:
                current_module = (
                    f"{package_name}.{rel_path.replace(os.sep, '.')}"
                )

            # Check if this directory has an __init__.py (making it a package)
            if '__init__.py' in files:
                # Get the top-level module name
                parts = current_module.split('.')
                if len(parts) >= 2:  # camel.something
                    top_level = parts[1]
                    if top_level not in module_structure:
                        module_structure[top_level] = {
                            'display_name': get_module_display_name(top_level),
                            'modules': [],
                        }

        # Now get all actual importable modules
        all_modules = get_all_modules(package_name)

        # Organize modules by top-level package
        for module in all_modules:
            parts = module.split('.')
            if len(parts) >= 2:  # camel.something
                top_level = parts[1]
                if top_level in module_structure:
                    module_structure[top_level]['modules'].append(module)

        return module_structure

    except ImportError as e:
        print(f"Error discovering package structure: {e}")
        return {}


def update_module_mappings():
    """Update MODULE_NAME_DISPLAY based on discovered modules"""
    global MODULE_NAME_DISPLAY, MODULE_ORDER

    structure = discover_module_structure()

    # Update display names for discovered modules
    for module_name in structure.keys():
        if module_name not in MODULE_NAME_DISPLAY:
            # Convert snake_case to Title Case
            display_name = module_name.replace('_', ' ').title()
            MODULE_NAME_DISPLAY[module_name] = display_name

    # Update module order to include newly discovered modules
    discovered_modules = list(structure.keys())
    existing_order = [m for m in MODULE_ORDER if m in discovered_modules]
    new_modules = [m for m in discovered_modules if m not in MODULE_ORDER]

    # Keep existing order for known modules, add new ones at the end
    MODULE_ORDER[:] = existing_order + sorted(new_modules)

    return structure


def main():
    parser = argparse.ArgumentParser(
        description="Generate API documentation and update mint.json configuration"
    )
    parser.add_argument(
        "--output_dir",
        type=str,
        default="docs/mintlify/reference",
        help="Output directory for MDX files",
    )
    parser.add_argument(
        "--mint_json",
        type=str,
        default="docs/mintlify/docs.json",
        help="Path to docs.json file",
    )
    parser.add_argument(
        "--package",
        type=str,
        default="camel",
        help="Package name to generate documentation for",
    )
    parser.add_argument(
        "--clean",
        action="store_true",
        help="Clean output directory before generating new files",
    )
    parser.add_argument(
        "--skip_generation",
        action="store_true",
        help="Skip API documentation generation, only update mint.json",
    )
    parser.add_argument(
        "--incremental",
        action="store_true",
        help="Only process modules that have been changed recently",
    )
    parser.add_argument(
        "--since_hours",
        type=int,
        default=24,
        help="Hours to look back for changed files (used with --incremental)",
    )
    args = parser.parse_args()

    if not args.skip_generation:
        # Update module mappings based on discovered structure
        print("Discovering module structure...")
        structure = update_module_mappings()
        print(f"Discovered {len(structure)} top-level modules")

        # Create output directory
        os.makedirs(args.output_dir, exist_ok=True)

        # Clean output directory (if needed)
        if args.clean:
            print(f"Cleaning output directory: {args.output_dir}")
            for file in glob.glob(os.path.join(args.output_dir, "*.mdx")):
                # Preserve the index.mdx file (API Reference landing page)
                if os.path.basename(file) == "index.mdx":
                    print(f"  Preserving {os.path.basename(file)}")
                    continue
                os.remove(file)
                print(f"  Removed {os.path.basename(file)}")

        # Get modules to process
        if args.incremental:
            print(
                f"Looking for modules changed in the last {args.since_hours} hours..."
            )
            modules = get_changed_modules(args.package, args.since_hours)
            if not modules:
                print("No modules have been changed recently.")
                return
            print(f"Found {len(modules)} changed modules")
        else:
            print(f"Discovering all modules in {args.package}...")
            modules = get_all_modules(args.package)

        # Generate documentation
        print(f"Generating documentation for {len(modules)} modules...")
        generated_count, skipped_count = generate_custom_docs(
            modules, args.output_dir, args.package
        )

        print(
            f"\nGenerated: {generated_count} files, Skipped: {skipped_count} files"
        )

    # Build module tree and update mint.json
    print("\nUpdating mint.json configuration...")

    # Get generated MDX files
    mdx_files = list(Path(args.output_dir).glob("*.mdx"))
    if not mdx_files:
        print(f"No MDX files found in {args.output_dir}")
        return

    print(f"Found {len(mdx_files)} MDX files")

    # Build module tree
    module_tree = build_module_tree(mdx_files)

    # Convert to navigation format
    navigation = convert_tree_to_navigation(module_tree)

    # Update mint.json
    if update_docs_json(args.mint_json, navigation):
        print(
            f"Updated {args.mint_json} with {len(navigation)} navigation groups"
        )

    # Print navigation structure summary
    print("\nNavigation structure summary:")
    for item in navigation:
        if isinstance(item, str):
            print(f"- {item}")
        else:
            print(f"- Group: {item['group']} ({len(item['pages'])} pages)")

    print("\nAPI documentation build completed successfully!")
    print(f"Documentation files: {args.output_dir}")
    print(f"Configuration file: {args.mint_json}")

    if not args.skip_generation:
        print("\nDocumentation generated using custom AST parser.")
        print("To preview your Mintlify documentation, run:")
        print("  cd docs/mintlify && npx mintlify dev")


if __name__ == "__main__":
    main()