Skip to content

🌟 The Multi-Agent Framework: First AI Software Company, Towards Natural Language Programming

13.8 KB
Raw Blame
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time    : 2023/5/11 19:26
@Author  : alexanderwu
@File    : design_api.py
@Modified By: mashenquan, 2023/11/27.
            1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name.
            2. According to the design in Section 2.2.3.5.3 of RFC 135, add incremental iteration functionality.
@Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD.
@Modified By: mashenquan, 2024/5/31. Implement Chapter 3 of RFC 236.
"""
import json
from pathlib import Path
from typing import List, Optional, Union

from pydantic import BaseModel, Field

from metagpt.actions import Action
from metagpt.actions.design_api_an import (
    DATA_STRUCTURES_AND_INTERFACES,
    DESIGN_API_NODE,
    PROGRAM_CALL_FLOW,
    REFINED_DATA_STRUCTURES_AND_INTERFACES,
    REFINED_DESIGN_NODE,
    REFINED_PROGRAM_CALL_FLOW,
)
from metagpt.const import DATA_API_DESIGN_FILE_REPO, SEQ_FLOW_FILE_REPO
from metagpt.logs import logger
from metagpt.schema import AIMessage, Document, Documents, Message
from metagpt.tools.tool_registry import register_tool
from metagpt.utils.common import (
    aread,
    awrite,
    rectify_pathname,
    save_json_to_markdown,
    to_markdown_code_block,
)
from metagpt.utils.mermaid import mermaid_to_file
from metagpt.utils.project_repo import ProjectRepo
from metagpt.utils.report import DocsReporter, GalleryReporter

NEW_REQ_TEMPLATE = """
### Legacy Content
{old_design}

### New Requirements
{context}
"""


@register_tool(include_functions=["run"])
class WriteDesign(Action):
    name: str = ""
    i_context: Optional[str] = None
    desc: str = (
        "Based on the PRD, think about the system design, and design the corresponding APIs, "
        "data structures, library tables, processes, and paths. Please provide your design, feedback "
        "clearly and in detail."
    )
    repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
    input_args: Optional[BaseModel] = Field(default=None, exclude=True)

    async def run(
        self,
        with_messages: List[Message] = None,
        *,
        user_requirement: str = "",
        prd_filename: str = "",
        legacy_design_filename: str = "",
        extra_info: str = "",
        output_pathname: str = "",
        **kwargs,
    ) -> Union[AIMessage, str]:
        """
        Write a system design.

        Args:
            user_requirement (str): The user's requirements for the system design.
            prd_filename (str, optional): The filename of the Product Requirement Document (PRD).
            legacy_design_filename (str, optional): The filename of the legacy design document.
            extra_info (str, optional): Additional information to be included in the system design.
            output_pathname (str, optional): The output file path of the document.

        Returns:
            str: The file path of the generated system design.

        Example:
            # Write a new system design and save to the path name.
            >>> user_requirement = "Write system design for a snake game"
            >>> extra_info = "Your extra information"
            >>> output_pathname = "snake_game/docs/system_design.json"
            >>> action = WriteDesign()
            >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, output_pathname=output_pathname)
            >>> print(result)
            System Design filename: "/absolute/path/to/snake_game/docs/system_design.json"

            # Rewrite an existing system design and save to the path name.
            >>> user_requirement = "Write system design for a snake game, include new features such as a web UI"
            >>> extra_info = "Your extra information"
            >>> legacy_design_filename = "/absolute/path/to/snake_game/docs/system_design.json"
            >>> output_pathname = "/absolute/path/to/snake_game/docs/system_design_new.json"
            >>> action = WriteDesign()
            >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, legacy_design_filename=legacy_design_filename, output_pathname=output_pathname)
            >>> print(result)
            System Design filename: "/absolute/path/to/snake_game/docs/system_design_new.json"

            # Write a new system design with the given PRD(Product Requirement Document) and save to the path name.
            >>> user_requirement = "Write system design for a snake game based on the PRD at /absolute/path/to/snake_game/docs/prd.json"
            >>> extra_info = "Your extra information"
            >>> prd_filename = "/absolute/path/to/snake_game/docs/prd.json"
            >>> output_pathname = "/absolute/path/to/snake_game/docs/sytem_design.json"
            >>> action = WriteDesign()
            >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, prd_filename=prd_filename, output_pathname=output_pathname)
            >>> print(result)
            System Design filename: "/absolute/path/to/snake_game/docs/sytem_design.json"

            # Rewrite an existing system design with the given PRD(Product Requirement Document) and save to the path name.
            >>> user_requirement = "Write system design for a snake game, include new features such as a web UI"
            >>> extra_info = "Your extra information"
            >>> prd_filename = "/absolute/path/to/snake_game/docs/prd.json"
            >>> legacy_design_filename = "/absolute/path/to/snake_game/docs/system_design.json"
            >>> output_pathname = "/absolute/path/to/snake_game/docs/system_design_new.json"
            >>> action = WriteDesign()
            >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, prd_filename=prd_filename, legacy_design_filename=legacy_design_filename, output_pathname=output_pathname)
            >>> print(result)
            System Design filename: "/absolute/path/to/snake_game/docs/system_design_new.json"
        """
        if not with_messages:
            return await self._execute_api(
                user_requirement=user_requirement,
                prd_filename=prd_filename,
                legacy_design_filename=legacy_design_filename,
                extra_info=extra_info,
                output_pathname=output_pathname,
            )

        self.input_args = with_messages[-1].instruct_content
        self.repo = ProjectRepo(self.input_args.project_path)
        changed_prds = self.input_args.changed_prd_filenames
        changed_system_designs = [
            str(self.repo.docs.system_design.workdir / i)
            for i in list(self.repo.docs.system_design.changed_files.keys())
        ]

        # For those PRDs and design documents that have undergone changes, regenerate the design content.
        changed_files = Documents()
        for filename in changed_prds:
            doc = await self._update_system_design(filename=filename)
            changed_files.docs[filename] = doc

        for filename in changed_system_designs:
            if filename in changed_files.docs:
                continue
            doc = await self._update_system_design(filename=filename)
            changed_files.docs[filename] = doc
        if not changed_files.docs:
            logger.info("Nothing has changed.")
        # Wait until all files under `docs/system_designs/` are processed before sending the publish message,
        # leaving room for global optimization in subsequent steps.
        kvs = self.input_args.model_dump()
        kvs["changed_system_design_filenames"] = [
            str(self.repo.docs.system_design.workdir / i)
            for i in list(self.repo.docs.system_design.changed_files.keys())
        ]
        return AIMessage(
            content="Designing is complete. "
            + "\n".join(
                list(self.repo.docs.system_design.changed_files.keys())
                + list(self.repo.resources.data_api_design.changed_files.keys())
                + list(self.repo.resources.seq_flow.changed_files.keys())
            ),
            instruct_content=AIMessage.create_instruct_value(kvs=kvs, class_name="WriteDesignOutput"),
            cause_by=self,
        )

    async def _new_system_design(self, context):
        node = await DESIGN_API_NODE.fill(req=context, llm=self.llm, schema=self.prompt_schema)
        return node

    async def _merge(self, prd_doc, system_design_doc):
        context = NEW_REQ_TEMPLATE.format(old_design=system_design_doc.content, context=prd_doc.content)
        node = await REFINED_DESIGN_NODE.fill(req=context, llm=self.llm, schema=self.prompt_schema)
        system_design_doc.content = node.instruct_content.model_dump_json()
        return system_design_doc

    async def _update_system_design(self, filename) -> Document:
        root_relative_path = Path(filename).relative_to(self.repo.workdir)
        prd = await Document.load(filename=filename, project_path=self.repo.workdir)
        old_system_design_doc = await self.repo.docs.system_design.get(root_relative_path.name)
        async with DocsReporter(enable_llm_stream=True) as reporter:
            await reporter.async_report({"type": "design"}, "meta")
            if not old_system_design_doc:
                system_design = await self._new_system_design(context=prd.content)
                doc = await self.repo.docs.system_design.save(
                    filename=prd.filename,
                    content=system_design.instruct_content.model_dump_json(),
                    dependencies={prd.root_relative_path},
                )
            else:
                doc = await self._merge(prd_doc=prd, system_design_doc=old_system_design_doc)
                await self.repo.docs.system_design.save_doc(doc=doc, dependencies={prd.root_relative_path})
            await self._save_data_api_design(doc)
            await self._save_seq_flow(doc)
            md = await self.repo.resources.system_design.save_pdf(doc=doc)
            await reporter.async_report(self.repo.workdir / md.root_relative_path, "path")
        return doc

    async def _save_data_api_design(self, design_doc, output_filename: Path = None):
        m = json.loads(design_doc.content)
        data_api_design = m.get(DATA_STRUCTURES_AND_INTERFACES.key) or m.get(REFINED_DATA_STRUCTURES_AND_INTERFACES.key)
        if not data_api_design:
            return
        pathname = output_filename or self.repo.workdir / DATA_API_DESIGN_FILE_REPO / Path(
            design_doc.filename
        ).with_suffix("")
        await self._save_mermaid_file(data_api_design, pathname)
        logger.info(f"Save class view to {str(pathname)}")

    async def _save_seq_flow(self, design_doc, output_filename: Path = None):
        m = json.loads(design_doc.content)
        seq_flow = m.get(PROGRAM_CALL_FLOW.key) or m.get(REFINED_PROGRAM_CALL_FLOW.key)
        if not seq_flow:
            return
        pathname = output_filename or self.repo.workdir / Path(SEQ_FLOW_FILE_REPO) / Path(
            design_doc.filename
        ).with_suffix("")
        await self._save_mermaid_file(seq_flow, pathname)
        logger.info(f"Saving sequence flow to {str(pathname)}")

    async def _save_mermaid_file(self, data: str, pathname: Path):
        pathname.parent.mkdir(parents=True, exist_ok=True)
        await mermaid_to_file(self.config.mermaid.engine, data, pathname)
        image_path = pathname.parent / f"{pathname.name}.svg"
        if image_path.exists():
            await GalleryReporter().async_report(image_path, "path")

    async def _execute_api(
        self,
        user_requirement: str = "",
        prd_filename: str = "",
        legacy_design_filename: str = "",
        extra_info: str = "",
        output_pathname: str = "",
    ) -> str:
        prd_content = ""
        if prd_filename:
            prd_filename = rectify_pathname(path=prd_filename, default_filename="prd.json")
            prd_content = await aread(filename=prd_filename)
        context = "### User Requirements\n{user_requirement}\n### Extra_info\n{extra_info}\n### PRD\n{prd}\n".format(
            user_requirement=to_markdown_code_block(user_requirement),
            extra_info=to_markdown_code_block(extra_info),
            prd=to_markdown_code_block(prd_content),
        )
        async with DocsReporter(enable_llm_stream=True) as reporter:
            await reporter.async_report({"type": "design"}, "meta")
            if not legacy_design_filename:
                node = await self._new_system_design(context=context)
                design = Document(content=node.instruct_content.model_dump_json())
            else:
                old_design_content = await aread(filename=legacy_design_filename)
                design = await self._merge(
                    prd_doc=Document(content=context), system_design_doc=Document(content=old_design_content)
                )

            if not output_pathname:
                output_pathname = Path(output_pathname) / "docs" / "system_design.json"
            elif not Path(output_pathname).is_absolute():
                output_pathname = self.config.workspace.path / output_pathname
            output_pathname = rectify_pathname(path=output_pathname, default_filename="system_design.json")
            await awrite(filename=output_pathname, data=design.content)
            output_filename = output_pathname.parent / f"{output_pathname.stem}-class-diagram"
            await self._save_data_api_design(design_doc=design, output_filename=output_filename)
            output_filename = output_pathname.parent / f"{output_pathname.stem}-sequence-diagram"
            await self._save_seq_flow(design_doc=design, output_filename=output_filename)
            md_output_filename = output_pathname.with_suffix(".md")
            await save_json_to_markdown(content=design.content, output_filename=md_output_filename)
            await reporter.async_report(md_output_filename, "path")
        return f'System Design filename: "{str(output_pathname)}". \n The System Design has been completed.'