The agent that turns websites into APIs!
feat: add @docs tag for OpenAPI spec generation from HAR files
kalil0321c7a5068f1520
diff --git a/src/reverse_api/base_engineer.py b/src/reverse_api/base_engineer.pyindex 75317f6..35d4189 100644--- a/src/reverse_api/base_engineer.py+++ b/src/reverse_api/base_engineer.py@@ -7,7 +7,7 @@ from typing import Anyfrom .messages import MessageStorefrom .sync import FileSyncWatcher, get_available_directoryfrom .tui import ClaudeUI-from .utils import generate_folder_name, get_scripts_dir+from .utils import generate_folder_name, get_docs_dir, get_scripts_dirclass BaseEngineer(ABC):@@ -26,13 +26,21 @@ class BaseEngineer(ABC):sdk: str = "claude",is_fresh: bool = False,output_language: str = "python",+ output_mode: str = "client",):self.run_id = run_idself.har_path = har_pathself.prompt = promptself.model = modelself.additional_instructions = additional_instructions- self.scripts_dir = get_scripts_dir(run_id, output_dir)+ self.output_mode = output_mode++ # Select output directory based on mode+ if output_mode == "docs":+ self.scripts_dir = get_docs_dir(run_id, output_dir)+ else:+ self.scripts_dir = get_scripts_dir(run_id, output_dir)+self.ui = ClaudeUI(verbose=verbose)self.usage_metadata: dict[str, Any] = {}self.message_store = MessageStore(run_id, output_dir)@@ -50,10 +58,15 @@ class BaseEngineer(ABC):# Generate local directory namebase_name = generate_folder_name(self.prompt, sdk=self.sdk)- scripts_base_path = Path.cwd() / "scripts"++ # Choose base path based on output mode+ if self.output_mode == "docs":+ base_path = Path.cwd() / "docs"+ else:+ base_path = Path.cwd() / "scripts"# Get available directory (won't overwrite existing non-empty dirs)- local_dir = get_available_directory(scripts_base_path, base_name)+ local_dir = get_available_directory(base_path, base_name)self.local_scripts_dir = local_dir@@ -99,7 +112,9 @@ class BaseEngineer(ABC):}.get(self.output_language, ".py")def _get_client_filename(self) -> str:- """Return the API client filename."""+ """Return the output filename based on mode."""+ if self.output_mode == "docs":+ return "openapi.json"return f"api_client{self._get_output_extension()}"def _get_run_command(self) -> str:@@ -234,16 +249,73 @@ After your analysis, generate the files:1. Save the Python script to: {self.scripts_dir}/{client_filename}2. Save the documentation to: {self.scripts_dir}/README.md"""+ def _get_docs_instructions(self) -> str:+ """Return OpenAPI documentation generation instructions."""+ return f"""4. **Generate an OpenAPI 3.0 specification** that documents these API endpoints:+ - Use OpenAPI 3.0.0 format (https://swagger.io/specification/)+ - Include all discovered endpoints with:+ - Correct HTTP methods (GET, POST, PUT, DELETE, etc.)+ - Path parameters and query parameters+ - Request body schemas (JSON Schema format)+ - Response schemas for common status codes (200, 400, 401, 404, 500)+ - Authentication/security schemes (API keys, Bearer tokens, OAuth, etc.)+ - Organize endpoints into logical tags/groups (e.g., "Users", "Products", "Orders")+ - Infer meaningful descriptions for:+ - Each endpoint's purpose (what it does)+ - Parameters (what they control)+ - Response fields (what they represent)+ - Include examples where patterns are clear from the HAR data+ - Use JSON Schema $ref for shared components/schemas+ - Document authentication requirements in security schemes+ - Add a servers array with the base URL from the HAR file++5. **Enhance documentation with AI inference**:+ - Analyze request/response patterns to infer parameter types and constraints+ - Group related endpoints into logical operations+ - Generate human-readable descriptions (not just field names)+ - Identify required vs optional parameters based on HAR observations+ - Add example values from actual captured requests+ - Document error responses observed in the HAR+ - Note rate limiting headers if present+ - Describe authentication flow if multi-step++6. **Create supplementary documentation**:+ - Generate a README.md file that explains:+ - API overview and purpose+ - Authentication method and how to obtain credentials+ - Base URL and versioning+ - Common use cases with example requests+ - Rate limiting information (if observed)+ - Any special headers or requirements+ - Link to view the OpenAPI spec (e.g., in Swagger UI)++After your analysis, generate the files:++1. Save the OpenAPI spec to: {self.scripts_dir}/openapi.json+2. Save the README to: {self.scripts_dir}/README.md+3. Optionally create: {self.scripts_dir}/examples.md with curl examples++Your OpenAPI spec should be production-ready and suitable for:+- API documentation portals (Swagger UI, Redoc, Stoplight)+- Code generation (OpenAPI Generator, swagger-codegen)+- API testing (Postman, Insomnia)+- Contract testing and validation"""+def _build_analysis_prompt(self) -> str:"""Build the prompt for analyzing the HAR file."""- language_name = {- "python": "Python",- "javascript": "JavaScript",- "typescript": "TypeScript",- }.get(self.output_language, "Python")-- base_prompt = f"""You are tasked with analyzing a HAR (HTTP Archive) file to reverse engineer API calls,- and generate production-ready {language_name} code that replicates those calls.+ if self.output_mode == "docs":+ mode_description = "generate an OpenAPI 3.0 specification documenting"+ task_description = "OpenAPI documentation"+ else:+ language_name = {+ "python": "Python",+ "javascript": "JavaScript",+ "typescript": "TypeScript",+ }.get(self.output_language, "Python")+ mode_description = f"reverse engineer API calls and generate production-ready {language_name} code that replicates"+ task_description = f"{language_name} API client"++ base_prompt = f"""You are tasked with analyzing a HAR (HTTP Archive) file to {mode_description} those calls.Here is the HAR file path you need to analyze:<har_path>@@ -261,7 +333,7 @@ Here is the output directory where you should save your generated files:</output_dir>**IMPORTANT: You have access to the AskUserQuestion tool to ask clarifying questions during your analysis.**-Use this tool when you need to clarify functional requirements, prioritize features, choose between implementation approaches, or gather any other information that would help you generate better code.+Use this tool when you need to clarify functional requirements, prioritize features, choose between implementation approaches, or gather any other information that would help you generate better {task_description}.Your task is to:@@ -289,24 +361,24 @@ Your task is to:- When multiple authentication methods are found, ask which to prioritize- If uncertain about feature priorities, ask the user- When implementation approaches are ambiguous, ask for preferences- - Use the tool for any clarifications that would improve the final code+ - Use the tool for any clarifications that would improve the final output-{self._get_language_instructions()}+{self._get_docs_instructions() if self.output_mode == "docs" else self._get_language_instructions()}-Before generating your code, use a scratchpad to plan your approach:+Before generating your output, use a scratchpad to plan your approach:<scratchpad>In your scratchpad:- Summarize the key API endpoints found in the HAR file- Note the authentication mechanism being used- Identify any patterns or commonalities between requests-- Plan the structure of your {language_name} code-- Consider potential issues (rate limiting, bot detection, etc.)-- Decide whether `requests` will be sufficient or if Playwright is needed+- Plan the structure of your {task_description}+- Consider potential issues (rate limiting, versioning, etc.)+{"- Decide whether `requests` will be sufficient or if Playwright is needed" if self.output_mode != "docs" else ""}- Identify any ambiguities or questions you should ask the user using AskUserQuestion</scratchpad>-If your first attempt doesn't work, analyze what went wrong and try again. Document each attempt and what you learned.+{"" if self.output_mode == "docs" else """If your first attempt doesn't work, analyze what went wrong and try again. Document each attempt and what you learned.<attempt_log>For each attempt (up to 5), document:@@ -316,15 +388,15 @@ For each attempt (up to 5), document:- What you changed for the next attempt</attempt_log>-After testing, provide your final response with:+"""}After {"documenting" if self.output_mode == "docs" else "testing"}, provide your final response with:- A summary of the APIs discovered- The authentication method used-- Whether the implementation works+- {"The completeness and accuracy of the OpenAPI spec" if self.output_mode == "docs" else "Whether the implementation works"}- Any limitations or caveats- The paths to the generated filesYour final output should confirm that the files have been created and provide a brief summary of what was accomplished.-Do not include the full code in your response - just confirm the files were saved and summarize the key findings.+Do not include the full {"spec" if self.output_mode == "docs" else "code"} in your response - just confirm the files were saved and summarize the key findings."""if self.additional_instructions:base_prompt += f"\n\nAdditional instructions:\n{self.additional_instructions}"@@ -334,15 +406,15 @@ Do not include the full code in your response - just confirm the files were saveThis session uses tag-based context loading:-- **@id <run_id>**: Re-engineer mode active+- **@id <run_id>** {"@docs" if self.output_mode == "docs" else ""}: {"Documentation" if self.output_mode == "docs" else "Re-engineer"} mode active- Target run: {self.run_id}- HAR location: {self.har_path.parent}- - Existing scripts: {self.scripts_dir}+ - Existing {"docs" if self.output_mode == "docs" else "scripts"}: {self.scripts_dir}- Message history: {self.message_store.messages_path.parent} (available for reference if needed)- Fresh mode: {str(self.is_fresh).lower()}By default, treat this as an iterative refinement. The user's prompt describes-changes or improvements to make to the existing script. If fresh mode is enabled,+changes or improvements to make to the existing {"documentation" if self.output_mode == "docs" else "script"}. If fresh mode is enabled,ignore previous implementation and start from scratch.Note: Full message history is available at the messages path above if you needdiff --git a/src/reverse_api/cli.py b/src/reverse_api/cli.pyindex e46f2be..6ec56c3 100644--- a/src/reverse_api/cli.py+++ b/src/reverse_api/cli.py@@ -103,7 +103,7 @@ def prompt_interactive_options(elif mode_state["mode"] == "engineer" and text:if text.startswith("@"):# Tag completion- tags = ["@id", "@help"] # @docs, @build coming in next phases+ tags = ["@id", "@docs", "@help"] # @build coming in next phases# specific check for @id completionid_match = re.match(r"@id\s+(.*)", text)@@ -373,6 +373,8 @@ def repl_loop():# Handle empty inputif not raw_input:console.print(" [dim]Usage:[/dim] @id <run_id> [instructions]")+ console.print(" [dim] [/dim] @docs <prompt> (latest run)")+ console.print(" [dim] [/dim] @id <run_id> @docs")console.print(" [dim] [/dim] <run_id> (to switch context)")continue@@ -381,13 +383,22 @@ def repl_loop():target_run_id = parsed["run_id"]is_fresh = parsed["fresh"]+ is_docs = parsed["docs"]user_text = parsed["prompt"]main_prompt = Noneadd_instr = Noneif parsed["is_tag_command"]:- # Explicit @id command+ # Explicit @id or @docs command+ # If @docs without @id, use latest run+ if is_docs and not target_run_id:+ latest_runs = session_manager.get_history(limit=1)+ if not latest_runs:+ console.print(" [red]error:[/red] no runs found in history")+ continue+ target_run_id = latest_runs[0]["run_id"]+if not target_run_id:console.print(" [red]error:[/red] invalid @id syntax")continue@@ -421,6 +432,7 @@ def repl_loop():model=options.get("model"),additional_instructions=add_instr,is_fresh=is_fresh,+ output_mode="docs" if is_docs else "client",)continue@@ -1329,6 +1341,7 @@ def run_engineer(output_dir=None,additional_instructions=None,is_fresh=False,+ output_mode="client",):"""Shared logic for reverse engineering."""if not har_path or not prompt:@@ -1342,7 +1355,7 @@ def run_engineer(console.print(f" [red]not found:[/red] {run_id}")return Noneif not prompt:- prompt = "Reverse engineer captured APIs" # Default+ prompt = "Reverse engineer captured APIs" if output_mode == "client" else "Generate OpenAPI documentation"else:if not prompt:prompt = run_data["prompt"]@@ -1369,6 +1382,7 @@ def run_engineer(additional_instructions=additional_instructions,is_fresh=is_fresh,output_language=output_language,+ output_mode=output_mode,)else:result = run_reverse_engineering(@@ -1382,39 +1396,55 @@ def run_engineer(additional_instructions=additional_instructions,is_fresh=is_fresh,output_language=output_language,+ output_mode=output_mode,)if result:# Skip manual copy if real-time sync is enabled (files already synced)if not enable_sync:- # Automatically copy scripts to current directory with a readable name- scripts_dir = Path(result["script_path"]).parent+ # Automatically copy to current directory with a readable name+ output_dir_path = Path(result["script_path"]).parentbase_name = generate_folder_name(prompt, sdk=sdk)- scripts_base_path = Path.cwd() / "scripts"++ # Choose base path based on output mode+ if output_mode == "docs":+ base_path = Path.cwd() / "docs"+ else:+ base_path = Path.cwd() / "scripts"from .sync import get_available_directory# Get available directory (won't overwrite existing non-empty dirs)- local_dir = get_available_directory(scripts_base_path, base_name)+ local_dir = get_available_directory(base_path, base_name)local_dir.mkdir(parents=True, exist_ok=True)import shutil- for item in scripts_dir.iterdir():+ for item in output_dir_path.iterdir():if item.is_file():shutil.copy2(item, local_dir / item.name)- console.print(" [dim]>[/dim] [white]decoding complete[/white]")- console.print(f" [dim]>[/dim] [white]{result['script_path']}[/white]")- console.print(f" [dim]>[/dim] [white]copied to ./scripts/{local_dir.name}[/white]\n")+ # Different messages for docs vs client mode+ if output_mode == "docs":+ console.print(" [dim]>[/dim] [white]documentation complete[/white]")+ console.print(f" [dim]>[/dim] [white]{result['script_path']}[/white]")+ console.print(f" [dim]>[/dim] [white]copied to ./docs/{local_dir.name}[/white]\n")+ else:+ console.print(" [dim]>[/dim] [white]decoding complete[/white]")+ console.print(f" [dim]>[/dim] [white]{result['script_path']}[/white]")+ console.print(f" [dim]>[/dim] [white]copied to ./scripts/{local_dir.name}[/white]\n")else:# With sync enabled, just show completion- console.print(" [dim]>[/dim] [white]decoding complete[/white]")+ if output_mode == "docs":+ console.print(" [dim]>[/dim] [white]documentation complete[/white]")+ else:+ console.print(" [dim]>[/dim] [white]decoding complete[/white]")console.print(f" [dim]>[/dim] [white]{result['script_path']}[/white]\n")session_manager.update_run(run_id=run_id,sdk=sdk,+ output_mode=output_mode,usage=result.get("usage", {}),paths={"script_path": result.get("script_path")},)diff --git a/src/reverse_api/engineer.py b/src/reverse_api/engineer.pyindex fe86bc3..33fa5a9 100644--- a/src/reverse_api/engineer.py+++ b/src/reverse_api/engineer.py@@ -287,6 +287,7 @@ def run_reverse_engineering(enable_sync: bool = False,is_fresh: bool = False,output_language: str = "python",+ output_mode: str = "client",) -> dict[str, Any] | None:"""Run reverse engineering with the specified SDK.@@ -297,6 +298,7 @@ def run_reverse_engineering(enable_sync: Enable real-time file syncing during engineeringis_fresh: Whether to start fresh (ignore previous scripts)output_language: Target language - "python", "javascript", or "typescript"+ output_mode: Output mode - "client" for API client code, "docs" for OpenAPI specification"""if sdk == "opencode":from .opencode_engineer import OpenCodeEngineer@@ -315,6 +317,7 @@ def run_reverse_engineering(sdk=sdk,is_fresh=is_fresh,output_language=output_language,+ output_mode=output_mode,)else:engineer = ClaudeEngineer(@@ -329,6 +332,7 @@ def run_reverse_engineering(sdk=sdk,is_fresh=is_fresh,output_language=output_language,+ output_mode=output_mode,)# Start sync before analysisdiff --git a/src/reverse_api/session.py b/src/reverse_api/session.pyindex 1945c26..20a3c70 100644--- a/src/reverse_api/session.py+++ b/src/reverse_api/session.py@@ -39,6 +39,7 @@ class SessionManager:"model": kwargs.get("model"),"mode": kwargs.get("mode", "manual"), # Track which mode was used"sdk": kwargs.get("sdk"),+ "output_mode": kwargs.get("output_mode", "client"), # Track output format (client/docs)"usage": kwargs.get("usage", {}),"paths": kwargs.get("paths", {}),}diff --git a/src/reverse_api/utils.py b/src/reverse_api/utils.pyindex 67b0d85..a3520e2 100644--- a/src/reverse_api/utils.py+++ b/src/reverse_api/utils.py@@ -222,6 +222,7 @@ def parse_engineer_prompt(input_text: str) -> dict:dict: {"run_id": str | None,"fresh": bool,+ "docs": bool,"prompt": str,"is_tag_command": bool}@@ -230,24 +231,40 @@ def parse_engineer_prompt(input_text: str) -> dict:return {"run_id": None,"fresh": False,+ "docs": False,"prompt": "","is_tag_command": False,}- # Regex for @id <run_id> [--fresh] <prompt>+ # Check for standalone @docs first+ if input_text.strip().startswith("@docs"):+ docs_match = re.match(r"@docs(?:\s+(.*))?", input_text.strip())+ if docs_match:+ return {+ "run_id": None, # Will use latest run+ "fresh": False,+ "docs": True,+ "prompt": docs_match.group(1) or "",+ "is_tag_command": True,+ }++ # Enhanced regex for @id <run_id> [--fresh] [@docs] <prompt># Group 1: run_id# Group 2: fresh flag (optional)- # Group 3: prompt (optional)- pattern = r"@id\s+([a-zA-Z0-9-_]+)(?:\s+(--fresh))?(?:\s+(.*))?"+ # Group 3: docs flag (optional)+ # Group 4: prompt (optional)+ pattern = r"@id\s+([a-zA-Z0-9-_]+)(?:\s+(--fresh))?(?:\s+(@docs))?(?:\s+(.*))?"match = re.match(pattern, input_text.strip())if match:run_id = match.group(1)fresh = bool(match.group(2))- remaining_prompt = match.group(3) or ""+ docs = bool(match.group(3))+ remaining_prompt = match.group(4) or ""return {"run_id": run_id,"fresh": fresh,+ "docs": docs,"prompt": remaining_prompt,"is_tag_command": True,}@@ -255,6 +272,7 @@ def parse_engineer_prompt(input_text: str) -> dict:return {"run_id": None,"fresh": False,+ "docs": False,"prompt": input_text.strip(),"is_tag_command": False,}@@ -310,6 +328,14 @@ def get_scripts_dir(run_id: str, output_dir: str | None = None) -> Path:return scripts_dir+def get_docs_dir(run_id: str, output_dir: str | None = None) -> Path:+ """Get the docs directory for a specific run."""+ base_dir = get_base_output_dir(output_dir)+ docs_dir = base_dir / "docs" / run_id+ docs_dir.mkdir(parents=True, exist_ok=True)+ return docs_dir++def get_messages_path(run_id: str, output_dir: str | None = None) -> Path:"""Get the messages file path for a specific run."""base_dir = get_base_output_dir(output_dir)