AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
docs(platform): move engineering notes out of the published docs folder (#15163)
Nicholas Tindlef3e05a73e1a4
diff --git a/.github/workflows/scripts/test_docs_layout.py b/.github/workflows/scripts/test_docs_layout.pynew file mode 100644index 0000000000..095d05310d--- /dev/null+++ b/.github/workflows/scripts/test_docs_layout.py@@ -0,0 +1,154 @@+"""docs/platform is part of the public docs site, so it holds only published pages.++GitBook publishes a page once its folder's SUMMARY.md lists it. A file that sits+in a published folder without being listed is one line away from going public,+which is how internal notes end up in the site navigation. Notes that are not+for the site live in docs/engineering instead.+"""++import posixpath+import re+import tempfile+import unittest+from pathlib import Path+from urllib.parse import unquote++DOCS = Path(__file__).resolve().parents[3] / "docs"+PUBLISHED = ("home", "platform", "integrations")+# A Markdown link target that is a .md file, written bare or as+# <a path with spaces>. The target has to end at ".md": "page.md.old" is not one.+SUMMARY_LINK = re.compile(r"\]\((?:<([^>#]+\.md)(?:#[^>]*)?>|([^)#\s]+\.md)(?=[)#\s]))")+HAS_SCHEME = re.compile(r"^[a-z][a-z0-9+.-]*:", re.IGNORECASE)+HTML_COMMENT = re.compile(r"<!--.*?-->", re.DOTALL)+++def listed_pages(folder: Path) -> set:+ """Paths of the local pages a folder's SUMMARY.md lists, relative to it."""+ summary = (folder / "SUMMARY.md").read_text(encoding="utf-8")+ # A commented-out entry is not in the navigation, so it lists nothing.+ summary = HTML_COMMENT.sub("", summary)+ links = (bracketed or bare for bracketed, bare in SUMMARY_LINK.findall(summary))+ return {+ posixpath.normpath(unquote(link))+ for link in links+ if not HAS_SCHEME.match(link)+ }+++def unlisted_pages(folder: Path) -> list:+ """Markdown files in a folder that its SUMMARY.md does not list.++ Files under .gitbook are GitBook's own (reusable content it includes into+ pages), not pages, and are never listed.+ """+ listed = listed_pages(folder)+ return sorted(+ page.relative_to(folder).as_posix()+ for page in folder.rglob("*.md")+ if page != folder / "SUMMARY.md"+ and ".gitbook" not in page.relative_to(folder).parts+ and page.relative_to(folder).as_posix() not in listed+ )+++class DocsLayoutTests(unittest.TestCase):+ def test_every_platform_page_is_in_the_summary(self):+ self.assertEqual(+ unlisted_pages(DOCS / "platform"),+ [],+ "These files are in docs/platform but not in docs/platform/SUMMARY.md. "+ "docs/platform is the public docs site, so move them to "+ "docs/engineering. Only list a file in SUMMARY.md, which publishes "+ "it, if it is a page meant for the site. See docs/AGENTS.md.",+ )++ def test_summaries_list_only_pages_in_their_own_folder(self):+ for name in PUBLISHED:+ folder = DOCS / name+ for link in sorted(listed_pages(folder)):+ page = (folder / link).resolve()+ with self.subTest(summary=f"docs/{name}/SUMMARY.md", link=link):+ self.assertTrue(+ folder.resolve() in page.parents,+ "A SUMMARY.md publishes what it lists, so it may only "+ f"list pages inside docs/{name}.",+ )+ self.assertTrue(page.is_file(), "The listed page does not exist.")+++class SummaryParsingTests(unittest.TestCase):+ def folder(self, summary: str, *files: str) -> Path:+ """A temporary folder with the given SUMMARY.md and empty files."""+ directory = tempfile.TemporaryDirectory()+ self.addCleanup(directory.cleanup)+ folder = Path(directory.name)+ (folder / "SUMMARY.md").write_text(summary, encoding="utf-8")+ for name in files:+ (folder / name).parent.mkdir(parents=True, exist_ok=True)+ (folder / name).write_text("", encoding="utf-8")+ return folder++ def test_reads_bare_bracketed_and_encoded_links(self):+ folder = self.folder(+ "* [A](a.md)\n"+ " * [B](./sub/b.md#section)\n"+ "* [C](<with space.md>)\n"+ "* [D](with%20percent.md)\n"+ '* [E](titled.md "A title")\n'+ "* [F](<bracketed.md#section>)\n"+ )++ self.assertEqual(+ listed_pages(folder),+ {+ "a.md",+ "sub/b.md",+ "with space.md",+ "with percent.md",+ "titled.md",+ "bracketed.md",+ },+ )++ def test_ignores_links_to_other_sites(self):+ folder = self.folder("* [Elsewhere](https://example.com/page.md)\n")++ self.assertEqual(listed_pages(folder), set())++ def test_target_must_end_at_the_md_extension(self):+ """Listing page.md.old must not count as listing page.md."""+ folder = self.folder(+ "* [Old](page.md.old)\n* [Old](<page.md.old>)\n", "page.md"+ )++ self.assertEqual(listed_pages(folder), set())+ self.assertEqual(unlisted_pages(folder), ["page.md"])++ def test_commented_out_entry_lists_nothing(self):+ folder = self.folder(+ "* [A](a.md)\n<!-- * [B](b.md) -->\n<!--\n* [C](c.md)\n-->\n",+ "a.md",+ "b.md",+ "c.md",+ )++ self.assertEqual(listed_pages(folder), {"a.md"})+ self.assertEqual(unlisted_pages(folder), ["b.md", "c.md"])++ def test_gitbook_reusable_content_is_not_a_page(self):+ folder = self.folder(+ "* [A](a.md)\n", "a.md", ".gitbook/includes/snippet.md", "sub/b.md"+ )++ self.assertEqual(unlisted_pages(folder), ["sub/b.md"])++ def test_only_the_folders_own_summary_is_exempt(self):+ folder = self.folder(+ "* [A](a.md)\n", "a.md", "b.md", "sub/SUMMARY.md", "sub/c.md"+ )++ self.assertEqual(unlisted_pages(folder), ["b.md", "sub/SUMMARY.md", "sub/c.md"])+++if __name__ == "__main__":+ unittest.main()diff --git a/AGENTS.md b/AGENTS.mdindex 71677b9305..2f4d50be72 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -11,6 +11,13 @@ This guide provides context for coding agents when updating the **autogpt_platfoSee `docs/platform/getting-started.md` for setup instructions.+## Documentation++- `docs/home/`, `docs/platform/` and `docs/integrations/` are the public docs site (agpt.co/docs). Only pages written for people outside the team go there: people using, self-hosting or contributing to AutoGPT.+- The team's working documents go in `docs/engineering/`, which is not on the docs site: analytics and tracking plans, rollout plans, runbooks, internal references, and design and architecture notes. It is still public, like the rest of the repository, so no secrets or confidential material.++See `docs/AGENTS.md` before adding a document.+## Code style- Format Python code with `poetry run format`.diff --git a/autogpt_platform/backend/AGENTS.md b/autogpt_platform/backend/AGENTS.mdindex 11aee6129e..554f523db2 100644--- a/autogpt_platform/backend/AGENTS.md+++ b/autogpt_platform/backend/AGENTS.md@@ -216,7 +216,7 @@ yield "image_url", result_url## Workspace & Media Files-**Read [Workspace & Media Architecture](../../docs/platform/workspace-media-architecture.md) when:**+**Read [Workspace & Media Architecture](../../docs/engineering/workspace-media-architecture.md) when:**- Working on CoPilot file upload/download features- Building blocks that handle `MediaFileType` inputs/outputs- Modifying `WorkspaceManager` or `store_media_file()`diff --git a/autogpt_platform/backend/backend/util/posthog_client.py b/autogpt_platform/backend/backend/util/posthog_client.pyindex a4d210d4d8..acc59c3713 100644--- a/autogpt_platform/backend/backend/util/posthog_client.py+++ b/autogpt_platform/backend/backend/util/posthog_client.py@@ -3,7 +3,7 @@The client batches events on a background thread, so one instance perprocess is both cheaper and safer than one per module. Emitters send through:func:`capture`, which adds the base properties every event carries-(``environment`` and ``source``, see ``docs/platform/tracking-plan.md``),+(``environment`` and ``source``, see ``docs/engineering/tracking-plan.md``),treats a missing client as "analytics disabled" and never lets trackingraise into the request or execution that produced the event."""diff --git a/autogpt_platform/backend/backend/util/posthog_events.py b/autogpt_platform/backend/backend/util/posthog_events.pyindex 0c6cd6d16f..df43b29c84 100644--- a/autogpt_platform/backend/backend/util/posthog_events.py+++ b/autogpt_platform/backend/backend/util/posthog_events.py@@ -1,6 +1,6 @@"""Every PostHog event name the backend sends, in one place.-``docs/platform/tracking-plan.md`` says what each event means, which side+``docs/engineering/tracking-plan.md`` says what each event means, which sidesends it and which properties it carries. This module is the code half ofthat plan: emitters take the name from here instead of spelling it out.The names follow the product analytics plan ("Every Second Counts"). Thediff --git a/autogpt_platform/backend/backend/util/product_analytics.py b/autogpt_platform/backend/backend/util/product_analytics.pyindex 898ea2774f..5c9da8f88a 100644--- a/autogpt_platform/backend/backend/util/product_analytics.py+++ b/autogpt_platform/backend/backend/util/product_analytics.py@@ -42,7 +42,7 @@ count a copilot-started run through the chat turn that asked for it, so atask is ``agent_run_started`` with ``via`` other than ``copilot``, plusevery ``chat_message_sent``. Event names live in``backend.util.posthog_events``; the full list and the task filter are in-``docs/platform/tracking-plan.md``.+``docs/engineering/tracking-plan.md``.Every emitter is best-effort: tracking can never break the work it describes."""diff --git a/autogpt_platform/frontend/src/services/analytics/posthog-base-properties.ts b/autogpt_platform/frontend/src/services/analytics/posthog-base-properties.tsindex 750f729fb6..80cd386000 100644--- a/autogpt_platform/frontend/src/services/analytics/posthog-base-properties.ts+++ b/autogpt_platform/frontend/src/services/analytics/posthog-base-properties.ts@@ -1,7 +1,7 @@import { environment } from "@/services/environment";// The base properties every PostHog event carries (see-// docs/platform/tracking-plan.md), registered once as super properties so+// docs/engineering/tracking-plan.md), registered once as super properties so// no call site has to repeat them. `source` names the emitter; the backend// sends `platform` or `chat_copilot`. Vercel previews share the production// NEXT_PUBLIC_APP_ENV, so they are told apart here, as indiff --git a/autogpt_platform/frontend/src/services/analytics/posthog-events.ts b/autogpt_platform/frontend/src/services/analytics/posthog-events.tsindex d551cbfb63..6b7283b41e 100644--- a/autogpt_platform/frontend/src/services/analytics/posthog-events.ts+++ b/autogpt_platform/frontend/src/services/analytics/posthog-events.ts@@ -1,6 +1,6 @@// Every PostHog event name the browser sends, in one place. What each one// means, who sends it and which properties it carries is in-// docs/platform/tracking-plan.md; the backend's list is+// docs/engineering/tracking-plan.md; the backend's list is// backend/util/posthog_events.py.//// The names follow the product analytics plan ("Every Second Counts"). Neverdiff --git a/docs/AGENTS.md b/docs/AGENTS.mdindex 67cb1fc4f3..919bc33683 100644--- a/docs/AGENTS.md+++ b/docs/AGENTS.md@@ -1,5 +1,16 @@# Documentation Guidelines+## Where a Document Goes++- **Published pages** live in `docs/home/`, `docs/platform/` and `docs/integrations/`. GitBook publishes them at agpt.co/docs, and a page appears on the site once that folder's `SUMMARY.md` lists it. Put a page there only if it is written for people outside the team: people using, self-hosting or contributing to AutoGPT.+- **Engineering notes** live in `docs/engineering/`: the team's working documents, such as analytics and tracking plans, rollout plans, runbooks, internal references, and design and architecture notes. They are not on the docs site. Don't list one in a `SUMMARY.md`.++`docs/engineering/` is not private. The repository is public and AutoPilot's documentation search indexes the folder, so keep secrets and confidential material out of it, as with any file in the repository.++When unsure, write the document in `docs/engineering/`. Listing a page in a `SUMMARY.md` publishes it, so do that only for a page meant for the site.++Every Markdown page under `docs/platform/` must be listed in `docs/platform/SUMMARY.md`: every `.md` file other than `SUMMARY.md` itself and GitBook's own files under a `.gitbook/` directory. A `SUMMARY.md` may list only pages inside its own folder. `.github/workflows/scripts/test_docs_layout.py` checks those two things on every pull request. It cannot tell whether a listed page is meant for the public; that is for the author and the reviewer to judge.+## Block Documentation Manual SectionsWhen updating manual sections (`<!-- MANUAL: ... -->`) in block documentation files (e.g., `docs/integrations/basic.md`), follow these formats:diff --git a/docs/engineering/README.md b/docs/engineering/README.mdnew file mode 100644index 0000000000..512d8fe96e--- /dev/null+++ b/docs/engineering/README.md@@ -0,0 +1,10 @@+# Engineering notes++Working documents for the team and the agents that work on AutoGPT: analytics and tracking plans, rollout plans, runbooks, internal references, and design and architecture notes.++Nothing in this folder is on the docs site. The site at agpt.co/docs is built from `docs/home/`, `docs/platform/` and `docs/integrations/`, and shows the pages listed in each folder's `SUMMARY.md`. A page written for people outside the team, whether they use, self-host or contribute to AutoGPT, goes there. Everything else goes here.++These notes are still public. The repository is public, and AutoPilot's documentation search indexes every Markdown file under `docs/`, this folder included.++- Don't list a file from this folder in a `SUMMARY.md`.+- To link to a note from a published page, use its GitHub URL on `dev`. The relative path does not exist on the docs site.diff --git a/docs/platform/activation-metrics.md b/docs/engineering/activation-metrics.mdsimilarity index 100%rename from docs/platform/activation-metrics.mdrename to docs/engineering/activation-metrics.mddiff --git a/docs/platform/codex-subscription-preview.md b/docs/engineering/codex-subscription-preview.mdsimilarity index 100%rename from docs/platform/codex-subscription-preview.mdrename to docs/engineering/codex-subscription-preview.mddiff --git a/docs/platform/feature-flags.md b/docs/engineering/feature-flags.mdsimilarity index 100%rename from docs/platform/feature-flags.mdrename to docs/engineering/feature-flags.mddiff --git a/docs/platform/contributing/hosted-mcp-catalog.md b/docs/engineering/hosted-mcp-catalog.mdsimilarity index 89%rename from docs/platform/contributing/hosted-mcp-catalog.mdrename to docs/engineering/hosted-mcp-catalog.mdindex 7d295cee1c..6d86cec947 100644--- a/docs/platform/contributing/hosted-mcp-catalog.md+++ b/docs/engineering/hosted-mcp-catalog.md@@ -1,6 +1,6 @@# Maintaining MCP Integrations-[`mcp_catalog.json`](../../../autogpt_platform/backend/backend/integrations/mcp_catalog.json) defines the official services shown in Integrations and the AutoPilot MCP guide. [`mcp_catalog.py`](../../../autogpt_platform/backend/backend/integrations/mcp_catalog.py) validates the records and exposes them through `get_mcp_catalog()`.+[`mcp_catalog.json`](../../autogpt_platform/backend/backend/integrations/mcp_catalog.json) defines the official services shown in Integrations and the AutoPilot MCP guide. [`mcp_catalog.py`](../../autogpt_platform/backend/backend/integrations/mcp_catalog.py) validates the records and exposes them through `get_mcp_catalog()`.## Add or update a servicediff --git a/docs/platform/contributing/oauth-integration-flow.md b/docs/engineering/oauth-integration-flow.mdsimilarity index 100%rename from docs/platform/contributing/oauth-integration-flow.mdrename to docs/engineering/oauth-integration-flow.mddiff --git a/docs/platform/org-feature-map.md b/docs/engineering/org-feature-map.mdsimilarity index 100%rename from docs/platform/org-feature-map.mdrename to docs/engineering/org-feature-map.mddiff --git a/docs/platform/releases/clay-rock-v1.md b/docs/engineering/releases/clay-rock-v1.mdsimilarity index 100%rename from docs/platform/releases/clay-rock-v1.mdrename to docs/engineering/releases/clay-rock-v1.mddiff --git a/docs/platform/releases/clay-rock-v2.1.md b/docs/engineering/releases/clay-rock-v2.1.mdsimilarity index 100%rename from docs/platform/releases/clay-rock-v2.1.mdrename to docs/engineering/releases/clay-rock-v2.1.mddiff --git a/docs/platform/contributing/tests.md b/docs/engineering/tests.mdsimilarity index 100%rename from docs/platform/contributing/tests.mdrename to docs/engineering/tests.mddiff --git a/docs/platform/tracking-plan.md b/docs/engineering/tracking-plan.mdsimilarity index 100%rename from docs/platform/tracking-plan.mdrename to docs/engineering/tracking-plan.mddiff --git a/docs/platform/workspace-media-architecture.md b/docs/engineering/workspace-media-architecture.mdsimilarity index 100%rename from docs/platform/workspace-media-architecture.mdrename to docs/engineering/workspace-media-architecture.mddiff --git a/docs/platform/SUMMARY.md b/docs/platform/SUMMARY.mdindex 06a201e5af..d6af31c552 100644--- a/docs/platform/SUMMARY.md+++ b/docs/platform/SUMMARY.md@@ -12,8 +12,6 @@* [Scheduling & Triggers](scheduling-and-triggers.md)* [Templates](templates.md)* [Credits & Billing](credits-and-billing.md)-* [Activation Metrics & Experiments](activation-metrics.md)-* [PostHog Tracking Plan](tracking-plan.md)* [Integrations & Credentials](integrations-and-credentials.md)* [Data Flow & Execution](data-flow-and-execution.md)* [Sharing & Exporting Agents](sharing-and-exporting.md)diff --git a/docs/platform/integrations-and-credentials.md b/docs/platform/integrations-and-credentials.mdindex f1b3da15f7..40a4ae80e4 100644--- a/docs/platform/integrations-and-credentials.md+++ b/docs/platform/integrations-and-credentials.md@@ -50,7 +50,7 @@ Open **Settings → Integrations**, find a service with the **MCP** badge, and f- If prompted, select your account's region or enter the official server URL for your tenant or deployment.- Review the requested permissions. **Allow changes**, when offered, adds the service's optional actions.-For catalog updates, see [Maintaining MCP Integrations](contributing/hosted-mcp-catalog.md).+For catalog updates, see [Maintaining MCP Integrations](https://github.com/Significant-Gravitas/AutoGPT/blob/dev/docs/engineering/hosted-mcp-catalog.md).## Managing Credentials