AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(frontend/marketplace): include public expert pages in a maintained sitemap (SECRT-2750) (#14901)
Abhimanyu Yadava4af9376c3c6
diff --git a/autogpt_platform/frontend/src/app/__tests__/robots.test.ts b/autogpt_platform/frontend/src/app/__tests__/robots.test.tsnew file mode 100644index 0000000000..af8e08fb74--- /dev/null+++ b/autogpt_platform/frontend/src/app/__tests__/robots.test.ts@@ -0,0 +1,102 @@+import { ADMIN_PAGES, PROTECTED_PAGES } from "@/lib/auth/helpers";+import { afterEach, describe, expect, test, vi } from "vitest";+import robots from "../robots";++afterEach(() => {+ vi.unstubAllEnvs();+});++function getRules() {+ const { rules } = robots();+ return Array.isArray(rules) ? rules[0] : rules;+}++function toList(value: string | string[] | undefined): string[] {+ if (value === undefined) return [];+ return Array.isArray(value) ? value : [value];+}++// Google's robots.txt matching: the longest matching rule wins, and an allow+// beats a disallow of the same length. Prefixes match the path plus query.+function isCrawlable(url: string): boolean {+ const rules = getRules();+ const longest = (prefixes: string[]) =>+ Math.max(+ -1,+ ...prefixes.filter((p) => url.startsWith(p)).map((p) => p.length),+ );+ return longest(toList(rules.allow)) >= longest(toList(rules.disallow));+}++describe("robots", () => {+ test("allows everything by default", () => {+ const rules = getRules();++ expect(rules.userAgent).toBe("*");+ expect(toList(rules.allow)).toContain("/");+ expect(isCrawlable("/marketplace/experts/expert-1")).toBe(true);+ });++ test("disallows the protected, admin, api and auth prefixes", () => {+ const disallow = getRules().disallow;++ for (const prefix of [+ ...PROTECTED_PAGES,+ ...ADMIN_PAGES,+ "/api",+ "/auth",+ ]) {+ expect(disallow).toContain(prefix);+ }+ });++ test("leaves the marketplace crawlable", () => {+ const disallow = getRules().disallow;+ const list = Array.isArray(disallow) ? disallow : [disallow];++ expect(list.some((p) => p?.startsWith("/marketplace"))).toBe(false);+ });++ test.each([+ "/api/proxy/api/experts/templates",+ "/api/proxy/api/experts/templates?include_archived=false",+ "/api/proxy/api/integrations/providers/system",+ "/api/proxy/api/store/agents?search_query=email&sorted_by=runs",+ "/api/proxy/api/store/agents/creator/agent-slug",+ "/api/proxy/api/store/creators?featured=true",+ "/api/proxy/api/store/creators/creator",+ "/api/proxy/api/store/categories",+ "/api/proxy/api/store/search?search_query=email",+ "/api/proxy/api/store/skills?page=1&page_size=20",+ "/api/proxy/api/store/skills/my-skill",+ "/api/proxy/api/store/skills/my-skill/files/SKILL.md",+ "/api/proxy/api/store/media/user-1/images/thumb.png",+ "/api/store/media/user-1/images/thumb.png",+ ])("lets crawlers render the marketplace through %s", (url) => {+ expect(isCrawlable(url)).toBe(true);+ });++ test.each([+ "/api/auth/user",+ "/api/proxy/api/auth/user",+ "/api/proxy/api/experts",+ "/api/proxy/api/experts/expert-1",+ "/api/proxy/api/experts/expert-1/credentials",+ "/api/proxy/api/library/agents",+ "/api/proxy/api/store/profile",+ "/api/proxy/api/store/submissions",+ "/api/proxy/api/store/my-unpublished-agents",+ "/api/proxy/api/store/skills/submissions",+ "/api/proxy/api/store/skills/submissions/version-1",+ "/api/proxy/api/integrations/credentials",+ "/auth/login",+ ])("keeps %s off limits", (url) => {+ expect(isCrawlable(url)).toBe(false);+ });++ test("points at the sitemap on the site URL", () => {+ vi.stubEnv("NEXT_PUBLIC_FRONTEND_BASE_URL", "https://platform.agpt.co");++ expect(robots().sitemap).toBe("https://platform.agpt.co/sitemap.xml");+ });+});diff --git a/autogpt_platform/frontend/src/app/__tests__/sitemap.test.ts b/autogpt_platform/frontend/src/app/__tests__/sitemap.test.tsnew file mode 100644index 0000000000..8d2c7e17a6--- /dev/null+++ b/autogpt_platform/frontend/src/app/__tests__/sitemap.test.ts@@ -0,0 +1,161 @@+import type { ExpertTemplate } from "@/app/api/__generated__/models/expertTemplate";+import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";+import { listExpertTemplates } from "@/app/api/__generated__/endpoints/experts/experts";+import sitemap, { dynamic } from "../sitemap";++vi.mock("@/app/api/__generated__/endpoints/experts/experts", () => ({+ listExpertTemplates: vi.fn(),+}));++const SITE_URL = "https://platform.agpt.co";++function template(overrides: Partial<ExpertTemplate>): ExpertTemplate {+ return {+ id: "expert-id",+ name: "Expert",+ avatar_url: null,+ role: "analyst",+ tagline: null,+ bio: null,+ skills: [],+ identity: "",+ voice_preferences: "",+ boundaries: "",+ protected_soul_rules: [],+ is_template: true,+ source_template_id: null,+ is_archived: false,+ workflows: [],+ ...overrides,+ };+}++function mockTemplates(templates: ExpertTemplate[]) {+ vi.mocked(listExpertTemplates).mockResolvedValue({+ data: templates,+ status: 200,+ headers: new Headers(),+ });+}++function urlsOf(entries: Awaited<ReturnType<typeof sitemap>>): string[] {+ return entries.map((entry) => entry.url);+}++beforeEach(() => {+ vi.stubEnv("NEXT_PUBLIC_FRONTEND_BASE_URL", SITE_URL);+ vi.spyOn(console, "error").mockImplementation(() => {});+});++afterEach(() => {+ vi.unstubAllEnvs();+ vi.restoreAllMocks();+});++describe("sitemap", () => {+ test("lists the marketplace and one absolute URL per public expert", async () => {+ mockTemplates([+ template({ id: "d91d9897-5c65-45c6-ba16-0dd5c24404ac" }),+ template({ id: "7a25f32e-26e4-4a4e-9902-aed163e61c1d" }),+ ]);++ const urls = urlsOf(await sitemap());++ expect(urls).toEqual([+ `${SITE_URL}/marketplace`,+ `${SITE_URL}/marketplace/experts/d91d9897-5c65-45c6-ba16-0dd5c24404ac`,+ `${SITE_URL}/marketplace/experts/7a25f32e-26e4-4a4e-9902-aed163e61c1d`,+ ]);+ });++ test("builds URLs from getSiteUrl so dev and prod differ", async () => {+ vi.stubEnv("NEXT_PUBLIC_FRONTEND_BASE_URL", "https://dev-builder.agpt.co");+ mockTemplates([template({ id: "abc" })]);++ const urls = urlsOf(await sitemap());++ expect(urls).toEqual([+ "https://dev-builder.agpt.co/marketplace",+ "https://dev-builder.agpt.co/marketplace/experts/abc",+ ]);+ expect(urls.every((url) => new URL(url).protocol === "https:")).toBe(true);+ });++ test("excludes archived templates", async () => {+ mockTemplates([+ template({ id: "live" }),+ template({ id: "gone", is_archived: true }),+ ]);++ const urls = urlsOf(await sitemap());++ expect(urls).toContain(`${SITE_URL}/marketplace/experts/live`);+ expect(urls).not.toContain(`${SITE_URL}/marketplace/experts/gone`);+ });++ test("excludes rows that are not templates", async () => {+ mockTemplates([+ template({ id: "live" }),+ template({ id: "hired", is_template: false }),+ ]);++ const urls = urlsOf(await sitemap());++ expect(urls).toContain(`${SITE_URL}/marketplace/experts/live`);+ expect(urls).not.toContain(`${SITE_URL}/marketplace/experts/hired`);+ });++ test("renders on request and caches the expert list by fetch for an hour", async () => {+ mockTemplates([template({ id: "live" })]);++ await sitemap();++ expect(dynamic).toBe("force-dynamic");+ expect(listExpertTemplates).toHaveBeenCalledWith(undefined, {+ next: { revalidate: 3600 },+ });+ });++ test("still returns the static entries when the fetch throws", async () => {+ vi.mocked(listExpertTemplates).mockRejectedValue(new Error("backend down"));++ const urls = urlsOf(await sitemap());++ expect(urls).toEqual([`${SITE_URL}/marketplace`]);+ expect(console.error).toHaveBeenCalled();+ });++ test("still returns the static entries on a non-200 response", async () => {+ vi.mocked(listExpertTemplates).mockResolvedValue({+ data: { detail: [] },+ status: 422,+ headers: new Headers(),+ });++ const urls = urlsOf(await sitemap());++ expect(urls).toEqual([`${SITE_URL}/marketplace`]);+ expect(console.error).toHaveBeenCalled();+ });++ test("never lists private pages", async () => {+ mockTemplates([template({ id: "live" })]);++ const urls = urlsOf(await sitemap());+ const privatePrefixes = [+ "/copilot",+ "/library",+ "/settings",+ "/team",+ "/build",+ "/home",+ "/admin",+ "/profile",+ ];++ for (const url of urls) {+ const path = new URL(url).pathname;+ expect(privatePrefixes.some((p) => path.startsWith(p))).toBe(false);+ }+ });+});diff --git a/autogpt_platform/frontend/src/app/robots.ts b/autogpt_platform/frontend/src/app/robots.tsnew file mode 100644index 0000000000..fbd9f7d1f4--- /dev/null+++ b/autogpt_platform/frontend/src/app/robots.ts@@ -0,0 +1,45 @@+import { ADMIN_PAGES, PROTECTED_PAGES } from "@/lib/auth/helpers";+import { getSiteUrl } from "@/lib/metadata";+import type { MetadataRoute } from "next";++const PRIVATE_PREFIXES = ["/api", "/auth"];++// Marketplace pages render their bodies from browser fetches through+// /api/proxy (expert profiles, search, skills) and load store media through+// the same proxy or its /api/store/media rewrite. Crawlers apply robots.txt+// to those rendering fetches too, so a blanket /api disallow would leave them+// indexing a loading shell. Only the anonymous, read-only marketplace routes+// are opened; longest-match wins, so these beat the /api disallow.+const PUBLIC_API_PREFIXES = [+ "/api/store/media/",+ "/api/proxy/api/store/media/",+ "/api/proxy/api/store/agents",+ "/api/proxy/api/store/creators",+ "/api/proxy/api/store/categories",+ "/api/proxy/api/store/search",+ "/api/proxy/api/store/skills",+ "/api/proxy/api/experts/templates",+ "/api/proxy/api/integrations/providers/system",+];++// Private routes nested under an allowed prefix need their own, longer+// disallow to keep winning the match.+const PRIVATE_API_PREFIXES = ["/api/proxy/api/store/skills/submissions"];++export default function robots(): MetadataRoute.Robots {+ const siteUrl = getSiteUrl();+ const allow = ["/", ...PUBLIC_API_PREFIXES];+ const disallow = Array.from(+ new Set([+ ...PRIVATE_PREFIXES,+ ...PRIVATE_API_PREFIXES,+ ...PROTECTED_PAGES,+ ...ADMIN_PAGES,+ ]),+ );++ return {+ rules: { userAgent: "*", allow, disallow },+ sitemap: new URL("/sitemap.xml", siteUrl).toString(),+ };+}diff --git a/autogpt_platform/frontend/src/app/sitemap.ts b/autogpt_platform/frontend/src/app/sitemap.tsnew file mode 100644index 0000000000..7bba4199f4--- /dev/null+++ b/autogpt_platform/frontend/src/app/sitemap.ts@@ -0,0 +1,48 @@+import { listExpertTemplates } from "@/app/api/__generated__/endpoints/experts/experts";+import type { ExpertTemplate } from "@/app/api/__generated__/models/expertTemplate";+import { getSiteUrl } from "@/lib/metadata";+import type { MetadataRoute } from "next";++// Rendered on request: the frontend image is built without a reachable+// backend, so a prerendered sitemap would ship without expert pages and+// serve that for its whole first revalidation window. The expert list is+// instead cached by fetch for an hour once it has loaded successfully, so+// experts published or archived later appear or disappear without a deploy.+export const dynamic = "force-dynamic";++const EXPERT_LIST_REVALIDATE_SECONDS = 3600;+const STATIC_PATHS = ["/marketplace"];++export default async function sitemap(): Promise<MetadataRoute.Sitemap> {+ const siteUrl = getSiteUrl();+ const paths = [...STATIC_PATHS, ...(await listPublicExpertPaths())];++ return paths.map((path) => ({ url: new URL(path, siteUrl).toString() }));+}++async function listPublicExpertPaths(): Promise<string[]> {+ try {+ const response = await listExpertTemplates(undefined, {+ next: { revalidate: EXPERT_LIST_REVALIDATE_SECONDS },+ });+ if (response.status !== 200) {+ console.error(+ "sitemap: expert templates request failed",+ response.status,+ );+ return [];+ }+ return response.data+ .filter(isPublicTemplate)+ .map((template) => `/marketplace/experts/${template.id}`);+ } catch (error) {+ // Only 200 responses enter the fetch cache, so an outage is never kept:+ // the crawler still gets the static pages and the next request retries.+ console.error("sitemap: could not list expert templates", error);+ return [];+ }+}++function isPublicTemplate(template: ExpertTemplate): boolean {+ return template.is_template && !template.is_archived;+}