The Open Framework for Building Agents
docs: enable search engine indexing (#8)
Andrew Barba05f0b60505fb
diff --git a/apps/docs/app/[lang]/layout.tsx b/apps/docs/app/[lang]/layout.tsxindex bc96dfe0e..1be076122 100644--- a/apps/docs/app/[lang]/layout.tsx+++ b/apps/docs/app/[lang]/layout.tsx@@ -5,20 +5,14 @@ import type { Metadata } from "next";import { GeistdocsProvider } from "@/components/geistdocs/provider";import { config } from "@/lib/geistdocs/config";import { mono, sans } from "@/lib/geistdocs/fonts";+import { getSiteOrigin } from "@/lib/geistdocs/url";import { cn } from "@/lib/utils";-/**- * Site-wide metadata. The `robots` block emits- * `<meta name="robots" content="noindex, nofollow">` into every page- * while Eve is pre-1.0 and the docs are still in flux. Flip both- * fields to `true` (and update `app/robots.ts` + the `X-Robots-Tag`- * header in `next.config.ts`) when we're ready to invite search- * traffic.- */export const metadata: Metadata = {+ metadataBase: new URL(getSiteOrigin()),robots: {- index: false,- follow: false,+ index: true,+ follow: true,},};diff --git a/apps/docs/app/robots.ts b/apps/docs/app/robots.tsindex 23c97029e..5553ea773 100644--- a/apps/docs/app/robots.ts+++ b/apps/docs/app/robots.ts@@ -1,28 +1,13 @@import type { MetadataRoute } from "next";-/**- * Tell crawlers to leave the docs site alone for now.- *- * Eve is pre-1.0 and the docs surface is still evolving; we don't want- * the early site indexed by search engines and then ranking stale- * answers as the framework keeps moving. Once we're ready to invite- * traffic this file flips back to `allow: "/"` and the- * `<meta name="robots">` directive in `app/[lang]/layout.tsx` flips to- * `index: true, follow: true`.- *- * `disallow: "/"` here is paired with:- * - `metadata.robots = { index: false, follow: false }` in the root- * layout, which renders `<meta name="robots" content="noindex,- * nofollow">` into every page.- * - An `X-Robots-Tag: noindex, nofollow` HTTP header set in- * `next.config.ts` so non-HTML responses (rss feeds, llms.txt, etc.)- * are also covered.- */+import { getSiteOrigin } from "@/lib/geistdocs/url";+export default function robots(): MetadataRoute.Robots {return {rules: {userAgent: "*",- disallow: "/",+ allow: "/",},+ sitemap: `${getSiteOrigin()}/sitemap.xml`,};}diff --git a/apps/docs/next.config.ts b/apps/docs/next.config.tsindex 83593baaf..476d0d57a 100644--- a/apps/docs/next.config.ts+++ b/apps/docs/next.config.ts@@ -30,29 +30,6 @@ const config: NextConfig = {],},- /**- * Belt-and-suspenders crawler block: also send the `noindex, nofollow`- * directive on every response as an HTTP header so non-HTML routes- * (rss feeds, llms.txt, OpenGraph images, the sitemap itself) inherit- * the same policy as the `<meta name="robots">` tag in the root- * layout. Drop this block — and the matching `disallow: "/"` in- * `app/robots.ts` and `metadata.robots` in `app/[lang]/layout.tsx` —- * when we're ready for search traffic.- */- async headers() {- return [- {- source: "/:path*",- headers: [- {- key: "X-Robots-Tag",- value: "noindex, nofollow",- },- ],- },- ];- },-async redirects() {return [{