Skip to content

Apache DataFusion SQL Query Engine

feat: Add robots.txt to site (#24333)

Narendrabbf739a3cfe1
diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
index 77da7db87e..74e4279a6a 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -61,6 +61,7 @@ datafusion/proto-common/src/generated/prost.rs
.github/ISSUE_TEMPLATE/feature_request.yml
.github/workflows/docs.yaml
docs/source/llms.txt
+docs/source/robots.txt
**/node_modules/*
datafusion/wasmtest/pkg/*
clippy.toml
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 81ffce3ad1..68ba628bba 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -112,7 +112,7 @@ html_static_path = ["_static"]
# Copy agent-facing files (llms.txt) verbatim to the site root so they
# resolve at the conventional URL `https://datafusion.apache.org/llms.txt`.
-html_extra_path = ["llms.txt"]
+html_extra_path = ["llms.txt", "robots.txt"]
# Base URL of the published site. Required by sphinx-sitemap to build
# absolute <loc> entries in the generated sitemap.xml.
diff --git a/docs/source/robots.txt b/docs/source/robots.txt
new file mode 100644
index 0000000000..9c80d8219c
--- /dev/null
+++ b/docs/source/robots.txt
@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /
+
+Sitemap: https://datafusion.apache.org/sitemap.xml