Apache DataFusion SQL Query Engine
feat: Add robots.txt to site (#24333)
Narendrabbf739a3cfe1
diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txtindex 77da7db87e..74e4279a6a 100644--- a/dev/release/rat_exclude_files.txt+++ b/dev/release/rat_exclude_files.txt@@ -61,6 +61,7 @@ datafusion/proto-common/src/generated/prost.rs.github/ISSUE_TEMPLATE/feature_request.yml.github/workflows/docs.yamldocs/source/llms.txt+docs/source/robots.txt**/node_modules/*datafusion/wasmtest/pkg/*clippy.tomldiff --git a/docs/source/conf.py b/docs/source/conf.pyindex 81ffce3ad1..68ba628bba 100644--- a/docs/source/conf.py+++ b/docs/source/conf.py@@ -112,7 +112,7 @@ html_static_path = ["_static"]# Copy agent-facing files (llms.txt) verbatim to the site root so they# resolve at the conventional URL `https://datafusion.apache.org/llms.txt`.-html_extra_path = ["llms.txt"]+html_extra_path = ["llms.txt", "robots.txt"]# Base URL of the published site. Required by sphinx-sitemap to build# absolute <loc> entries in the generated sitemap.xml.diff --git a/docs/source/robots.txt b/docs/source/robots.txtnew file mode 100644index 0000000000..9c80d8219c--- /dev/null+++ b/docs/source/robots.txt@@ -0,0 +1,4 @@+User-agent: *+Allow: /++Sitemap: https://datafusion.apache.org/sitemap.xml