A platform for community discussion. Free, open, simple.
FIX: Treat Perplexity-User and Claude-User as crawlers (#44319)
Natalie Tay0c800337af25
diff --git a/config/site_settings.yml b/config/site_settings.ymlindex f409897db38..cb75f43c039 100644--- a/config/site_settings.yml+++ b/config/site_settings.yml@@ -3157,7 +3157,7 @@ security:list_type: compactcrawler_user_agents:hidden: true- default: "rss|bot|spider|crawler|facebook|archive|wayback|ping|monitor|lighthouse|google-inspectiontool|gptbot|claudebot|anthropic-ai|brightbot|googleother|yeti"+ default: "rss|bot|spider|crawler|facebook|archive|wayback|ping|monitor|lighthouse|google-inspectiontool|gptbot|claudebot|anthropic-ai|brightbot|googleother|yeti|perplexity-user|claude-user"type: listlist_type: compactbrowser_update_user_agents:diff --git a/spec/lib/crawler_detection_spec.rb b/spec/lib/crawler_detection_spec.rbindex 1cc5248c5a9..891363b31d9 100644--- a/spec/lib/crawler_detection_spec.rb+++ b/spec/lib/crawler_detection_spec.rb@@ -42,6 +42,13 @@ RSpec.describe CrawlerDetection docrawler! "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3694.0 Mobile Safari/537.36 Chrome-Lighthouse"end+ it "returns true for AI agents fetching pages for a person" do+ # https://docs.perplexity.ai/guides/bots+ crawler! "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)"+ # https://support.claude.com/en/articles/8896518+ crawler! "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; [email protected])"+ end+it "returns true when VIA header contains 'web.archive.org'" docrawler! "Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot)"crawler! "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36",