Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

chore: update refs and bot detection

Carl-Gerhard Lindesvärde8a0602cda5a
diff --git a/apps/api/src/bots/bots.ts b/apps/api/src/bots/bots.ts
index 13aaaffe9..1eb50b28e 100644
--- a/apps/api/src/bots/bots.ts
+++ b/apps/api/src/bots/bots.ts
@@ -30,8 +30,8 @@ const bots = [
},
},
{
- includes: '360Spider',
- name: '360Spider',
+ regex: '(360Spider(?:-Image|-Video)?)',
+ name: '$1',
category: 'Search bot',
url: 'https://www.so.com/help/help_3_2.html',
producer: {
@@ -110,8 +110,8 @@ const bots = [
},
},
{
- includes: 'Amazonbot',
- name: 'Amazon Bot',
+ includes: 'Amazonbot-Video',
+ name: 'Amazonbot-Video',
category: 'Crawler',
url: 'https://developer.amazon.com/support/amazonbot',
producer: {
@@ -119,6 +119,36 @@ const bots = [
url: 'https://www.amazon.com/',
},
},
+ {
+ includes: 'Amazonbot',
+ name: 'Amazonbot',
+ category: 'AI Search Crawler',
+ url: 'https://developer.amazon.com/amazonbot',
+ producer: {
+ name: 'Amazon.com, Inc.',
+ url: 'https://www.amazon.com/',
+ },
+ },
+ {
+ includes: 'Amzn-SearchBot',
+ name: 'Amzn-SearchBot',
+ category: 'AI Search Crawler',
+ url: 'https://developer.amazon.com/amazonbot',
+ producer: {
+ name: 'Amazon.com, Inc.',
+ url: 'https://www.amazon.com/',
+ },
+ },
+ {
+ includes: 'Amzn-User',
+ name: 'Amzn-User',
+ category: 'AI Assistant',
+ url: 'https://developer.amazon.com/amazonbot',
+ producer: {
+ name: 'Amazon.com, Inc.',
+ url: 'https://www.amazon.com/',
+ },
+ },
{
includes: 'AmazonAdBot',
name: 'Amazon AdBot',
@@ -129,6 +159,36 @@ const bots = [
url: 'https://www.amazon.com/',
},
},
+ {
+ includes: 'NovaAct',
+ name: 'NovaAct',
+ category: 'AI Agent',
+ url: 'https://nova.amazon.com/act',
+ producer: {
+ name: 'Amazon.com, Inc.',
+ url: 'https://www.amazon.com/',
+ },
+ },
+ {
+ includes: 'AmazonBuyForMe',
+ name: 'AmazonBuyForMe',
+ category: 'AI Agent',
+ url: 'https://buyforme.amazon/',
+ producer: {
+ name: 'Amazon.com, Inc.',
+ url: 'https://www.amazon.com/',
+ },
+ },
+ {
+ includes: 'amazon-kendra',
+ name: 'Amazon Kendra',
+ category: 'Crawler',
+ url: 'https://docs.aws.amazon.com/kendra/latest/dg/stop-web-crawler.html',
+ producer: {
+ name: 'Amazon.com, Inc.',
+ url: 'https://www.amazon.com/',
+ },
+ },
{
regex: 'Amazon[ -]Route ?53[ -]Health[ -]Check[ -]Service',
name: 'Amazon Route53 Health Check',
@@ -158,10 +218,30 @@ const bots = [
url: 'https://www.apache.org/foundation/',
},
},
+ {
+ includes: 'Apache/',
+ name: 'Apache',
+ category: 'Service Agent',
+ url: 'https://www.apache.org/',
+ producer: {
+ name: 'The Apache Software Foundation',
+ url: 'https://www.apache.org/foundation/',
+ },
+ },
+ {
+ includes: 'Applebot-Extended',
+ name: 'Applebot-Extended',
+ category: 'AI Data Scraper',
+ url: 'https://support.apple.com/en-us/119829',
+ producer: {
+ name: 'Apple Inc',
+ url: 'https://www.apple.com/',
+ },
+ },
{
includes: 'Applebot',
name: 'Applebot',
- category: 'Crawler',
+ category: 'AI Search Crawler',
url: 'https://support.apple.com/en-us/119829',
producer: {
name: 'Apple Inc',
@@ -661,7 +741,7 @@ const bots = [
{
includes: 'DuckAssistBot',
name: 'DuckAssistBot',
- category: 'Search bot',
+ category: 'AI Assistant',
url: 'https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/',
producer: {
name: 'DuckDuckGo',
@@ -750,7 +830,7 @@ const bots = [
{
includes: 'meta-externalagent',
name: 'Meta-ExternalAgent',
- category: 'Crawler',
+ category: 'AI Assistant',
url: 'https://developers.facebook.com/docs/sharing/webmasters/web-crawlers',
producer: {
name: 'Meta Platforms, Inc.',
@@ -915,7 +995,7 @@ const bots = [
url: 'https://github.com/OJ/gobuster',
},
{
- includes: 'ichiro/mobile goo',
+ includes: 'ichiro',
name: 'Goo',
category: 'Search bot',
url: 'http://search.goo.ne.jp/option/use/sub4/sub4-1',
@@ -971,7 +1051,7 @@ const bots = [
url: 'https://cloud.google.com/scheduler',
producer: {
name: 'Google Inc.',
- url: 'https://www.google.com',
+ url: 'https://www.google.com/',
},
},
{
@@ -988,10 +1068,10 @@ const bots = [
includes: 'GoogleStackdriverMonitoring',
name: 'Google Stackdriver Monitoring',
category: 'Site Monitor',
- url: 'https://cloud.google.com/monitoring',
+ url: 'https://docs.cloud.google.com/monitoring/uptime-checks/using-uptime-checks',
producer: {
name: 'Google Inc.',
- url: 'https://www.google.com',
+ url: 'https://www.google.com/',
},
},
{
@@ -1007,8 +1087,28 @@ const bots = [
{
includes: 'Google-CloudVertexBot',
name: 'Google-CloudVertexBot',
+ category: 'AI Data Scraper',
+ url: 'https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers#google-cloudvertexbot',
+ producer: {
+ name: 'Google Inc.',
+ url: 'https://www.google.com/',
+ },
+ },
+ {
+ includes: 'Google-Firebase',
+ name: 'Google-Firebase',
+ category: 'Service Agent',
+ url: 'https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests',
+ producer: {
+ name: 'Google Inc.',
+ url: 'https://www.google.com/',
+ },
+ },
+ {
+ includes: 'Google-ilp',
+ name: 'Google-ilp',
category: 'Crawler',
- url: 'https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers#google-cloudvertexbot',
+ url: 'https://www.google.com/',
producer: {
name: 'Google Inc.',
url: 'https://www.google.com/',
@@ -1108,7 +1208,7 @@ const bots = [
regex: 'deepcrawl\\.com',
name: 'Lumar',
category: 'Crawler',
- url: 'https://deepcrawl.com/bot',
+ url: 'https://www.lumar.io/spdr/',
producer: {
name: 'Lumar',
url: 'https://www.lumar.io/',
@@ -1124,9 +1224,29 @@ const bots = [
url: 'https://www.google.com/',
},
},
+ {
+ includes: 'Google-Site-Verification',
+ name: 'Google-Site-Verification',
+ category: 'Validator',
+ url: 'https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers#google_site_verifier',
+ producer: {
+ name: 'Google Inc.',
+ url: 'https://www.google.com/',
+ },
+ },
+ {
+ includes: 'Google-Read-Aloud',
+ name: 'Google-Read-Aloud',
+ category: 'Crawler',
+ url: 'https://developers.google.com/crawling/docs/crawlers-fetchers/read-aloud-user-agent',
+ producer: {
+ name: 'Google Inc.',
+ url: 'https://www.google.com/',
+ },
+ },
{
regex:
- 'Adwords-(?:DisplayAds|Express|Instant)|Google Web Preview|Google[ -]Publisher[ -]Plugin|Google-(?:adstxt|Ads-Conversions|Ads-Qualify|Adwords|AMPHTML|Assess|Extended|HotelAdsVerifier|InspectionTool|Lens|PageRenderer|Read-Aloud|Shopping-Quality|Site-Verification|Sites-Thumbnails|speakr|Stale-Content-Probe|Test|Youtube-Links)|(?:AdsBot|APIs|Feedfetcher|Mediapartners)-Google(?:-Mobile)?|Google(?:AdSenseInfeed|AssociationService|bot|Other|Prober|Producer|Sites)|Google.*/\\+/web/snippet',
+ 'Adwords-(?:DisplayAds|Express|Instant)|Google Web Preview|Google[ -]Publisher[ -]Plugin|Google-(?:adstxt|Ads-Conversions|Ads-Qualify|Adwords|AMPHTML|Assess|BusinessLinkVerification|HotelAdsVerifier|InspectionTool|Lens|PageRenderer|Shopping-Quality|Sites-Thumbnails|speakr|Stale-Content-Probe|Test|Youtube-Links)|(?:AdsBot|APIs|Feedfetcher|Mediapartners)-Google(?:-Mobile)?|Google(?:AdSenseInfeed|AssociationService|bot|Other|Prober|Producer|Sites)|Google.*/\\+/web/snippet',
name: 'Googlebot',
category: 'Search bot',
url: 'https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers',
@@ -1135,6 +1255,86 @@ const bots = [
url: 'https://www.google.com/',
},
},
+ {
+ includes: 'Google-NotebookLM',
+ name: 'Google-NotebookLM',
+ category: 'AI Assistant',
+ url: 'https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers',
+ producer: {
+ name: 'Google Inc.',
+ url: 'https://www.google.com/',
+ },
+ },
+ {
+ includes: 'Google-Extended',
+ name: 'Google-Extended',
+ category: 'AI Data Scraper',
+ url: 'https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers',
+ producer: {
+ name: 'Google Inc.',
+ url: 'https://www.google.com/',
+ },
+ },
+ {
+ includes: 'Gemini-Deep-Research',
+ name: 'Gemini-Deep-Research',
+ category: 'AI Assistant',
+ url: 'https://gemini.google/overview/deep-research/',
+ producer: {
+ name: 'Google Inc.',
+ url: 'https://www.google.com/',
+ },
+ },
+ {
+ includes: 'GoogleAgent-Mariner',
+ name: 'GoogleAgent-Mariner',
+ category: 'AI Agent',
+ url: 'https://developers.google.com/search/docs/crawling-indexing/google-agent-mariner',
+ producer: {
+ name: 'Google Inc.',
+ url: 'https://www.google.com/',
+ },
+ },
+ {
+ includes: 'GoogleAgent-Search',
+ name: 'GoogleAgent-Search',
+ category: 'AI Assistant',
+ url: 'https://developers.google.com/search/docs/crawling-indexing/google-agent-search',
+ producer: {
+ name: 'Google Inc.',
+ url: 'https://www.google.com/',
+ },
+ },
+ {
+ includes: 'Bard-AI',
+ name: 'Bard-AI',
+ category: 'AI Search Crawler',
+ url: 'https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers',
+ producer: {
+ name: 'Google Inc.',
+ url: 'https://www.google.com/',
+ },
+ },
+ {
+ includes: 'Gemini-AI',
+ name: 'Gemini-AI',
+ category: 'AI Search Crawler',
+ url: 'https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers',
+ producer: {
+ name: 'Google Inc.',
+ url: 'https://www.google.com/',
+ },
+ },
+ {
+ includes: 'Google-Pinpoint',
+ name: 'Google-Pinpoint',
+ category: 'AI Assistant',
+ url: 'https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers#google-pinpoint',
+ producer: {
+ name: 'Google Inc.',
+ url: 'https://www.google.com/',
+ },
+ },
{
regex: '^Google$',
name: 'Googlebot',
@@ -1579,14 +1779,34 @@ const bots = [
url: 'https://onlineornot.com',
},
},
+ {
+ includes: 'webzio-extended',
+ name: 'webzio-extended',
+ category: 'AI Data Scraper',
+ url: 'https://docs.webz.io/reference/how-it-works-video',
+ producer: {
+ name: 'Webz.io Ltd',
+ url: 'https://webz.io/',
+ },
+ },
+ {
+ includes: 'webzio',
+ name: 'webzio',
+ category: 'AI Search Crawler',
+ url: 'https://docs.webz.io/reference/how-it-works-video',
+ producer: {
+ name: 'Webz.io Ltd',
+ url: 'https://webz.io/',
+ },
+ },
{
includes: 'omgili',
- name: 'Omgili bot',
+ name: 'Omgilibot',
category: 'Search bot',
- url: 'http://www.omgili.com/Crawler.html',
+ url: 'https://webz.io/blog/api/what-is-the-omgili-bot-and-why-is-it-crawling-your-website/',
producer: {
- name: 'Omgili',
- url: 'http://www.omgili.com',
+ name: 'Webz.io Ltd',
+ url: 'https://webz.io/',
},
},
{
@@ -2192,13 +2412,13 @@ const bots = [
},
},
{
- includes: 'TurnitinBot',
+ regex: '^Turnitin(?:Bot)?',
name: 'TurnitinBot',
category: 'Crawler',
- url: 'http://www.turnitin.com/robot/crawlerinfo.html',
+ url: 'https://www.turnitin.com/robot/crawlerinfo.html',
producer: {
name: 'iParadigms, LLC.',
- url: 'http://www.turnitin.com',
+ url: 'https://www.turnitin.com/',
},
},
{
@@ -2225,6 +2445,7 @@ const bots = [
includes: 'Twingly Recon',
name: 'Twingly Recon',
category: 'Crawler',
+ url: 'https://app.twingly.com/public-docs/crawler',
producer: {
name: 'Twingly',
url: 'https://www.twingly.com',
@@ -2566,8 +2787,8 @@ const bots = [
},
},
{
- includes: 'Y!J-BRW',
- name: 'Yahoo! Japan BRW',
+ regex: 'Y!J-(ASR|BR[IJOWY]|[BPW]SC)',
+ name: 'Yahoo! Japan $1',
category: 'Crawler',
url: 'https://support.yahoo-net.jp/PccSearch/s/article/H000007955',
producer: {
@@ -2576,8 +2797,8 @@ const bots = [
},
},
{
- includes: 'Y!J-WSC',
- name: 'Yahoo! Japan WSC',
+ includes: 'Y!J',
+ name: 'Yahoo! Japan',
category: 'Crawler',
url: 'https://support.yahoo-net.jp/PccSearch/s/article/H000007955',
producer: {
@@ -2586,30 +2807,30 @@ const bots = [
},
},
{
- includes: 'Y!J-ASR',
- name: 'Yahoo! Japan ASR',
+ regex: 'YandexBot.+MirrorDetector',
+ name: 'YandexBot-MirrorDetector',
category: 'Crawler',
- url: 'https://support.yahoo-net.jp/PccSearch/s/article/H000007955',
+ url: 'https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html',
producer: {
- name: 'Yahoo! Japan Corp.',
- url: 'https://www.yahoo.co.jp/',
+ name: 'Yandex LLC',
+ url: 'https://yandex.com/company/',
},
},
{
- regex: '^Y!J',
- name: 'Yahoo! Japan',
- category: 'Crawler',
- url: 'https://support.yahoo-net.jp/PccSearch/s/article/H000007955',
+ regex:
+ '(Yandex(?:(?:\\.Gazeta |Accessibility|Com|Mobile|MobileScreenShot|RenderResources|Screenshot|Sprav)?Bot|(?:AdNet|Antivirus|Blogs|Calendar|Catalog|Dialogs|Direct(?:Dyn)?|Favicons|ForDomain|ImageResizer|Images|Market|Media(?:naBot)?|Metrika|News(?:links)?|OntoDB(?:API)?|Pagechecker|Partner|RCA|SearchShop|(?:News|Site)links|Tracker|Turbo|Userproxy|Verticals|Vertis|Video(?:Parser)?|Webmaster))|YaDirectFetcher)',
+ name: '$1',
+ category: 'Search bot',
+ url: 'https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html',
producer: {
- name: 'Yahoo! Japan Corp.',
- url: 'https://www.yahoo.co.jp/',
+ name: 'Yandex LLC',
+ url: 'https://yandex.com/company/',
},
},
{
- regex:
- 'Yandex(?:(?:\\.Gazeta |Accessibility|Additional|Com|Mobile|MobileScreenShot|RenderResources|Screenshot|Sprav)?Bot|(?:Additional|AdNet|Antivirus|Blogs|Calendar|Catalog|Dialogs|Direct|Favicons|ForDomain|ImageResizer|Images|Market|Media|Metrika|News|OntoDB(?:API)?|Pagechecker|Partner|RCA|SearchShop|(?:News|Site)links|Tracker|Turbo|Userproxy|Verticals|Vertis|Video|Webmaster))|YaDirectFetcher',
- name: 'Yandex Bot',
- category: 'Search bot',
+ includes: 'YandexAdditional',
+ name: 'YandexAdditionalBot',
+ category: 'AI Search Crawler',
url: 'https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html',
producer: {
name: 'Yandex LLC',
@@ -2842,15 +3063,11 @@ const bots = [
name: 'Inktomi Slurp',
},
{
- includes: 'Speedy Spider',
+ regex: 'Speedy ?Spider',
name: 'Speedy',
},
{
- includes: 'ScoutJet',
- name: 'ScoutJet',
- },
- {
- regex: 'nrsbot|netresearch',
+ regex: 'nrsbot|NetResearchServer',
name: 'NetResearchServer',
},
{
@@ -2864,14 +3081,22 @@ const bots = [
{
includes: 'charlotte',
name: 'Charlotte',
+ category: 'Crawler',
+ url: 'https://web.archive.org/web/20070225121634/http://www.searchme.com/support/pages/spider.php',
+ producer: {
+ name: 'Searchme, Inc.',
+ url: 'https://web.archive.org/web/20070217233037/http://www.searchme.com/',
+ },
},
{
includes: 'Pompos',
name: 'Pompos',
- },
- {
- includes: 'ichiro',
- name: 'ichiro',
+ category: 'Crawler',
+ url: 'https://web.archive.org/web/20090124084536/http://dir.com/pompos.html',
+ producer: {
+ name: 'ILIAD SA',
+ url: 'https://www.iliad.fr/',
+ },
},
{
includes: 'PagePeeker',
@@ -2890,6 +3115,12 @@ const bots = [
{
includes: 'Willow Internet Crawler',
name: 'Willow Internet Crawler',
+ category: 'Crawler',
+ url: 'https://twotrees.com/company/indexing-web-crawlers/',
+ producer: {
+ name: 'Twotrees Technologies, LLC',
+ url: 'https://twotrees.com/',
+ },
},
{
includes: 'EmailWolf',
@@ -2927,11 +3158,6 @@ const bots = [
url: 'https://www.spotify.com',
},
},
- {
- includes: 'The Knowledge AI',
- name: 'The Knowledge AI',
- category: 'Crawler',
- },
{
includes: 'Embedly',
name: 'Embedly',
@@ -3072,6 +3298,16 @@ const bots = [
url: 'https://bytedance.com/',
},
},
+ {
+ includes: 'TikTokSpider',
+ name: 'TikTokSpider',
+ category: 'Search bot',
+ url: 'https://www.tiktok.com/',
+ producer: {
+ name: 'TikTok Inc.',
+ url: 'https://www.tiktok.com/',
+ },
+ },
{
includes: 'WikiDo',
name: 'WikiDo',
@@ -3200,11 +3436,6 @@ const bots = [
category: 'Service bot',
url: 'https://www.grammarly.com',
},
- {
- includes: 'Robozilla',
- name: 'Robozilla',
- category: 'Crawler',
- },
{
includes: 'Domains Project',
name: 'Domains Project',
@@ -3469,13 +3700,13 @@ const bots = [
},
},
{
- regex: 'scaninfo@(?:expanseinc|paloaltonetworks)\\.com',
- name: 'Expanse',
+ regex: '(?:expanseinc|paloaltonetworks)\\.com',
+ name: 'Cortex Xpanse',
category: 'Security Checker',
- url: 'https://expanse.co/',
+ url: 'https://docs-cortex.paloaltonetworks.com/r/1/Cortex-Xpanse/Scanning-activity',
producer: {
- name: 'Expanse Inc.',
- url: 'https://expanse.co/',
+ name: 'Palo Alto Networks, Inc.',
+ url: 'https://www.paloaltonetworks.com/',
},
},
{
@@ -3796,28 +4027,12 @@ const bots = [
category: 'Crawler',
url: 'https://github.com/VIPnytt/SitemapParser/',
},
- {
- regex: '^Turnitin',
- name: 'Turnitin',
- category: 'Crawler',
- url: 'https://turnitin.com/robot/crawlerinfo.html',
- },
{
regex: 'DMBrowser|DMBrowser-[UB]V',
name: 'Dotcom Monitor',
category: 'Site Monitor',
url: 'https://www.dotcom-monitor.com',
},
- {
- includes: 'ThinkChaos/',
- name: 'ThinkChaos',
- category: 'Crawler',
- },
- {
- includes: 'Thinkbot/',
- name: 'Thinkbot',
- category: 'Crawler',
- },
{
includes: 'DataForSeoBot',
name: 'DataForSeoBot',
@@ -4096,16 +4311,6 @@ const bots = [
category: 'Crawler',
url: 'https://domaincrawler.com/about-us/',
},
- {
- includes: 'DNSResearchBot',
- name: 'DNSResearchBot',
- category: 'Crawler',
- },
- {
- includes: 'GitCrawlerBot',
- name: 'GitCrawlerBot',
- category: 'Crawler',
- },
{
includes: 'AdAuth',
name: 'AdAuth',
@@ -4482,10 +4687,20 @@ const bots = [
url: 'https://www.elsevier.com/',
},
},
+ {
+ includes: 'Doximity-Diffbot',
+ name: 'Doximity-Diffbot',
+ category: 'AI Data Scraper',
+ url: 'https://docs.diffbot.com/docs/getting-started-with-crawl',
+ producer: {
+ name: 'Diffbot Technologies Corp.',
+ url: 'https://www.diffbot.com/',
+ },
+ },
{
regex: '.+diffbot',
name: 'Diffbot',
- category: 'Crawler',
+ category: 'AI Data Scraper',
url: 'https://docs.diffbot.com/docs/getting-started-with-crawl',
producer: {
name: 'Diffbot Technologies Corp.',
@@ -4617,7 +4832,7 @@ const bots = [
{
includes: 'OAI-SearchBot',
name: 'OAI-SearchBot',
- category: 'Crawler',
+ category: 'AI Search Crawler',
url: 'https://platform.openai.com/docs/bots',
producer: {
name: 'OpenAI OpCo, LLC',
@@ -4627,7 +4842,7 @@ const bots = [
{
includes: 'GPTBot',
name: 'GPTBot',
- category: 'Crawler',
+ category: 'AI Data Scraper',
url: 'https://platform.openai.com/docs/bots',
producer: {
name: 'OpenAI OpCo, LLC',
@@ -4637,7 +4852,17 @@ const bots = [
{
includes: 'ChatGPT-User',
name: 'ChatGPT-User',
- category: 'Crawler',
+ category: 'AI Assistant',
+ url: 'https://platform.openai.com/docs/bots',
+ producer: {
+ name: 'OpenAI OpCo, LLC',
+ url: 'https://openai.com/',
+ },
+ },
+ {
+ includes: 'ChatGPT-Browser',
+ name: 'ChatGPT-Browser',
+ category: 'AI Assistant',
url: 'https://platform.openai.com/docs/bots',
producer: {
name: 'OpenAI OpCo, LLC',
@@ -4792,15 +5017,49 @@ const bots = [
},
},
{
- includes: 'ClaudeBot',
+ regex: 'ClaudeBot|Anthropic-Claude',
name: 'ClaudeBot',
- category: 'Crawler',
- url: 'https://github.com/ClaudeBot/ClaudeBot',
+ category: 'AI Data Scraper',
+ url: 'https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler',
+ producer: {
+ name: 'Anthropic PBC',
+ url: 'https://www.anthropic.com/',
+ },
+ },
+ {
+ includes: 'Claude-SearchBot',
+ name: 'Claude-SearchBot',
+ category: 'AI Search Crawler',
+ url: 'https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler',
+ producer: {
+ name: 'Anthropic PBC',
+ url: 'https://www.anthropic.com/',
+ },
+ },
+ {
+ includes: 'Claude-User',
+ name: 'Claude-User',
+ category: 'AI Assistant',
+ url: 'https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler',
+ producer: {
+ name: 'Anthropic PBC',
+ url: 'https://www.anthropic.com/',
+ },
+ },
+ {
+ includes: 'Claude-Web',
+ name: 'Claude-Web',
+ category: 'AI Search Crawler',
+ url: 'https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler',
+ producer: {
+ name: 'Anthropic PBC',
+ url: 'https://www.anthropic.com/',
+ },
},
{
includes: 'Imagesift',
name: 'ImageSift',
- category: 'Crawler',
+ category: 'AI Data Scraper',
url: 'https://imagesift.com/',
producer: {
name: 'Castle Global, Inc.',
@@ -4943,7 +5202,7 @@ const bots = [
{
includes: 'YouBot',
name: 'YouBot',
- category: 'Crawler',
+ category: 'AI Search Crawler',
url: 'https://about.you.com/youbot/',
producer: {
name: 'SuSea, Inc.',
@@ -4979,8 +5238,8 @@ const bots = [
{
includes: 'GitHubCopilotChat',
name: 'GitHubCopilotChat',
- category: 'Crawler',
- url: 'https://github.com/aaamoon/copilot-gpt4-service',
+ category: 'AI Assistant',
+ url: 'https://github.com/microsoft/vscode-copilot-chat',
},
{
regex: '^pdrl\\.fm',
@@ -4997,7 +5256,7 @@ const bots = [
{
includes: 'anthropic-ai',
name: 'Anthropic AI',
- category: 'Crawler',
+ category: 'AI Data Scraper',
url: 'https://www.anthropic.com/',
producer: {
name: 'Anthropic, PBC',
@@ -5047,7 +5306,7 @@ const bots = [
{
includes: 'cohere-ai',
name: 'Cohere AI',
- category: 'Crawler',
+ category: 'AI Data Scraper',
url: 'https://cohere.com/',
producer: {
name: 'Cohere, Inc.',
@@ -5055,20 +5314,40 @@ const bots = [
},
},
{
- includes: 'PerplexityBot',
- name: 'PerplexityBot',
- category: 'Crawler',
- url: 'https://docs.perplexity.ai/guides/bots',
+ includes: 'Cohere-Command',
+ name: 'Cohere-Command',
+ category: 'AI Data Scraper',
+ url: 'https://cohere.com/',
producer: {
- name: 'Perplexity AI, Inc.',
- url: 'https://www.perplexity.ai/',
+ name: 'Cohere, Inc.',
+ url: 'https://cohere.com/',
},
},
{
- includes: 'Perplexity-User',
- name: 'Perplexity-User',
- category: 'Crawler',
- url: 'https://docs.perplexity.ai/guides/bots',
+ includes: 'cohere-training',
+ name: 'cohere-training-data-crawler',
+ category: 'AI Data Scraper',
+ url: 'https://cohere.com/',
+ producer: {
+ name: 'Cohere, Inc.',
+ url: 'https://cohere.com/',
+ },
+ },
+ {
+ includes: 'PerplexityBot',
+ name: 'PerplexityBot',
+ category: 'AI Search Crawler',
+ url: 'https://docs.perplexity.ai/guides/bots',
+ producer: {
+ name: 'Perplexity AI, Inc.',
+ url: 'https://www.perplexity.ai/',
+ },
+ },
+ {
+ includes: 'Perplexity-User',
+ name: 'Perplexity-User',
+ category: 'AI Assistant',
+ url: 'https://docs.perplexity.ai/guides/bots',
producer: {
name: 'Perplexity AI, Inc.',
url: 'https://www.perplexity.ai/',
@@ -5527,14 +5806,19 @@ const bots = [
category: 'Site Monitor',
},
{
- includes: 'WebMon',
- name: 'WebMon',
- category: 'Site Monitor',
+ includes: 'WebMoney Advisor',
+ name: 'WebMoney Advisor',
+ category: 'Service Agent',
+ url: 'https://advisor.web.money/en/',
+ producer: {
+ name: 'DEED Baltic, UAB',
+ url: 'http://www.deed.lt/',
+ },
},
{
- includes: 'AdsTxtCrawlerTP',
- name: 'AdsTxtCrawlerTP',
- category: 'Crawler',
+ includes: 'WebMon/',
+ name: 'WebMon',
+ category: 'Site Monitor',
},
{
includes: 'fragFINN',
@@ -6329,7 +6613,7 @@ const bots = [
{
includes: 'IbouBot',
name: 'IbouBot',
- category: 'Search bot',
+ category: 'AI Search Crawler',
url: 'https://ibou.io/iboubot.html',
},
{
@@ -6345,21 +6629,688 @@ const bots = [
{
includes: 'TerraCotta',
name: 'TerraCotta',
- category: 'Crawler',
+ category: 'AI Data Scraper',
url: 'https://github.com/CeramicTeam/CeramicTerracotta',
producer: {
name: 'Ceramic, Inc.',
url: 'https://ceramic.ai/',
},
},
+ {
+ includes: 'KeybaseBot',
+ name: 'KeybaseBot',
+ category: 'Social Media Agent',
+ url: 'http://keybase.io',
+ producer: {
+ name: 'Zoom Video Communications, Inc',
+ url: 'https://www.zoom.com/',
+ },
+ },
+ {
+ includes: 'Replicate-Bot',
+ name: 'Replicate-Bot',
+ category: 'AI Data Scraper',
+ url: 'https://replicate.com/',
+ producer: {
+ name: 'Replicate, Inc.',
+ url: 'https://replicate.com/',
+ },
+ },
+ {
+ regex: 'cypex\\.ai',
+ name: 'Cypex',
+ category: 'Security Checker',
+ url: 'https://cypex.ai/scanning/',
+ producer: {
+ name: 'Cypex',
+ url: 'https://cypex.ai/',
+ },
+ },
+ {
+ includes: 'fhms-its-research-scanner',
+ name: 'FHMS ITS Research Scanner',
+ category: 'Security Checker',
+ url: 'https://fb02itsscan02.fh-muenster.de/',
+ producer: {
+ name: 'University of Applied Sciences Muenster',
+ url: 'https://www.fh-muenster.de/',
+ },
+ },
+ {
+ includes: 'Together-Bot',
+ name: 'Together-Bot',
+ category: 'AI Data Scraper',
+ url: 'https://www.together.ai/',
+ producer: {
+ name: 'Together Computer Inc.',
+ url: 'https://www.together.ai/',
+ },
+ },
+ {
+ includes: 'xAI-Bot',
+ name: 'xAI-Bot',
+ category: 'AI Data Scraper',
+ url: 'https://x.ai/',
+ producer: {
+ name: 'X.AI LLC',
+ url: 'https://x.ai/',
+ },
+ },
+ {
+ includes: 'Groq-Bot',
+ name: 'Groq-Bot',
+ category: 'AI Data Scraper',
+ url: 'https://groq.com/',
+ producer: {
+ name: 'Groq, Inc.',
+ url: 'https://groq.com/',
+ },
+ },
+ {
+ regex: 'bigsur\\.ai',
+ name: 'Big Sur AI',
+ category: 'AI Assistant',
+ url: 'https://bigsur.ai/',
+ producer: {
+ name: 'Big Sur AI, Inc.',
+ url: 'https://bigsur.ai/',
+ },
+ },
+ {
+ includes: 'FirecrawlAgent',
+ name: 'FirecrawlAgent',
+ category: 'AI Data Scraper',
+ url: 'https://www.firecrawl.dev/',
+ producer: {
+ name: 'SideGuide Technologies, Inc.',
+ url: 'https://www.sideguide.dev/',
+ },
+ },
+ {
+ includes: 'SecurityHeaders',
+ name: 'SecurityHeaders',
+ category: 'Security Checker',
+ url: 'https://securityheaders.com/',
+ producer: {
+ name: 'Snyk Limited',
+ url: 'https://snyk.io/',
+ },
+ },
+ {
+ includes: 'HaloBot',
+ name: 'HaloBot',
+ category: 'Crawler',
+ url: 'https://www.haloscan.com/',
+ producer: {
+ name: 'ARCHI301',
+ url: 'https://www.haloscan.com/',
+ },
+ },
+ {
+ includes: 'RSiteAuditor',
+ name: 'RSiteAuditor',
+ category: 'Crawler',
+ url: 'https://help.agencyanalytics.com/en/articles/3469586-allow-our-site-auditor-via-robots-txt-and-firewall',
+ producer: {
+ name: 'My SEO Tool Inc',
+ url: 'https://agencyanalytics.com/',
+ },
+ },
+ {
+ regex: 'ORTc\\.me',
+ name: 'OpenRobotsTXT',
+ category: 'Crawler',
+ url: 'https://openrobotstxt.org/ua',
+ producer: {
+ name: 'Majestic-12 Limited',
+ url: 'https://majestic.com/',
+ },
+ },
+ {
+ regex: 'obsrvr\\.net',
+ name: 'Observer',
+ category: 'Security Checker',
+ url: 'https://obsrvr.net/about',
+ },
+ {
+ includes: 'AliyunSecBot',
+ name: 'AliyunSecBot',
+ category: 'Security Checker',
+ url: 'https://service.alibaba.com/',
+ },
+ {
+ includes: 'Mediumbot',
+ name: 'Mediumbot',
+ category: 'Crawler',
+ url: 'https://medium.com/',
+ producer: {
+ name: 'A Medium Corporation',
+ url: 'https://medium.com/',
+ },
+ },
+ {
+ includes: 'Heexybot',
+ name: 'Heexybot',
+ category: 'Search bot',
+ url: 'https://heexy.org/bot',
+ },
+ {
+ includes: 'ForwardQR',
+ name: 'ForwardQR',
+ category: 'Service Agent',
+ url: 'https://forwardqr.com/bot.html',
+ },
+ {
+ includes: 'CybaaBot',
+ name: 'CybaaBot',
+ category: 'Security Checker',
+ url: 'https://cybaa.io/bot-policy',
+ producer: {
+ name: 'Cybaa Ltd',
+ url: 'https://cybaa.io/',
+ },
+ },
+ {
+ includes: 'CybaaAgent',
+ name: 'CybaaAgent',
+ category: 'Service Agent',
+ url: 'https://cybaa.io/bot-policy',
+ producer: {
+ name: 'Cybaa Ltd',
+ url: 'https://cybaa.io/',
+ },
+ },
+ {
+ includes: 'DVbot',
+ name: 'DVbot',
+ category: 'Service Agent',
+ url: 'https://doubleverify.com/',
+ producer: {
+ name: 'DoubleVerify Inc.',
+ url: 'https://doubleverify.com/',
+ },
+ },
+ {
+ includes: 'Assetnote',
+ name: 'Assetnote',
+ category: 'Security Checker',
+ url: 'https://www.assetnote.io/',
+ producer: {
+ name: 'Assetnote Pty Ltd',
+ url: 'https://www.assetnote.io/',
+ },
+ },
+ {
+ includes: 'UGAResearchAgent',
+ name: 'UGAResearchAgent',
+ category: 'Security Checker',
+ url: 'https://nislabuga-scan.uga.edu/',
+ producer: {
+ name: 'School of Computing at the University of Georgia (UGA)',
+ url: 'https://computing.uga.edu/',
+ },
+ },
+ {
+ includes: 'SERankingBacklinksBot',
+ name: 'SERankingBacklinksBot',
+ category: 'Crawler',
+ url: 'https://help.seranking.com/hc/en-us/articles/23020353159964-SERankingBacklinksBot-Crawler',
+ producer: {
+ name: 'SER Acquisition Inc,',
+ url: 'https://seranking.com/',
+ },
+ },
+ {
+ includes: 'VertexWP',
+ name: 'VertexWP',
+ category: 'Security Checker',
+ url: 'https://vertexwp.com/bot',
+ },
+ {
+ includes: 'Friendica',
+ name: 'Friendica',
+ category: 'Social Media Agent',
+ },
+ {
+ includes: 'Webliobot',
+ name: 'Webliobot',
+ category: 'Crawler',
+ url: 'https://help.weblio.jp/ejje/jpdictionary/j_info/j_i_bcontents/h0021',
+ producer: {
+ name: 'GRAS Group, Inc.',
+ url: 'https://gras-group.co.jp/',
+ },
+ },
+ {
+ includes: 'sqalix',
+ name: 'sqalix',
+ category: 'Crawler',
+ url: 'https://www.sqalix.com/',
+ },
+ {
+ includes: 'RecordedFuture',
+ name: 'RecordedFuture',
+ category: 'Security Checker',
+ url: 'https://www.recordedfuture.com/',
+ producer: {
+ name: 'Recorded Future, Inc.',
+ url: 'https://www.recordedfuture.com/',
+ },
+ },
+ {
+ includes: 'Valimail',
+ name: 'Valimail',
+ category: 'Service Agent',
+ url: 'https://www.valimail.com/domain-checker/',
+ producer: {
+ name: 'Valimail Inc.',
+ url: 'https://www.valimail.com/',
+ },
+ },
+ {
+ regex: 'FindFiles\\.net',
+ name: 'FindFiles.net',
+ category: 'Crawler',
+ url: 'https://findfiles.net/bot',
+ producer: {
+ name: 'FindFiles.net UG',
+ url: 'https://findfiles.net/',
+ },
+ },
+ {
+ includes: 'WanscannerBot',
+ name: 'WanscannerBot',
+ category: 'Security Checker',
+ url: 'https://abuse.pend.re/',
+ },
+ {
+ includes: 'Kagibot',
+ name: 'Kagibot',
+ category: 'Search bot',
+ url: 'https://kagi.com/bot',
+ producer: {
+ name: 'Kagi Inc.',
+ url: 'https://kagi.com/',
+ },
+ },
+ {
+ includes: 'Ai2Bot-Dolma',
+ name: 'Ai2Bot-Dolma',
+ category: 'AI Data Scraper',
+ url: 'https://allenai.org/crawler',
+ producer: {
+ name: 'The Allen Institute for Artificial Intelligence',
+ url: 'https://allenai.org/',
+ },
+ },
+ {
+ includes: 'Ai2Bot-DeepResearchEval',
+ name: 'Ai2Bot-DeepResearchEval',
+ category: 'AI Assistant',
+ url: 'https://allenai.org/crawler',
+ producer: {
+ name: 'The Allen Institute for Artificial Intelligence',
+ url: 'https://allenai.org/',
+ },
+ },
+ {
+ includes: 'Ai2Bot',
+ name: 'Ai2Bot',
+ category: 'AI Data Scraper',
+ url: 'https://allenai.org/crawler',
+ producer: {
+ name: 'The Allen Institute for Artificial Intelligence',
+ url: 'https://allenai.org/',
+ },
+ },
+ {
+ includes: 'WARDBot',
+ name: 'WARDBot',
+ category: 'Site Monitor',
+ url: 'https://ward.ai/robot',
+ producer: {
+ name: 'W.A.R.D. LLC',
+ url: 'https://ward.ai/',
+ },
+ },
+ {
+ includes: 'Character-AI',
+ name: 'Character-AI',
+ category: 'AI Assistant',
+ url: 'https://character.ai/',
+ producer: {
+ name: 'Character Technologies, Inc.',
+ url: 'https://character.ai/',
+ },
+ },
+ {
+ includes: 'DeepseekBot',
+ name: 'DeepseekBot',
+ category: 'AI Data Scraper',
+ url: 'https://www.deepseek.com/bot',
+ producer: {
+ name: 'Hangzhou Deep Search Artificiell Intelligens Basic Technology Research Co., Ltd.',
+ url: 'https://www.deepseek.com/',
+ },
+ },
+ {
+ includes: 'MistralAI-User',
+ name: 'MistralAI-User',
+ category: 'AI Assistant',
+ url: 'https://docs.mistral.ai/robots',
+ producer: {
+ name: 'Mistral AI',
+ url: 'https://mistral.ai/',
+ },
+ },
+ {
+ includes: 'Crawlspace',
+ name: 'Crawlspace',
+ category: 'AI Data Scraper',
+ url: 'https://crawlspace.dev/',
+ producer: {
+ name: 'Crawlspace Company',
+ url: 'https://crawlspace.dev/',
+ },
+ },
+ {
+ includes: 'Devin',
+ name: 'Devin',
+ category: 'AI Assistant',
+ url: 'https://cognition.ai/',
+ producer: {
+ name: 'Cognition AI, Inc.',
+ url: 'https://cognition.ai/',
+ },
+ },
+ {
+ includes: 'Andibot',
+ name: 'Andibot',
+ category: 'AI Search Crawler',
+ url: 'https://andisearch.com/',
+ producer: {
+ name: 'Andi',
+ url: 'https://andisearch.com/',
+ },
+ },
+ {
+ includes: 'HypeStat',
+ name: 'HypeStat',
+ category: 'Crawler',
+ url: 'https://hypestat.com/crawler',
+ },
+ {
+ includes: 'StatusNestBacklinkSpider',
+ name: 'StatusNestBacklinkSpider',
+ category: 'Crawler',
+ url: 'https://statusnest.com/bot',
+ },
+ {
+ includes: 'RunPod-Bot',
+ name: 'RunPod-Bot',
+ category: 'AI Data Scraper',
+ url: 'https://www.runpod.io/',
+ producer: {
+ name: 'RunPod Inc.',
+ url: 'https://www.runpod.io/',
+ },
+ },
+ {
+ includes: 'HuggingFace-Bot',
+ name: 'HuggingFace-Bot',
+ category: 'AI Data Scraper',
+ url: 'https://huggingface.co/',
+ producer: {
+ name: 'Hugging Face, Inc',
+ url: 'https://huggingface.co/',
+ },
+ },
+ {
+ includes: 'LinkBloom',
+ name: 'LinkBloom',
+ category: 'Crawler',
+ url: 'https://hamidsoltani.com/linkbloom',
+ },
+ {
+ includes: 'img2dataset',
+ name: 'img2dataset',
+ category: 'Crawler',
+ url: 'https://github.com/rom1504/img2dataset',
+ },
+ {
+ includes: 'iAskBot',
+ name: 'iAskBot',
+ category: 'AI Data Scraper',
+ url: 'https://iask.ai/',
+ producer: {
+ name: 'Ai Search Inc.',
+ url: 'https://iask.ai/',
+ },
+ },
+ {
+ includes: 'LinerBot',
+ name: 'LinerBot',
+ category: 'AI Assistant',
+ url: 'https://docs.getliner.com/docs/linerbot',
+ producer: {
+ name: 'Liner',
+ url: 'https://liner.com/',
+ },
+ },
+ {
+ includes: 'Poggio-Citations',
+ name: 'Poggio-Citations',
+ category: 'AI Assistant',
+ url: 'https://docs.poggio.io/dev-resources/robots',
+ producer: {
+ name: 'Poggio Labs, Inc.',
+ url: 'https://poggio.io/',
+ },
+ },
+ {
+ includes: 'QualifiedBot',
+ name: 'QualifiedBot',
+ category: 'AI Assistant',
+ url: 'https://www.qualified.com/legal/qualified-crawler-user-agent',
+ producer: {
+ name: 'Qualified.com, Inc.',
+ url: 'https://www.qualified.com/',
+ },
+ },
+ {
+ includes: 'ZanistaBot',
+ name: 'ZanistaBot',
+ category: 'AI Search Crawler',
+ url: 'https://zanista.ai/crawler-info',
+ producer: {
+ name: 'Zanista AI LTD.',
+ url: 'https://zanista.ai/',
+ },
+ },
+ {
+ includes: 'ChatGLM-Spider',
+ name: 'ChatGLM-Spider',
+ category: 'AI Data Scraper',
+ url: 'https://chatglm.cn/',
+ producer: {
+ name: 'Beijing Zhipu Huazhang Technology Co., Ltd.',
+ url: 'https://zhipuai.cn/',
+ },
+ },
+ {
+ regex: 'gochitchat\\.ai',
+ name: 'Sider',
+ category: 'AI Data Scraper',
+ url: 'https://sider.ai/',
+ producer: {
+ name: 'Sider AI',
+ url: 'https://sider.ai/',
+ },
+ },
+ {
+ includes: 'SBIntuitionsBot',
+ name: 'SBIntuitionsBot',
+ category: 'AI Data Scraper',
+ url: 'https://www.sbintuitions.co.jp/en/bot/',
+ producer: {
+ name: 'SB Intuitions Corp.',
+ url: 'https://www.sbintuitions.co.jp/',
+ },
+ },
+ {
+ includes: 'ias_crawler',
+ name: 'IAS Crawler',
+ category: 'AI Data Scraper',
+ url: 'https://integralads.com/ias-privacy-data-management/policies/site-indexing-policy/',
+ producer: {
+ name: 'Integral Ad Science, Inc.',
+ url: 'https://integralads.com/',
+ },
+ },
+ {
+ includes: 'ias_wombles',
+ name: 'IAS Wombles',
+ category: 'AI Data Scraper',
+ url: 'https://integralads.com/ias-privacy-data-management/policies/site-indexing-policy/',
+ producer: {
+ name: 'Integral Ad Science, Inc.',
+ url: 'https://integralads.com/',
+ },
+ },
+ {
+ includes: 'ShapBot',
+ name: 'ShapBot',
+ category: 'AI Data Scraper',
+ url: 'https://docs.parallel.ai/resources/crawler',
+ producer: {
+ name: 'Parallel Web Systems Inc.',
+ url: 'https://parallel.ai/',
+ },
+ },
+ {
+ includes: 'LumeWebScan',
+ name: 'LumeWebScan',
+ category: 'Crawler',
+ url: 'https://lumeweaver.com/',
+ producer: {
+ name: 'LumeWeaver Labs',
+ url: 'https://lumeweaver.com/',
+ },
+ },
+ {
+ includes: 'PanguBot',
+ name: 'PanguBot',
+ category: 'AI Data Scraper',
+ url: 'https://www.huawei.com/',
+ producer: {
+ name: 'Huawei Technologies Co., Ltd.',
+ url: 'https://www.huawei.com/',
+ },
+ },
+ {
+ includes: 'Manus-User',
+ name: 'Manus-User',
+ category: 'AI Assistant',
+ url: 'https://manus.im/help/Manus-user',
+ producer: {
+ name: 'Butterfly Effect Pte. Ltd.',
+ url: 'https://manus.im/',
+ },
+ },
+ {
+ includes: 'HubSpotContentSearchBot',
+ name: 'HubSpotContentSearchBot',
+ category: 'Crawler',
+ url: 'https://developers.hubspot.com/docs/cms/start-building/building-blocks/modules/content-search',
+ producer: {
+ name: 'HubSpot, Inc.',
+ url: 'https://www.hubspot.com/',
+ },
+ },
+ {
+ includes: 'ProRata',
+ name: 'ProRata',
+ category: 'AI Search Crawler',
+ url: 'https://platform.gist.ai/docs/gcn-overview',
+ producer: {
+ name: 'ProRataAI, Inc.',
+ url: 'https://prorata.ai/',
+ },
+ },
+ {
+ includes: 'blackboardally',
+ name: 'Blackboard Ally',
+ category: 'Crawler',
+ url: 'https://ally.ac/',
+ producer: {
+ name: 'Anthology Inc.',
+ url: 'https://www.anthology.com/',
+ },
+ },
+ {
+ regex: 'semantic-visions\\.com',
+ name: 'svEye',
+ category: 'Crawler',
+ url: 'https://sveye.semantic-visions.com/',
+ producer: {
+ name: 'Semantic Visions, s.r.o.',
+ url: 'https://www.semantic-visions.com/',
+ },
+ },
+ {
+ includes: 'CopyvioDetector',
+ name: 'CopyvioDetector',
+ category: 'Crawler',
+ url: 'https://copyvios.toolforge.org/',
+ producer: {
+ name: 'Ben Kurtovic',
+ url: 'https://github.com/earwig',
+ },
+ },
+ {
+ includes: 'Eyeotabot',
+ name: 'Eyeotabot',
+ category: 'Crawler',
+ url: 'https://www.eyeota.com/',
+ producer: {
+ name: 'Eyeota Pte Ltd',
+ url: 'https://www.eyeota.com/',
+ },
+ },
+ {
+ includes: 'YioopBot',
+ name: 'YioopBot',
+ category: 'Search bot',
+ url: 'https://www.yioop.com/bot',
+ producer: {
+ name: 'Chris Pollett',
+ url: 'https://www.pollett.org/',
+ },
+ },
+ {
+ includes: 'yoozBot',
+ name: 'yoozBot',
+ category: 'Search bot',
+ url: 'https://www.yooz.ir/',
+ producer: {
+ name: 'yooz',
+ url: 'https://www.yooz.ir/',
+ },
+ },
+ {
+ regex:
+ '(ABEvalBot|AdsTxtCrawlerTP|DNSResearchBot|Dormouse|FormFinder|GitCrawlerBot|HanaleiBot|ImportDomains|PrivacyPolicyBot|Robozilla|Secweb-Sectxt|SeoCherryBot|StudyBot|The Knowledge AI|Thinkbot|ThinkChaos|TprAdsTxtCrawler)',
+ name: '$1',
+ category: 'Crawler',
+ },
{
regex:
- 'nuhk|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr\\.com|teoma|oegp|http%20client|htdig|mogimogi|larbin|scrubby|searchsight|semanticdiscovery|snappy|zeal(?!ot)|dataparksearch|findlinks|BrowserMob|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|7Siters|centuryb\\.o\\.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|cortex|Re-re Studio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle Client|Hello,? world|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M CODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux Gnu \\(cow\\)|Test Certificate Info|iplabel|Magellan|TheSafex?Internetx?Search|Searcherx?web|kirkland-signature|LinkChain|survey-security-dot-txt|infrawatch|Time/|r00ts3c-owned-you|nvdorz|Root Slut|NiggaBalls|BotPoke|GlobalWebSearch|xx032_bo9vs83_2a|sslshed|geckotrail|Wordup|Keydrop|\\(compatible\\)|John Recon|SPARK COMMIT|masjesu|Komaru_The_Cat|Jesus Christ of Nazareth is LORD|Kowai|Hakai|LoliSec|LMAO|^xenu|^(?:chrome|firefox|Abcd|Dark|KvshClient|Node.js|Report Runner|url|Zeus|ZmEu)$|OnlyScans|TheInternetSearchx',
+ 'nuhk|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr\\.com|teoma|oegp|http%20client|htdig|mogimogi|larbin|scrubby|searchsight|semanticdiscovery|snappy|zeal(?!ot)|dataparksearch|findlinks|BrowserMob|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|7Siters|centuryb\\.o\\.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|cortex|Re-re Studio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle Client|Hello,? world|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M CODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux Gnu \\(cow\\)|Test Certificate Info|iplabel|Magellan|TheSafex?Internetx?Search|Searcherx?web|kirkland-signature|LinkChain|survey-security-dot-txt|infrawatch|Time/|r00ts3c-owned-you|nvdorz|Root Slut|NiggaBalls|BotPoke|GlobalWebSearch|xx032_bo9vs83_2a|sslshed|geckotrail|Wordup|Keydrop|\\(compatible\\)|John Recon|SPARK COMMIT|masjesu|Komaru_The_Cat|Jesus Christ of Nazareth is LORD|Kowai|Hakai|LoliSec|LMAO|^xenu|^(?:chrome|desktop|firefox|Abcd|Dark|KvshClient|node|Node\\.js|Report Runner|url|Zeus|ZmEu)$|OnlyScans|TheInternetSearchx|Laravel Reaver|bang2013|libredtail|Mozilliqa|Tiberius|honeygain|AW-WB-Filter|SaferSoftwashLeadGen|YourUserAgentHere|phpneuralnetwork',
name: 'Generic Bot',
},
{
regex:
- '[a-z0-9_-]*(?:(?<!cu|power[ _]|m[ _])bot(?![ _]TAB|[ _]?5[0-9]|[ _]Senior|[ _]Junior)|analyzer|appengine|archiver?|checker|collector|crawl|crawler|(?<!node-|uclient-|Mikrotik/\\d\\.[x\\d] |electron-)fetch(?:er)?|indexer|inspector|monitor|(?<!Microsoft |banshee-)project(?!or)|(?<!Google Wap |Blue |SpeedMode; )proxy|(?<!P)research|resolver|robots|(?<!Cam)scanner|scraper|script|searcher|(?<!-)security|spider(?! 8)|study|transcoder|uptime|user[ _]?agent|validator|-(?:AI|Extended|User)/)(?:[^a-z]|$)',
+ '[a-z0-9_-]*(?:(?<!cu|Hu|power[ _]|m[ _])bot(?![ _]TAB|[ _]?5[0-9]|[ _]Senior|[ _]Junior)|analyzer|appengine|archiver?|checker|collector|crawl|crawler|(?<!node-|uclient-|Mikrotik/\\d\\.[x\\d] |electron-)fetch(?:er)?|(?<!url)grabber|indexer|inspector|monitor|^parser|(?<!Microsoft |banshee-)project(?!or)|(?<!Google Wap |Blue |SpeedMode; )proxy|(?<!P)research|resolver|robots|(?<!Cam)scanner|scraper|script|searcher|(?<!-)security|spider(?! 8)|study|transcoder|uptime|user[ _]?agent|validator|-(?:AI|Extended|User)/)(?:[^a-z]|$)',
name: 'Generic Bot',
},
] as const;
diff --git a/packages/common/scripts/get-referrers.ts b/packages/common/scripts/get-referrers.ts
index 14808da03..eee3ff19f 100644
--- a/packages/common/scripts/get-referrers.ts
+++ b/packages/common/scripts/get-referrers.ts
@@ -129,7 +129,7 @@ async function main() {
).then((res) => res.json());
fs.writeFileSync(
- path.resolve(__dirname, '../../worker/src/referrers/index.ts'),
+ path.resolve(__dirname, '../server/referrers/index.ts'),
[
'// This file is generated by the script get-referrers.ts',
'',
diff --git a/packages/common/server/referrers/index.ts b/packages/common/server/referrers/index.ts
index bb1de3d15..fe40a88bf 100644
--- a/packages/common/server/referrers/index.ts
+++ b/packages/common/server/referrers/index.ts
@@ -2781,5 +2781,64 @@ const referrers: Record<string, { type: string; name: string }> = {
'office.com': { type: 'tech', name: 'Microsoft Office' },
'squarespace.com': { type: 'commerce', name: 'Squarespace' },
'teams.microsoft.com': { type: 'social', name: 'Microsoft Teams' },
+ 'chat.com': { type: 'ai', name: 'Chat.com' },
+ 'chatgpt.com': { type: 'ai', name: 'ChatGPT' },
+ 'openai.com': { type: 'ai', name: 'OpenAI' },
+ 'anthropic.com': { type: 'ai', name: 'Anthropic' },
+ 'claude.ai': { type: 'ai', name: 'Claude' },
+ 'gemini.google.com': { type: 'ai', name: 'Google Gemini' },
+ 'bard.google.com': { type: 'ai', name: 'Google Bard' },
+ 'copilot.microsoft.com': { type: 'ai', name: 'Microsoft Copilot' },
+ 'copilot.cloud.microsoft': { type: 'ai', name: 'Microsoft Copilot' },
+ 'perplexity.ai': { type: 'ai', name: 'Perplexity' },
+ 'you.com': { type: 'ai', name: 'You.com' },
+ 'poe.com': { type: 'ai', name: 'Poe' },
+ 'phind.com': { type: 'ai', name: 'Phind' },
+ 'huggingface.co': { type: 'ai', name: 'Hugging Face' },
+ 'hf.co': { type: 'ai', name: 'Hugging Face' },
+ 'character.ai': { type: 'ai', name: 'Character.AI' },
+ 'meta.ai': { type: 'ai', name: 'Meta AI' },
+ 'mistral.ai': { type: 'ai', name: 'Mistral' },
+ 'chat.mistral.ai': { type: 'ai', name: 'Mistral Le Chat' },
+ 'deepseek.com': { type: 'ai', name: 'DeepSeek' },
+ 'chat.deepseek.com': { type: 'ai', name: 'DeepSeek Chat' },
+ 'pi.ai': { type: 'ai', name: 'Pi' },
+ 'inflection.ai': { type: 'ai', name: 'Inflection' },
+ 'cohere.com': { type: 'ai', name: 'Cohere' },
+ 'coral.cohere.com': { type: 'ai', name: 'Cohere Coral' },
+ 'jasper.ai': { type: 'ai', name: 'Jasper' },
+ 'writesonic.com': { type: 'ai', name: 'Writesonic' },
+ 'copy.ai': { type: 'ai', name: 'Copy.ai' },
+ 'rytr.me': { type: 'ai', name: 'Rytr' },
+ 'notion.ai': { type: 'ai', name: 'Notion AI' },
+ 'grammarly.com': { type: 'ai', name: 'Grammarly' },
+ 'grok.com': { type: 'ai', name: 'Grok' },
+ 'x.ai': { type: 'ai', name: 'xAI' },
+ 'aistudio.google.com': { type: 'ai', name: 'Google AI Studio' },
+ 'labs.google.com': { type: 'ai', name: 'Google Labs' },
+ 'ai.google': { type: 'ai', name: 'Google AI' },
+ 'forefront.ai': { type: 'ai', name: 'Forefront' },
+ 'together.ai': { type: 'ai', name: 'Together AI' },
+ 'groq.com': { type: 'ai', name: 'Groq' },
+ 'replicate.com': { type: 'ai', name: 'Replicate' },
+ 'vercel.ai': { type: 'ai', name: 'Vercel AI' },
+ 'v0.dev': { type: 'ai', name: 'v0' },
+ 'bolt.new': { type: 'ai', name: 'Bolt' },
+ 'replit.com': { type: 'ai', name: 'Replit' },
+ 'cursor.com': { type: 'ai', name: 'Cursor' },
+ 'tabnine.com': { type: 'ai', name: 'Tabnine' },
+ 'codeium.com': { type: 'ai', name: 'Codeium' },
+ 'sourcegraph.com': { type: 'ai', name: 'Sourcegraph Cody' },
+ 'kimi.moonshot.cn': { type: 'ai', name: 'Kimi' },
+ 'moonshot.ai': { type: 'ai', name: 'Moonshot AI' },
+ 'doubao.com': { type: 'ai', name: 'Doubao' },
+ 'tongyi.aliyun.com': { type: 'ai', name: 'Tongyi Qianwen' },
+ 'yiyan.baidu.com': { type: 'ai', name: 'Ernie Bot' },
+ 'chatglm.cn': { type: 'ai', name: 'ChatGLM' },
+ 'zhipu.ai': { type: 'ai', name: 'Zhipu AI' },
+ 'minimax.chat': { type: 'ai', name: 'MiniMax' },
+ 'lmsys.org': { type: 'ai', name: 'LMSYS' },
+ 'chat.lmsys.org': { type: 'ai', name: 'LMSYS Chat' },
+ 'llama.meta.com': { type: 'ai', name: 'Meta Llama' },
} as const;
export default referrers;