OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
chore: update bots and referrers
Carl-Gerhard Lindesvärd113ab0a28dcf
diff --git a/apps/api/src/bots/bots.ts b/apps/api/src/bots/bots.tsindex 7b9096909..c6933bdc8 100644--- a/apps/api/src/bots/bots.ts+++ b/apps/api/src/bots/bots.ts@@ -4,7 +4,7 @@const bots = [{- regex: 'WireReaderBot(?:/([\\d+.]+))?',+ regex: 'WireReaderBot',name: 'WireReaderBot',category: 'Feed Fetcher',url: 'https://wirereader.app/',@@ -65,7 +65,7 @@ const bots = [producer: { name: 'Ahrefs Pte Ltd', url: 'https://ahrefs.com/robot' },},{- regex: 'AhrefsSiteAudit/[\\d.]+',+ regex: 'AhrefsSiteAudit',name: 'AhrefsSiteAudit',category: 'Site Monitor',url: 'https://ahrefs.com/robot/site-audit',@@ -86,14 +86,14 @@ const bots = [producer: { name: 'Alexa Internet', url: 'https://www.alexa.com' },},{- regex: 'Amazonbot/[\\d.]+',+ regex: 'Amazonbot',name: 'Amazon Bot',category: 'Crawler',url: 'https://developer.amazon.com/support/amazonbot',producer: { name: 'Amazon.com, Inc.', url: 'https://www.amazon.com/' },},{- regex: 'AmazonAdBot/[\\d.]+',+ regex: 'AmazonAdBot',name: 'Amazon AdBot',category: 'Crawler',url: 'https://adbot.amazon.com/',@@ -576,7 +576,7 @@ const bots = [producer: { name: 'Meta Platforms, Inc.', url: 'https://www.meta.com/' },},{- regex: 'FacebookBot/[\\d.]+',+ regex: 'FacebookBot',name: 'FacebookBot',category: 'Crawler',url: 'https://developers.facebook.com/docs/sharing/bot',@@ -621,7 +621,7 @@ const bots = [producer: { name: '', url: '' },},{- regex: 'Fever/[0-9]',+ regex: 'Fever/',name: 'Fever',url: 'http://feedafever.com/',category: 'Feed Fetcher',@@ -863,7 +863,7 @@ const bots = [url: 'https://vuhuv.com/bot.html',},{- regex: 'HTTPMon/[\\d.]+',+ regex: 'HTTPMon',name: 'HTTPMon',category: 'Site Monitor',url: 'http://www.httpmon.com',@@ -905,7 +905,7 @@ const bots = [url: '',producer: { name: '', url: 'https://ip-guide.com' },},- { regex: 'k6/[0-9.]+', name: 'K6', url: 'https://k6.io/' },+ { regex: 'k6/', name: 'K6', url: 'https://k6.io/' },{regex: 'kouio',name: 'Kouio',@@ -980,7 +980,7 @@ const bots = [producer: { name: '', url: '' },},{- regex: 'masscan-ng/[\\d.]+',+ regex: 'masscan-ng',name: 'masscan-ng',url: 'https://github.com/bi-zone/masscan-ng',category: 'Crawler',@@ -1136,7 +1136,7 @@ const bots = [url: 'https://nodeping.com',producer: { name: 'NodePing', url: 'https://nodeping.com' },},- { regex: 'Octopus [0-9]', name: 'Octopus' },+ { regex: 'Octopus [\\d.]+', name: 'Octopus' },{regex: 'OnlineOrNot\\.com_bot',name: 'OnlineOrNot Bot',@@ -1206,7 +1206,7 @@ const bots = [},},{- regex: 'Pocket(?:ImageCache|Parser)/[\\d.]+',+ regex: 'Pocket(?:ImageCache|Parser)',name: 'Pocket',category: 'Read-it-later Service',url: 'https://getpocket.com/pocketparser_ua',@@ -1347,7 +1347,14 @@ const bots = [producer: { name: 'Semrush Inc.', url: 'https://www.semrush.com/' },},{- regex: 'SerpReputationManagementAgent/[\\d.]+',+ regex: 'BacklinksExtendedBot',+ name: 'BacklinksExtendedBot',+ category: 'Crawler',+ url: 'https://www.semrush.com/bot/',+ producer: { name: 'Semrush Inc.', url: 'https://www.semrush.com/' },+ },+ {+ regex: 'SerpReputationManagementAgent',name: 'Semrush Reputation Management',category: 'Service Agent',url: 'https://www.semrush.com/bot/',@@ -1361,7 +1368,7 @@ const bots = [producer: { name: 'Semrush Inc.', url: 'https://www.semrush.com/' },},{- regex: 'SiteAuditBot/[\\d.]+',+ regex: 'SiteAuditBot',name: 'SiteAuditBot',category: 'Crawler',url: 'https://www.semrush.com/bot/',@@ -1381,13 +1388,6 @@ const bots = [url: 'http://www.seoengine.com/seoengbot.htm',producer: { name: 'SEO Engine', url: 'http://www.seoengine.com' },},- {- regex: 'SEOkicks-Robot',- name: 'SEOkicks-Robot',- category: 'Crawler',- url: 'http://www.seokicks.de/robot.html',- producer: { name: 'SEOkicks', url: 'https://www.seokicks.de/' },- },{regex: 'seoscanners\\.net',name: 'Seoscanners.net',@@ -1495,13 +1495,6 @@ const bots = [url: '',producer: { name: 'Sprinklr, Inc.', url: 'https://www.sprinklr.com/' },},- {- regex: 'sqlmap/',- name: 'sqlmap',- category: 'Security Checker',- url: 'http://sqlmap.org/',- producer: { name: 'sqlmap', url: 'http://sqlmap.org/' },- },{regex: 'SSL Labs',name: 'SSL Labs',@@ -1527,7 +1520,7 @@ const bots = [producer: { name: 'Superfeedr', url: 'https://superfeedr.com/' },},{- regex: 'Sparkler/[0-9]',+ regex: 'Sparkler',name: 'Sparkler',category: 'Crawler',url: 'https://github.com/USCDataScience/sparkler',@@ -1660,7 +1653,7 @@ const bots = [producer: { name: 'UkrNet Ltd', url: 'https://www.ukr.net/' },},{- regex: 'Uptime(?:bot)?/[\\d.]+',+ regex: 'Uptime(?:bot)?/',name: 'Uptimebot',category: 'Site Monitor',url: 'https://uptime.com/uptime-bot',@@ -1798,7 +1791,7 @@ const bots = [producer: { name: 'WPBeginner, LLC', url: 'https://www.wpbeginner.com/' },},{- regex: 'Automattic Analytics Crawler/[\\d.]+',+ regex: 'Automattic Analytics Crawler',name: 'Automattic Analytics',category: 'Crawler',url: 'https://wordpress.com/crawler/',@@ -1925,7 +1918,7 @@ const bots = [producer: { name: 'NetEase, Inc.', url: 'http://corp.163.com' },},{- regex: 'YOURLS v[0-9]',+ regex: 'YOURLS',name: 'Yourls',category: 'Crawler',url: 'http://yourls.org',@@ -1986,7 +1979,7 @@ const bots = [producer: { name: 'HubPages, Inc.', url: 'https://discover.hubpages.com/' },},{- regex: 'Pinterest(?:bot)?/[\\d.]+.*www\\.pinterest\\.com',+ regex: 'Pinterest(?:bot)?/.*www\\.pinterest\\.com',name: 'Pinterest',url: 'https://help.pinterest.com/en/business/article/pinterest-crawler',category: 'Crawler',@@ -2000,7 +1993,7 @@ const bots = [producer: { name: 'Site24x7', url: 'https://www.site24x7.com' },},{- regex: '.* HLB/[\\d.]+',+ regex: '.* HLB',name: 'Site24x7 Defacement Monitor',category: 'Site Monitor',url: 'https://support.site24x7.com/portal/en/kb/articles/default-user-agent-used-in-website-defacement-monitor',@@ -2021,7 +2014,7 @@ const bots = [producer: { name: 'Snapchat Inc.', url: 'https://www.snapchat.com/' },},{- regex: 'SnapchatAds/[\\d.]+',+ regex: 'SnapchatAds',name: 'Snapchat Ads',category: 'Crawler',url: 'https://businesshelp.snapchat.com/s/article/adsbot-crawler?language=en_US',@@ -2143,7 +2136,7 @@ const bots = [producer: { name: 'WooRank sprl', url: 'https://www.woorank.com/' },},{- regex: 'by Siteimprove\\.com',+ regex: 'Siteimprove',name: 'Siteimprove',category: 'Search bot',url: 'https://siteimprove.com/',@@ -2195,11 +2188,11 @@ const bots = [producer: { name: 'Effyis Inc', url: 'https://boardreader.com/' },},{- regex: 'IDG/IT',- name: 'IDG/IT',- category: 'Search bot',- url: 'https://spaziodati.eu/',- producer: { name: 'SpazioDati S.r.l.', url: 'https://spaziodati.eu/' },+ regex: 'IDG/(?:EU|IT|RU|UK)',+ name: 'IDG',+ category: 'Crawler',+ url: 'https://www.spaziodati.eu/',+ producer: { name: 'SpazioDati S.r.l.', url: 'https://www.spaziodati.eu/' },},{regex: 'Bytespider',@@ -2386,7 +2379,7 @@ const bots = [producer: { name: 'PPC Labs LLC', url: 'https://www.adbeat.com/' },},{- regex: '(?:BuiltWith|BW)/[\\d.]+',+ regex: '(?:BuiltWith|BW/)',name: 'BuiltWith',category: 'Crawler',url: 'https://builtwith.com/biup',@@ -2445,7 +2438,7 @@ const bots = [url: 'http://cloudsystemnetworks.com',},{- regex: 'HeartRails_Capture/[\\d.]+',+ regex: 'HeartRails_Capture',name: 'Heart Rails Capture',category: 'Service Agent',url: 'http://capture.heartrails.com',@@ -2458,7 +2451,7 @@ const bots = [producer: { name: 'RedHunt Labs Limited', url: 'https://redhuntlabs.com/' },},{- regex: 'DataXu/[\\d.]+',+ regex: 'DataXu',name: 'DataXu',category: 'Service Agent',url: 'https://advertising.roku.com/dataxu',@@ -2584,13 +2577,13 @@ const bots = [producer: { name: 'Hatena Co., Ltd.', url: 'https://www.hatena.ne.jp' },},{- regex: 'RyowlEngine/[\\d.]+',+ regex: 'RyowlEngine',name: 'Ryowl',category: 'Crawler',url: 'https://ryowl.org',},{- regex: 'OdklBot/[\\d.]+',+ regex: 'OdklBot',name: 'Odnoklassniki Bot',category: 'Crawler',url: 'https://odnoklassniki.ru',@@ -2608,7 +2601,7 @@ const bots = [url: 'https://www.zoominfo.com',},{- regex: 'WeViKaBot/[\\d.]+',+ regex: 'WeViKaBot',name: 'WeViKaBot',category: 'Crawler',url: 'http://www.wevika.de',@@ -2618,9 +2611,10 @@ const bots = [name: 'SEOkicks',category: 'Crawler',url: 'https://www.seokicks.de/robot.html',+ producer: { name: 'SEOkicks', url: 'https://www.seokicks.de/' },},{- regex: 'Plukkie/[\\d.]+',+ regex: 'Plukkie',name: 'Plukkie',category: 'Crawler',url: 'http://www.botje.com/plukkie.htm',@@ -2632,25 +2626,25 @@ const bots = [url: 'https://www.comscore.com/Web-Crawler',},{- regex: 'SurdotlyBot/[\\d.]+',+ regex: 'SurdotlyBot',name: 'SurdotlyBot',category: 'Crawler',url: 'http://sur.ly/bot.html',},{- regex: 'Gowikibot/[\\d.]+',+ regex: 'Gowikibot',name: 'Gowikibot',category: 'Crawler',url: 'http:/www.gowikibot.com',},{- regex: 'SabsimBot/[\\d.]+',+ regex: 'SabsimBot',name: 'SabsimBot',category: 'Crawler',url: 'https://sabsim.com',},{- regex: 'LumtelBot/[\\d.]+',+ regex: 'LumtelBot',name: 'LumtelBot',category: 'Crawler',url: 'https://umtel.com',@@ -2662,13 +2656,13 @@ const bots = [url: 'http://www.pipl.com/bot',},{- regex: 'woobot/[\\d.]+',+ regex: 'woobot',name: 'WooRank',category: 'Crawler',url: 'https://www.woorank.com/bot',},{- regex: 'Cookiebot/[\\d.]+',+ regex: 'Cookiebot',name: 'Cookiebot',category: 'Crawler',url: 'https://support.cookiebot.com/hc/en-us/articles/360014264140-Scanner-User-Agent',@@ -2685,7 +2679,7 @@ const bots = [},},{- regex: 'CensysInspect/[\\d.]+',+ regex: 'CensysInspect',name: 'CensysInspect',category: 'Security Checker',url: 'https://about.censys.io/',@@ -2702,26 +2696,26 @@ const bots = [},},{- regex: 'WellKnownBot/[\\d.]+',+ regex: 'WellKnownBot',name: 'WellKnownBot',category: 'Crawler',url: 'https://well-known.dev',},{- regex: 'Adsbot/[\\d.]+',+ regex: 'Adsbot',name: 'Adsbot',category: 'Crawler',url: 'https://seostar.co/robot/',},{- regex: 'MTRobot/[\\d.]+',+ regex: 'MTRobot',name: 'MTRobot',category: 'Crawler',url: 'https://metrics-tools.de/robot.html',producer: { name: 'Metrics Tools', url: 'https://metrics-tools.de/' },},{- regex: 'serpstatbot/[\\d.]+',+ regex: 'serpstatbot',name: 'serpstatbot',category: 'Crawler',url: 'http://serpstatbot.com/',@@ -2734,26 +2728,26 @@ const bots = [url: 'https://github.com/gocolly/colly/',},{- regex: 'l9tcpid/v[\\d.]+',+ regex: 'l9tcpid',name: 'l9tcpid',category: 'Security Checker',url: 'https://github.com/LeakIX/l9tcpid',},{- regex: 'l9explore/[\\d.]+',+ regex: 'l9explore',name: 'l9explore',category: 'Security Checker',url: 'https://github.com/LeakIX/l9explore',},{- regex: 'l9scan/|^Lkx-.*/[\\d.]+',+ regex: 'l9scan/|^Lkx-.*/',name: 'LeakIX',category: 'Security Checker',url: 'https://leakix.net/',producer: { name: 'BaDaaS SRL', url: 'https://leakix.net/' },},{- regex: 'MegaIndex\\.ru/[\\d.]+',+ regex: 'MegaIndex\\.ru',name: 'MegaIndex',category: 'Crawler',url: 'https://megaindex.com/crawler',@@ -2766,60 +2760,67 @@ const bots = [producer: { name: 'SISTRIX GmbH', url: 'https://www.sistrix.de/' },},{- regex: 'seolyt/[\\d.]+',- name: 'seolyt',+ regex: 'Seolyt(?:Bot)?',+ name: 'SeolytBot',category: 'Crawler',url: 'https://seolyt.com/',},{- regex: 'YaK/[\\d.]+',+ regex: 'YaK/',name: 'YaK',category: 'Crawler',url: 'https://www.linkfluence.com/',producer: { name: 'Linkfluence SAS', url: 'https://www.linkfluence.com/' },},{- regex: 'KomodiaBot/[\\d.]+',+ regex: 'KomodiaBot',name: 'KomodiaBot',category: 'Crawler',url: 'http://www.komodia.com/newwiki/index.php/URL_server_crawler',producer: { name: 'Komodia Inc.', url: 'https://www.komodia.com/' },},{- regex: 'KStandBot/[\\d.]+',+ regex: 'KStandBot',name: 'KStandBot',category: 'Crawler',url: 'https://url-classification.io/wiki/index.php?title=URL_server_crawler',producer: { name: 'Komodia Inc.', url: 'https://www.komodia.com/' },},{- regex: 'Neevabot/[\\d.]+',+ regex: 'Neevabot',name: 'Neevabot',category: 'Search bot',url: 'https://neeva.com/neevabot',producer: { name: 'Neeva Inc.', url: 'https://neeva.com/' },},{- regex: 'LinkPreview/[\\d.]+',+ regex: 'Chatwork LinkPreview',+ name: 'Chatwork LinkPreview',+ category: 'Service Agent',+ url: 'https://go.chatwork.com/en/',+ producer: { name: 'kubell Co., Ltd.', url: 'https://www.kubell.com/en/' },+ },+ {+ regex: 'LinkPreview',name: 'LinkPreview',category: 'Service Agent',url: 'https://www.linkpreview.net/',},{- regex: 'JungleKeyThumbnail/[\\d.]+',+ regex: 'JungleKeyThumbnail',name: 'JungleKeyThumbnail',category: 'Crawler',url: 'https://junglekey.com/',},{- regex: 'rocketmonitor(?: |bot/)[\\d.]+',+ regex: 'rocketmonitor(?:bot)?',name: 'RocketMonitorBot',category: 'Site Monitor',url: 'https://www.radiomast.io/docs/stream-monitoring/technical_details.html',producer: { name: 'Radio Mast, Inc.', url: 'https://www.radiomast.io/' },},{- regex: 'SitemapParser-VIPnytt/[\\d.]+',+ regex: 'SitemapParser-VIPnytt',name: 'SitemapParser-VIPnytt',category: 'Crawler',url: 'https://github.com/VIPnytt/SitemapParser/',@@ -2831,7 +2832,7 @@ const bots = [url: 'https://turnitin.com/robot/crawlerinfo.html',},{- regex: 'DMBrowser/[\\d.]+|DMBrowser-[UB]V',+ regex: 'DMBrowser|DMBrowser-[UB]V',name: 'Dotcom Monitor',category: 'Site Monitor',url: 'https://www.dotcom-monitor.com',@@ -2844,19 +2845,19 @@ const bots = [url: 'https://dataforseo.com/dataforseo-bot',},{- regex: 'Discordbot/[\\d.]+',+ regex: 'Discordbot',name: 'Discord Bot',category: 'Service Agent',url: 'https://discordapp.com',},{- regex: 'Linespider/[\\d.]+',+ regex: 'Linespider',name: 'Linespider',category: 'Crawler',url: 'https://lin.ee/4dwXkTH',},{- regex: 'Cincraw/[\\d.]+',+ regex: 'Cincraw',name: 'Cincraw',category: 'Crawler',url: 'http://cincrawdata.net/bot/',@@ -2895,19 +2896,19 @@ const bots = [},},{- regex: 'TigerBot/[\\d.]+',+ regex: 'TigerBot',name: 'TigerBot',category: 'Crawler',url: 'https://tiger.ch/',},{- regex: 'TestCrawler/[\\d.]+',+ regex: 'TestCrawler',name: 'TestCrawler',category: 'Crawler',url: 'https://www.comcepta.com/',},{- regex: 'CrowdTanglebot/[\\d.]+',+ regex: 'CrowdTanglebot',name: 'CrowdTangle',category: 'Crawler',url: 'https://help.crowdtangle.com/en/articles/3009319-crowdtangle-bot',@@ -2941,14 +2942,14 @@ const bots = [producer: { name: 'deepnoc, GmbH', url: 'https://deepnoc.com/' },},{- regex: 'Newslitbot/[\\d.]+',+ regex: 'Newslitbot',name: 'Newslitbot',category: 'Crawler',url: 'https://www.newslit.co/',producer: { name: 'Newslit, LLC.', url: 'https://www.newslit.co/' },},{- regex: 'um-(?:ANS|CC|FC|IC|LN)/[\\d.]+',+ regex: 'um-(?:ANS|CC|FC|IC|LN)',name: 'uMBot',category: 'Crawler',url: 'https://www.ubermetrics-technologies.com/',@@ -2958,7 +2959,7 @@ const bots = [},},{- regex: 'Abonti/[\\d.]+',+ regex: 'Abonti',name: 'Abonti',category: 'Crawler',url: 'http://abonti.com/',@@ -2981,7 +2982,7 @@ const bots = [},},{- regex: 'ev-crawler/[\\d.]+',+ regex: 'ev-crawler',name: 'Headline',category: 'Crawler',url: 'https://headline.com/legal/crawler',@@ -2991,7 +2992,7 @@ const bots = [},},{- regex: 'webprosbot/[\\d.]+',+ regex: 'webprosbot',name: 'WebPros',category: 'Crawler',url: 'https://webpros.com/',@@ -3005,7 +3006,7 @@ const bots = [producer: { name: 'Amazon.com, Inc.', url: 'https://www.amazon.com/' },},{- regex: 'Wheregoes\\.com Redirect Checker/[\\d.]+',+ regex: 'Wheregoes\\.com Redirect Checker',name: 'WhereGoes',category: 'Crawler',url: 'https://wheregoes.com/',@@ -3017,13 +3018,13 @@ const bots = [url: 'http://66.240.192.82/',},{- regex: 'InternetMeasurement/[\\d.]+',+ regex: 'InternetMeasurement',name: 'InternetMeasurement',category: 'Crawler',url: 'https://internet-measurement.com/',},{- regex: 'DomainAppender /[\\d.]+',+ regex: 'DomainAppender',name: 'DomainAppender',category: 'Crawler',url: 'https://www.profound.net/product/domain_append/',@@ -3033,7 +3034,7 @@ const bots = [},},{- regex: 'FreeWebMonitoring SiteChecker/[\\d.]+',+ regex: 'FreeWebMonitoring SiteChecker',name: 'FreeWebMonitoring',category: 'Site Monitor',url: 'https://www.freewebmonitoring.com/bot.html',@@ -3060,21 +3061,21 @@ const bots = [producer: { name: 'Jaohawi AB', url: 'https://adstxtlab.com/' },},{- regex: 'Iframely/[\\d.]+',+ regex: 'Iframely',name: 'Iframely',category: 'Crawler',url: 'https://iframely.com/',producer: { name: 'Itteco Software, Corp.', url: 'https://iframely.com/' },},{- regex: 'DomainStatsBot/[\\d.]+',+ regex: 'DomainStatsBot',name: 'DomainStatsBot',category: 'Crawler',url: 'https://domainstats.com/pages/our-bot',producer: { name: 'Domainstats Ltd', url: 'https://domainstats.com/' },},{- regex: 'aiHitBot/[\\d.]+',+ regex: 'aiHitBot',name: 'aiHitBot',category: 'Crawler',url: 'https://www.aihitdata.com/about',@@ -3088,7 +3089,7 @@ const bots = [{ regex: 'DNSResearchBot', name: 'DNSResearchBot', category: 'Crawler' },{ regex: 'GitCrawlerBot', name: 'GitCrawlerBot', category: 'Crawler' },{- regex: 'AdAuth/[\\d.]+',+ regex: 'AdAuth',name: 'AdAuth',category: 'Crawler',url: 'https://www.adauth.com',@@ -3166,7 +3167,7 @@ const bots = [},},{- regex: 'ScamadviserExternalHit/[\\d.]+',+ regex: 'ScamadviserExternalHit',name: 'Scamadviser External Hit',category: 'Crawler',url: 'https://www.scamadviser.com/',@@ -3183,26 +3184,20 @@ const bots = [producer: { name: 'Zaldamo, LLC.', url: 'https://www.zaldamo.com/' },},{- regex: 'AFB/[\\d.]+',+ regex: 'AFB',name: 'Allloadin Favicon Bot',category: 'Crawler',url: 'https://allloadin.com/',},{- regex: 'SeolytBot/[\\d.]+',- name: 'Seolyt Bot',- category: 'Crawler',- url: 'https://seolyt.com',- },- {- regex: 'LinkWalker/[\\d.]+',+ regex: 'LinkWalker',name: 'LinkWalker',category: 'Crawler',url: 'https://www.phishlabs.com/',producer: { name: 'PhishLabs, Inc.', url: 'https://www.phishlabs.com/' },},{- regex: 'RenovateBot/[\\d.]+',+ regex: 'RenovateBot',name: 'RenovateBot',category: 'Security Checker',url: 'https://github.com/renovatebot/renovate',@@ -3212,7 +3207,7 @@ const bots = [},},{- regex: 'INETDEX-BOT/[\\d.]+',+ regex: 'INETDEX-BOT',name: 'Inetdex Bot',category: 'Crawler',url: 'https://www.inetdex.com/',@@ -3242,28 +3237,28 @@ const bots = [},},{- regex: 'Nicecrawler/[\\d.]+',+ regex: 'Nicecrawler',name: 'NiceCrawler',category: 'Crawler',url: 'https://www.nicecrawler.com/',producer: { name: 'Intelium Corp.', url: 'https://www.intelium.com/' },},{- regex: 't3versionsBot/[\\d.]+',+ regex: 't3versionsBot',name: 't3versions',category: 'Crawler',url: 'https://www.t3versions.com/bot',producer: { name: 'Torben Hansen', url: 'https://www.t3versions.com/' },},{- regex: 'Crawlson/[\\d.]+',+ regex: 'Crawlson',name: 'Crawlson',category: 'Crawler',url: 'https://www.crawlson.com/about',producer: { name: 'Crawlson', url: 'https://www.crawlson.com/' },},{- regex: 'tchelebi/[\\d.]+',+ regex: 'tchelebi',name: 'tchelebi',category: 'Crawler',url: 'https://tchelebi.io/',@@ -3277,7 +3272,7 @@ const bots = [producer: { name: 'New Work SE', url: 'https://www.xing.com/' },},{- regex: 'RepoLookoutBot/v?[\\d.]+',+ regex: 'RepoLookoutBot',name: 'Repo Lookout',category: 'Security Checker',url: 'https://www.repo-lookout.org/',@@ -3291,7 +3286,7 @@ const bots = [producer: { name: 'MAMI Project', url: 'https://mami-project.eu/' },},{- regex: 'everyfeed-spider/[\\d.]+',+ regex: 'everyfeed-spider',name: 'Everyfeed',url: 'https://web.archive.org/web/20050930235914/http://www.everyfeed.com/',category: 'Feed Fetcher',@@ -3312,7 +3307,7 @@ const bots = [producer: { name: '', url: '' },},{- regex: 'Gregarius/[\\d.]+',+ regex: 'Gregarius',name: 'Gregarius',category: 'Feed Fetcher',url: 'https://web.archive.org/web/20100614011837/http://devlog.gregarius.net/docs/ua/',@@ -3336,22 +3331,21 @@ const bots = [producer: { name: 'Sectigo Limited', url: 'https://sectigo.com/' },},{- regex:- 'KlarnaBot-(?:DownloadProductImage|EnrichProducts|PriceWatcher)/[\\d.]+',+ regex: 'KlarnaBot-(?:DownloadProductImage|EnrichProducts|PriceWatcher)',name: 'KlarnaBot',category: 'Crawler',url: 'https://docs.klarna.com/klarna-bot/',producer: { name: 'Klarna Bank AB', url: 'https://www.klarna.com/' },},{- regex: 'Taboolabot/[\\d.]+',+ regex: 'Taboolabot',name: 'Taboolabot',category: 'Crawler',url: 'https://help.taboola.com/hc/en-us/articles/115002347594-The-Taboola-Crawler',producer: { name: 'Taboola, Inc.', url: 'https://www.taboola.com/' },},{- regex: 'Asana/[\\d.]+',+ regex: 'Asana',name: 'Asana',category: 'Crawler',url: 'https://asana.com/',@@ -3365,7 +3359,7 @@ const bots = [producer: { name: 'Google Inc.', url: 'https://www.google.com/' },},{- regex: 'URLinspectorBot/[\\d.]+',+ regex: 'URLinspectorBot',name: 'URLinspector',category: 'Site Monitor',url: 'https://www.urlinspector.com/bot/',@@ -3375,14 +3369,14 @@ const bots = [},},{- regex: 'EntferBot/[\\d.]+',+ regex: 'EntferBot',name: 'Entfer',category: 'Crawler',url: 'https://entfer.com/',producer: { name: 'Entfer Ltd.', url: 'https://entfer.com/' },},{- regex: 'TagInspector/[\\d.]+',+ regex: 'TagInspector',name: 'Tag Inspector',category: 'Crawler',url: 'https://taginspector.com/',@@ -3406,46 +3400,46 @@ const bots = [},},{- regex: 'DisqusAdstxtCrawler/[\\d.]+',+ regex: 'DisqusAdstxtCrawler',name: 'Disqus',category: 'Crawler',url: 'https://help.disqus.com/en/articles/1765357-ads-txt-implementation-guide',producer: { name: 'Disqus, Inc.', url: 'https://disqus.com/' },},{- regex: 'startmebot/[\\d.]+',+ regex: 'startmebot',name: 'start.me',category: 'Crawler',url: 'https://about.start.me/',producer: { name: 'start.me BV', url: 'https://about.start.me/' },},{- regex: '2ip bot/[\\d.]+',+ regex: '2ip bot',name: '2ip',category: 'Crawler',url: 'https://2ip.io/',},{- regex: 'ReqBin Curl Client/[\\d.]+',+ regex: 'ReqBin Curl Client',name: 'ReqBin',category: 'Crawler',url: 'https://reqbin.com/curl',},{- regex: 'XoviBot/[\\d.]+',+ regex: 'XoviBot',name: 'XoviBot',category: 'Crawler',url: 'https://www.xovibot.net',producer: { name: 'Xovi GmbH', url: 'http://www.xovi.de' },},{- regex: 'Overcast/[\\d.]+ Podcast Sync',+ regex: 'Overcast/.+Podcast Sync',name: 'Overcast Podcast Sync',category: 'Service Agent',url: 'https://overcast.fm/podcasterinfo',},{- regex: '^Verity/[\\d.]+',+ regex: '^Verity',name: 'GumGum Verity',category: 'Service Agent',url: 'https://gumgum.com/verity',@@ -3457,7 +3451,7 @@ const bots = [url: 'https://github.com/snarfed/hackermention',},{- regex: 'BitSightBot/[\\d.]+',+ regex: 'BitSightBot',name: 'BitSight',category: 'Security Checker',url: 'https://www.bitsight.com/',@@ -3467,7 +3461,7 @@ const bots = [},},{- regex: 'Ezgif/[\\d.]+',+ regex: 'Ezgif',name: 'Ezgif',category: 'Service Agent',url: 'https://ezgif.com/about',@@ -3483,7 +3477,7 @@ const bots = [},},{- regex: 'FemtosearchBot/[\\d.]+',+ regex: 'FemtosearchBot',name: 'Femtosearch',category: 'Crawler',url: 'http://femtosearch.com/',@@ -3493,7 +3487,7 @@ const bots = [},},{- regex: 'AdsTxtCrawler/[\\d.]+',+ regex: 'AdsTxtCrawler/',name: 'AdsTxtCrawler',category: 'Crawler',url: 'https://github.com/InteractiveAdvertisingBureau/adstxtcrawler',@@ -3510,7 +3504,7 @@ const bots = [producer: { name: 'Morningscore', url: 'https://morningscore.io/' },},{- regex: 'Uptime-Kuma/[\\d.]+',+ regex: 'Uptime-Kuma',name: 'Uptime-Kuma',category: 'Site Monitor',url: 'https://github.com/louislam/uptime-kuma',@@ -3523,7 +3517,7 @@ const bots = [producer: { name: 'OpenAI OpCo, LLC', url: 'https://openai.com/' },},{- regex: 'GPTBot/[\\d.]+',+ regex: 'GPTBot',name: 'GPTBot',category: 'Crawler',url: 'https://platform.openai.com/docs/bots',@@ -3537,7 +3531,7 @@ const bots = [producer: { name: 'OpenAI OpCo, LLC', url: 'https://openai.com/' },},{- regex: 'BrightEdge Crawler/[\\d.]+',+ regex: 'BrightEdge Crawler',name: 'BrightEdge',category: 'Crawler',url: 'https://www.brightedge.com/',@@ -3547,7 +3541,7 @@ const bots = [},},{- regex: 'sfFeedReader/[\\d.]+',+ regex: 'sfFeedReader',name: 'sfFeedReader',url: 'https://github.com/diem-project/sfFeed2Plugin',category: 'Feed Fetcher',@@ -3570,21 +3564,21 @@ const bots = [},},{- regex: 'newspaper/[\\d.]+',+ regex: 'newspaper',name: 'Scraping Robot',category: 'Crawler',url: 'https://scrapingrobot.com/',producer: { name: 'Sprious LLC', url: 'https://sprious.com/' },},{- regex: 'Ant(?:\\.com beta|Bot)(?:/([\\d+.]+))?',+ regex: 'Ant(?:\\.com beta|Bot)',name: 'Ant',category: 'Crawler',url: 'https://www.ant.com/',producer: { name: 'Ant.com Ltd.', url: 'https://www.ant.com/' },},{- regex: 'WebwikiBot/[\\d.]+',+ regex: 'WebwikiBot',name: 'Webwiki',category: 'Crawler',url: 'https://www.webwiki.com/',@@ -3604,7 +3598,7 @@ const bots = [producer: { name: 'InnoCraft Ltd', url: 'https://matomo.org/' },},{- regex: 'Prometheus/[\\d.]+',+ regex: 'Prometheus',name: 'Prometheus',category: 'Service Agent',url: 'https://github.com/prometheus/prometheus',@@ -3618,7 +3612,7 @@ const bots = [producer: { name: 'ArchiveTeam', url: 'https://wiki.archiveteam.org/' },},{- regex: 'MADBbot/[\\d.]+',+ regex: 'MADBbot',name: 'MADBbot',category: 'Crawler',url: 'https://madb.zapto.org/bot.html',@@ -3677,7 +3671,7 @@ const bots = [producer: { name: 'Tactikast' },},{- regex: 'Brightbot ([\\d+.]+)',+ regex: 'Brightbot',name: 'BrightBot',category: 'Crawler',url: 'https://www.brightbot.app/',@@ -3687,21 +3681,21 @@ const bots = [},},{- regex: 'DaspeedBot/([\\d+.]+)',+ regex: 'DaspeedBot',name: 'DaspeedBot',category: 'Crawler',url: 'https://daspeed.io/',producer: { name: 'DAWAP SARL', url: 'https://dawap.fr/' },},{- regex: 'StractBot(?:/([\\d+.]+))?',+ regex: 'StractBot',name: 'Stract',category: 'Crawler',url: 'https://stract.com/webmasters',producer: { name: 'Stract', url: 'https://github.com/StractOrg/stract/' },},{- regex: 'GeedoBot(?:/([\\d+.]+))?',+ regex: 'GeedoBot',name: 'GeedoBot',category: 'Crawler',url: 'https://geedo.com/bot/',@@ -3713,14 +3707,14 @@ const bots = [url: 'https://geedo.com/product-search/',},{- regex: 'BackupLand(?:/([\\d+.]+))?',+ regex: 'BackupLand',name: 'BackupLand',category: 'Crawler',url: 'https://go.backupland.com/',producer: { name: 'ООО «КВАРТА»', url: 'https://go.backupland.com/' },},{- regex: 'Konturbot(?:/([\\d+.]+))?',+ regex: 'Konturbot',name: 'Konturbot',category: 'Crawler',url: 'https://kontur.ru/',@@ -3734,19 +3728,19 @@ const bots = [producer: { name: 'ООО «МОДЕСКО»', url: 'https://www.modesco.ru/' },},{- regex: 'LetsearchBot(?:/([\\d+.]+))?',+ regex: 'LetsearchBot',name: 'LetSearch',category: 'Crawler',url: 'https://letsearch.ru/bots',},{- regex: 'Example3(?:/([\\d+.]+))?',+ regex: 'Example3',name: 'Example3',category: 'Crawler',url: 'https://www.example3.com/',},{- regex: 'StatOnlineRuBot(?:/([\\d+.]+))?',+ regex: 'StatOnlineRuBot',name: 'StatOnline.ru',category: 'Crawler',url: 'https://statonline.ru/',@@ -3829,14 +3823,14 @@ const bots = [producer: { name: 'Anthropic, PBC', url: 'https://www.anthropic.com/' },},{- regex: 'NetpeakCheckerBot/[\\d.]+',+ regex: 'NetpeakCheckerBot',name: 'Netpeak Checker',category: 'Crawler',url: 'https://netpeaksoftware.com/checker',producer: { name: 'Netpeak LTD', url: 'https://netpeaksoftware.com/' },},{- regex: 'SandobaCrawler/[\\d.]+',+ regex: 'SandobaCrawler',name: 'Sandoba//Crawler',category: 'Crawler',url: 'https://www.sandoba.com/en/crawler/',@@ -3853,7 +3847,7 @@ const bots = [producer: { name: 'Sirdata SAS', url: 'https://www.sirdata.com/' },},{- regex: 'CheckMarkNetwork/[\\d.]+',+ regex: 'CheckMarkNetwork',name: 'CheckMark Network',category: 'Crawler',url: 'https://www.checkmarknetwork.com/spider.html/',@@ -3870,7 +3864,7 @@ const bots = [producer: { name: 'Cohere, Inc.', url: 'https://cohere.com/' },},{- regex: 'PerplexityBot/[\\d.]+',+ regex: 'PerplexityBot',name: 'PerplexityBot',category: 'Crawler',url: 'https://docs.perplexity.ai/docs/perplexitybot',@@ -3897,13 +3891,13 @@ const bots = [producer: { name: 'Metadot, Corp.', url: 'https://www.metadot.com/' },},{- regex: 'Ruby, Twurly v[\\d.]+',+ regex: 'Ruby, Twurly v',name: 'Twurly',category: 'Crawler',url: 'https://twurly.org/',},{- regex: 'Mixnode(?:(?:Cache)?/[\\d.]+)?',+ regex: 'Mixnode(?:Cache)?',name: 'Mixnode',category: 'Crawler',url: 'https://www.mixnode.com/',@@ -3912,9 +3906,9 @@ const bots = [url: 'https://www.mixnode.com/',},},- { regex: 'CSSCheck/[\\d.]+', name: 'CSSCheck', category: 'Validator' },+ { regex: 'CSSCheck', name: 'CSSCheck', category: 'Validator' },{- regex: 'MicrosoftPreview/[\\d.]+',+ regex: 'MicrosoftPreview',name: 'Microsoft Preview',category: 'Service Agent',url: 'https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0',@@ -3934,7 +3928,7 @@ const bots = [},},{- regex: 'TinEye/[\\d.]+',+ regex: 'TinEye',name: 'TinEye',category: 'Crawler',url: 'https://tineye.com/',@@ -3968,7 +3962,7 @@ const bots = [},},{- regex: 'online-webceo-bot/[\\d.]+',+ regex: 'online-webceo-bot',name: 'WebCEO',category: 'Crawler',url: 'https://www.webceo.com/',@@ -3988,14 +3982,14 @@ const bots = [producer: { name: 'Vistex LTD', url: 'https://www.htmlyse.com/' },},{- regex: 'TrendsmapResolver/[\\d.]+',+ regex: 'TrendsmapResolver',name: 'Trendsmap',category: 'Crawler',url: 'https://www.trendsmap.com/',producer: { name: 'Trendsmap Pty Ltd', url: 'https://www.trendsmap.com/' },},{- regex: 'Shareaholic(?:bot)?/[\\d.]+',+ regex: 'Shareaholic(?:bot)?',name: 'Steve Bot',category: 'Crawler',url: 'https://www.shareaholic.com/steve',@@ -4031,19 +4025,12 @@ const bots = [url: 'https://www.whatsmyip.org/ua/',},{- regex: 'SenutoBot/[\\d.]+',+ regex: 'SenutoBot',name: 'Senuto',category: 'Crawler',url: 'https://www.senuto.com/',producer: { name: 'Senuto Sp. z o.o.', url: 'https://www.senuto.com/' },},- {- regex: 'spaziodati',- name: 'SpazioDati',- category: 'Crawler',- url: 'https://www.spaziodati.eu/',- producer: { name: 'SpazioDati s.r.l.', url: 'https://www.spaziodati.eu/' },- },{regex: 'GozleBot',name: 'Gozle',@@ -4052,7 +4039,7 @@ const bots = [producer: { name: 'Doly Horjun HJ', url: 'https://gozle.com.tm/' },},{- regex: 'Quantcastbot/[\\d.]+',+ regex: 'Quantcastbot',name: 'Quantcast',category: 'Crawler',url: 'https://www.quantcast.com/bot/',@@ -4090,14 +4077,14 @@ const bots = [},},{- regex: 'RuxitSynthetic/[\\d.]+',+ regex: 'RuxitSynthetic',name: 'RuxitSynthetic',category: 'Site Monitor',url: 'https://community.dynatrace.com/t5/Troubleshooting/Basic-Commands-for-Synthetic/ta-p/198164',producer: { name: 'Dynatrace LLC', url: 'https://www.dynatrace.com/' },},{- regex: 'DynatraceSynthetic/[\\d.]+',+ regex: 'DynatraceSynthetic',name: 'DynatraceSynthetic',category: 'Site Monitor',url: 'https://community.dynatrace.com/t5/Troubleshooting/Basic-Commands-for-Synthetic/ta-p/198164',@@ -4111,7 +4098,7 @@ const bots = [producer: { name: 'Sitebulb Limited', url: 'https://sitebulb.com/' },},{- regex: 'Monsidobot/[\\d.]+',+ regex: 'Monsidobot',name: 'Monsidobot',category: 'Crawler',url: 'https://monsido.com/bot-html',@@ -4141,7 +4128,7 @@ const bots = [url: 'https://www.google.com/script/start/',},{- regex: 'SiteOne-Crawler/[\\d.]+',+ regex: 'SiteOne-Crawler',name: 'SiteOne Crawler',category: 'Crawler',url: 'https://crawler.siteone.io/bot/',@@ -4165,7 +4152,7 @@ const bots = [},},{- regex: 'Paqlebot/[\\d.]+',+ regex: 'Paqlebot',name: 'Paqlebot',category: 'Crawler',url: 'https://www.paqle.dk/about/paqlebot',@@ -4184,7 +4171,7 @@ const bots = [url: 'https://github.com/matrix-org/synapse',},{- regex: 'OSZKbot/[\\d.]+',+ regex: 'OSZKbot',name: 'OSZKbot',category: 'Crawler',url: 'http://mekosztaly.oszk.hu/mia/',@@ -4201,7 +4188,7 @@ const bots = [producer: { name: 'SEO Cube S.r.l.', url: 'https://www.seocube.it/' },},{- regex: 'RavenCrawler/[\\d.]+',+ regex: 'RavenCrawler',name: 'RavenCrawler',category: 'Crawler',url: 'https://raventools.com/site-auditor/',@@ -4215,14 +4202,14 @@ const bots = [producer: { name: 'Kadolijst', url: 'https://www.kadolijst.nl/' },},{- regex: 'Dubbotbot/[\\d.]+',+ regex: 'Dubbotbot',name: 'Dubbotbot',category: 'Crawler',url: 'https://help.dubbot.com/en/articles/6746594-example-custom-user-agent',producer: { name: 'DubBot', url: 'https://dubbot.com/' },},{- regex: 'Swiftbot/[\\d.]+',+ regex: 'Swiftbot',name: 'Swiftbot',category: 'Crawler',url: 'https://swiftype.com/swiftbot',@@ -4242,13 +4229,9 @@ const bots = [url: 'https://www.thousandeyes.com/',producer: { name: 'Cisco Systems, Inc.', url: 'https://www.cisco.com/' },},- { regex: 'OmtrBot/[\\d.]+', name: 'OmtrBot', category: 'Site Monitor' },- { regex: 'WebMon/[\\d.]+', name: 'WebMon', category: 'Site Monitor' },- {- regex: 'AdsTxtCrawlerTP/[\\d.]+',- name: 'AdsTxtCrawlerTP',- category: 'Crawler',- },+ { regex: 'OmtrBot', name: 'OmtrBot', category: 'Site Monitor' },+ { regex: 'WebMon', name: 'WebMon', category: 'Site Monitor' },+ { regex: 'AdsTxtCrawlerTP', name: 'AdsTxtCrawlerTP', category: 'Crawler' },{regex: 'fragFINN',name: 'fragFINN',@@ -4264,7 +4247,7 @@ const bots = [producer: { name: 'Clickagy, LLC', url: 'https://www.clickagy.com/' },},{- regex: 'kiwitcms-gitops/[\\d.]+',+ regex: 'kiwitcms-gitops',name: 'Kiwi TCMS GitOps',category: 'Service Agent',url: 'https://kiwitcms.org',@@ -4333,14 +4316,14 @@ const bots = [},},{- regex: 'Uptimia(?:/[\\d.]+)?',+ regex: 'Uptimia',name: 'Uptimia',category: 'Site Monitor',url: 'https://www.uptimia.com/',producer: { name: 'JJ Online GmbH', url: 'https://www.uptimia.com/' },},{- regex: '2GDPR/[\\d.]+',+ regex: '2GDPR',name: '2GDPR',category: 'Service Agent',url: 'https://2gdpr.com/tos',@@ -4361,7 +4344,7 @@ const bots = [producer: { name: 'CheckHost', url: 'https://check-host.net/' },},{- regex: 'LAC_IAHarvester/[\\d.]+',+ regex: 'LAC_IAHarvester',name: 'LAC IA Harvester',category: 'Crawler',url: 'https://library-archives.canada.ca/eng/services/government-canada/web-social-media-preservation-program/Pages/web-archive.aspx',@@ -4371,7 +4354,7 @@ const bots = [},},{- regex: 'InsytfulBot/[\\d.]+',+ regex: 'InsytfulBot',name: 'InsytfulBot',category: 'Crawler',url: 'https://www.insytful.com/',@@ -4385,7 +4368,7 @@ const bots = [producer: { name: 'Statista, Inc.', url: 'https://www.statista.com/' },},{- regex: 'SubstackContentFetch/[\\d.]+',+ regex: 'SubstackContentFetch',name: 'Substack Content Fetch',category: 'Crawler',url: 'https://substack.com/',@@ -4416,7 +4399,7 @@ const bots = [producer: { name: 'Tenable, Inc.', url: 'https://www.tenable.com/' },},{- regex: 'Castopod/[\\d.]+',+ regex: 'Castopod',name: 'Castopod',category: 'Crawler',url: 'https://www.castopod.org/',@@ -4429,7 +4412,7 @@ const bots = [producer: { name: 'Elasticsearch B.V.', url: 'https://www.elastic.co/' },},{- regex: 'WDG_Validator/[\\d.]+',+ regex: 'WDG_Validator',name: 'WDG HTML Validator',category: 'Validator',url: 'http://www.htmlhelp.com/tools/validator/',@@ -4441,7 +4424,7 @@ const bots = [url: 'https://web.archive.org/web/20180910002802/http://www.aegis.network/',},{- regex: 'CrawlyProjectCrawler/[\\d.]+',+ regex: 'CrawlyProjectCrawler',name: 'Crawly Project',category: 'Crawler',url: 'https://web.archive.org/web/20240326141952/https://crawlyproject.digitaldragon.dev/',@@ -4459,20 +4442,20 @@ const bots = [url: 'https://github.com/openeasm/punkmap',},{- regex: 'GenomeCrawlerd/[\\d.]+',+ regex: 'GenomeCrawlerd',name: 'Deepfield Genome',category: 'Crawler',url: 'https://www.nokia.com/networks/ip-networks/deepfield/genome/',producer: { name: 'Nokia Corporation', url: 'https://www.nokia.com/' },},{- regex: 'Gaisbot/[\\d.]+',+ regex: 'Gaisbot',name: 'Gaisbot',category: 'Crawler',url: 'https://web.archive.org/web/20090604121511/https://gais.cs.ccu.edu.tw/robot.php',},{- regex: 'FAST-WebCrawler/[\\d.]+',+ regex: 'FAST-WebCrawler',name: 'AlltheWeb',category: 'Crawler',url: 'https://web.archive.org/web/20041020050801/http://www.alltheweb.com/help/webmaster/crawler',@@ -4484,7 +4467,7 @@ const bots = [url: 'https://ducks.party/',},{- regex: 'DepSpid/[\\d.]+',+ regex: 'DepSpid',name: 'DepSpid',category: 'Crawler',url: 'https://web.archive.org/web/20080321224033/http://about.depspid.net/',@@ -4589,7 +4572,7 @@ const bots = [producer: { name: 'Marginalia', url: 'https://www.marginalia.nu/' },},{- regex: 'vu-server-health-scanner/[\\d.]+',+ regex: 'vu-server-health-scanner',name: 'VU Server Health Scanner',category: 'Security Checker',url: 'https://130.37.198.75/index.html',@@ -4749,9 +4732,150 @@ const bots = [category: 'Crawler',url: 'https://github.com/nettrom/suggestbot',},+ {+ regex: 'cms-experiment',+ name: 'CMS Experiment',+ category: 'Security Checker',+ url: 'https://securitee.org/cms-experiment-fall2024/',+ },+ {+ regex: 'SiteCheckerBotCrawler',+ name: 'SiteCheckerBotCrawler',+ category: 'Crawler',+ url: 'https://sitechecker.pro/',+ producer: { name: 'Cyber Circus Limited', url: 'https://sitechecker.pro/' },+ },+ {+ regex: 'SBIder',+ name: 'SBIder',+ category: 'Crawler',+ url: 'https://www.sitesell.com/sbider.html',+ producer: { name: 'SiteSell Inc.', url: 'https://www.sitesell.com/' },+ },+ {+ regex: 'LightspeedSystemsCrawler',+ name: 'LightspeedSystemsCrawler',+ category: 'Crawler',+ url: 'https://www.lightspeedsystems.com/',+ producer: {+ name: 'Lightspeed Systems, Inc.',+ url: 'https://www.lightspeedsystems.com/',+ },+ },+ {+ regex: 'Research JLU',+ name: 'Research JLU',+ category: 'Crawler',+ url: 'https://www.uni-giessen.de/en/research',+ producer: {+ name: 'Justus Liebig University Giessen',+ url: 'https://www.uni-giessen.de/en',+ },+ },+ {+ regex: '(?:hgf|OS)AlphaXCrawl',+ name: 'AlphaXCrawl',+ category: 'Crawler',+ url: 'https://www.fim.uni-passau.de/en/data-science/research/open-search',+ producer: {+ name: 'University of Passau',+ url: 'https://www.uni-passau.de/en/',+ },+ },+ {+ regex: 'WPMU DEV',+ name: 'WPMU DEV',+ category: 'Crawler',+ url: 'https://wpmudev.com/docs/wpmu-dev-plugins/broken-link-checker/#broken-link-checker-user-agent',+ producer: { name: 'Incsub, LLC.', url: 'https://incsub.com/' },+ },+ {+ regex: 'SnoopSecInspect',+ name: 'SnoopSecInspect',+ category: 'Security Checker',+ url: 'https://web.archive.org/web/20241206193253/https://snoopsec.us.to/',+ },+ {+ regex: 'ModatScanner',+ name: 'ModatScanner',+ category: 'Security Checker',+ url: 'https://www.modat.io/scanning',+ producer: { name: 'Modat B.V.', url: 'https://www.modat.io/' },+ },+ {+ regex: 'researchcyber\\.net',+ name: 'researchcyber.net',+ category: 'Security Checker',+ url: 'https://web.archive.org/web/20241219082407/https://researchcyber.net/',+ },+ {+ regex: 'CrystalSemanticsBot',+ name: 'CrystalSemanticsBot',+ category: 'Crawler',+ url: 'https://web.archive.org/web/20121230203310/http://www.crystalsemantics.com/user-agent/',+ producer: {+ name: 'Crystal Semantics Ltd.',+ url: 'https://web.archive.org/web/20121029062239/http://www.crystalsemantics.com/',+ },+ },+ {+ regex: 'najdu\\.s\\.holubem\\.eu',+ name: 'najdu.s.holubem.eu',+ category: 'Crawler',+ url: 'https://najdu.s.holubem.eu/',+ },+ {+ regex: 'VORTEX/',+ name: 'VORTEX',+ category: 'Crawler',+ url: 'https://marty.anstey.ca/robots/vortex',+ },+ {+ regex: 'xtate/(\\d+\\.[.\\d]+)',+ name: 'xtate',+ category: 'Crawler',+ url: 'https://github.com/babycoff/xtate',+ },+ {+ regex: 'FediList Agent/',+ name: 'FediList',+ category: 'Social Media Agent',+ url: 'https://fedilist.com/',+ },+ {+ regex: 'Grafana/(\\d+\\.[.\\d]+)',+ name: 'Grafana',+ category: 'Site Monitor',+ url: 'https://github.com/grafana/grafana',+ producer: { name: 'Grafana Labs', url: 'https://grafana.com/' },+ },+ {+ regex: 'github-camo',+ name: 'Github Camo',+ category: 'Crawler',+ url: 'https://github.com/atmos/camo',+ producer: {+ name: 'Github',+ url: 'https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/about-anonymized-urls',+ },+ },+ {+ regex: 'Bluesky',+ name: 'Bluesky',+ category: 'Social Media Agent',+ url: 'https://bsky.app',+ producer: { name: 'Bluesky Social PBC', url: 'https://bsky.app' },+ },+ {+ regex: 'OpenGraph\\.io',+ name: 'OpenGraph.io',+ category: 'Crawler',+ url: 'https://www.opengraph.io',+ producer: { name: 'OpenGraph.io', url: 'https://www.opengraph.io' },+ },{regex:- 'nuhk|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr\\.com|teoma|oegp|http%20client|htdig|mogimogi|larbin|scrubby|searchsight|semanticdiscovery|snappy|vortex(?!(?: Build|Plus| CM62| HD65))|zeal(?!ot)|dataparksearch|findlinks|BrowserMob|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|7Siters|centuryb\\.o\\.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|cortex|Re-re Studio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle Client|Hello,? world|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M CODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux Gnu \\(cow\\)|Test Certificate Info|iplabel|Magellan|TheSafex?Internetx?Search|Searcherx?web|kirkland-signature|LinkChain|survey-security-dot-txt|infrawatch|Time/|r00ts3c-owned-you|nvdorz|Root Slut|NiggaBalls|BotPoke|GlobalWebSearch|xx032_bo9vs83_2a|sslshed|geckotrail|Wordup|Keydrop|^xenu|^(?:chrome|firefox|Abcd|Dark|KvshClient|Node.js|Report Runner|url|Zeus|ZmEu)$',+ 'nuhk|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr\\.com|teoma|oegp|http%20client|htdig|mogimogi|larbin|scrubby|searchsight|semanticdiscovery|snappy|zeal(?!ot)|dataparksearch|findlinks|BrowserMob|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|7Siters|centuryb\\.o\\.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|cortex|Re-re Studio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle Client|Hello,? world|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M CODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux Gnu \\(cow\\)|Test Certificate Info|iplabel|Magellan|TheSafex?Internetx?Search|Searcherx?web|kirkland-signature|LinkChain|survey-security-dot-txt|infrawatch|Time/|r00ts3c-owned-you|nvdorz|Root Slut|NiggaBalls|BotPoke|GlobalWebSearch|xx032_bo9vs83_2a|sslshed|geckotrail|Wordup|Keydrop|\\(compatible\\)|John Recon|SPARK COMMIT|masjesu|Komaru_The_Cat|Jesus Christ of Nazareth is LORD|Kowai|Hakai|LoliSec|LMAO|^xenu|^(?:chrome|firefox|Abcd|Dark|KvshClient|Node.js|Report Runner|url|Zeus|ZmEu)$|OnlyScans|TheInternetSearchx',name: 'Generic Bot',},{diff --git a/apps/worker/src/referrers/index.ts b/apps/worker/src/referrers/index.tsindex 6d1734d3f..bb1de3d15 100644--- a/apps/worker/src/referrers/index.ts+++ b/apps/worker/src/referrers/index.ts@@ -55,7 +55,7 @@ const referrers: Record<string, { type: string; name: string }> = {'lo.st': { type: 'search', name: 'Lo.st' },'www1.dastelefonbuch.de': { type: 'search', name: 'DasTelefonbuch' },'www.fireball.de': { type: 'search', name: 'Fireball' },- 'search.1und1.de': { type: 'search', name: '1und1' },+ 'suche.1und1.de': { type: 'search', name: '1und1' },'ricerca.virgilio.it': { type: 'search', name: 'Virgilio' },'ricercaimmagini.virgilio.it': { type: 'search', name: 'Virgilio' },'ricercavideo.virgilio.it': { type: 'search', name: 'Virgilio' },@@ -1162,6 +1162,7 @@ const referrers: Record<string, { type: string; name: string }> = {'clusty.com': { type: 'search', name: 'InfoSpace' },'www.weborama.com': { type: 'search', name: 'Weborama' },'search.bluewin.ch': { type: 'search', name: 'Bluewin' },+ 'search.brave.com': { type: 'search', name: 'Brave' },'search.bt.com': { type: 'search', name: 'British Telecommunications' },'www.neti.ee': { type: 'search', name: 'Neti' },'nigma.ru': { type: 'search', name: 'Nigma' },@@ -1206,6 +1207,8 @@ const referrers: Record<string, { type: string; name: string }> = {'www.toile.com': { type: 'search', name: 'La Toile Du Quebec Via Google' },'web.toile.com': { type: 'search', name: 'La Toile Du Quebec Via Google' },'www.paperball.de': { type: 'search', name: 'Paperball' },+ 'arianna.libero.it': { type: 'search', name: 'Arianna' },+ 'www.arianna.com': { type: 'search', name: 'Arianna' },'www.stepstone.de': { type: 'search', name: 'StepStone' },'www.stepstone.at': { type: 'search', name: 'StepStone' },'www.stepstone.be': { type: 'search', name: 'StepStone' },@@ -1496,7 +1499,7 @@ const referrers: Record<string, { type: string; name: string }> = {'www.walhello.com': { type: 'search', name: 'Walhello' },'www.walhello.de': { type: 'search', name: 'Walhello' },'www.walhello.nl': { type: 'search', name: 'Walhello' },- 'meta.ua': { type: 'search', name: 'Meta' },+ 'www.startsiden.no': { type: 'search', name: 'Startsiden' },'www.skynet.be': { type: 'search', name: 'Skynet' },'www.searchy.co.uk': { type: 'search', name: 'Searchy' },'search.findwide.com': { type: 'search', name: 'Findwide' },@@ -1518,7 +1521,6 @@ const referrers: Record<string, { type: string; name: string }> = {'search.lilo.org': { type: 'search', name: 'Lilo' },'search.naver.com': { type: 'search', name: 'Naver' },'www.zoeken.nl': { type: 'search', name: 'Zoeken' },- 'www.startsiden.no': { type: 'search', name: 'Startsiden' },'search.yam.com': { type: 'search', name: 'Yam' },'www.eniro.se': { type: 'search', name: 'Eniro' },'apollo7.de': { type: 'search', name: 'APOLL07' },@@ -1569,6 +1571,7 @@ const referrers: Record<string, { type: string; name: string }> = {'suche.gmx.net': { type: 'search', name: 'GMX' },'daemon-search.com': { type: 'search', name: 'Daemon search' },'my.daemon-search.com': { type: 'search', name: 'Daemon search' },+ 'meta.ua': { type: 'search', name: 'Meta.ua' },'so.m.sm.cn': { type: 'search', name: 'Shenma' },'yz.m.sm.cn': { type: 'search', name: 'Shenma' },'m.sm.cn': { type: 'search', name: 'Shenma' },@@ -1938,8 +1941,7 @@ const referrers: Record<string, { type: string; name: string }> = {'jyxo.1188.cz': { type: 'search', name: 'Jyxo' },'www.kataweb.it': { type: 'search', name: 'Kataweb' },'busca.uol.com.br': { type: 'search', name: 'uol.com.br' },- 'arianna.libero.it': { type: 'search', name: 'Arianna' },- 'www.arianna.com': { type: 'search', name: 'Arianna' },+ 'websearch.rakuten.co.jp': { type: 'search', name: 'Rakuten' },'www.mamma.com': { type: 'search', name: 'Mamma' },'mamma75.mamma.com': { type: 'search', name: 'Mamma' },'www.yatedo.com': { type: 'search', name: 'Yatedo' },@@ -1947,7 +1949,6 @@ const referrers: Record<string, { type: string; name: string }> = {'www.twingly.com': { type: 'search', name: 'Twingly' },'smart.delfi.lv': { type: 'search', name: 'Delfi latvia' },'www.pricerunner.co.uk': { type: 'search', name: 'PriceRunner' },- 'websearch.rakuten.co.jp': { type: 'search', name: 'Rakuten' },'www.google.com': { type: 'search', name: 'Google' },'www.google.ac': { type: 'search', name: 'Google' },'www.google.ad': { type: 'search', name: 'Google' },@@ -2395,8 +2396,10 @@ const referrers: Record<string, { type: string; name: string }> = {'email.telstra.com': { type: 'email', name: 'Bigpond' },'basic.messaging.bigpond.com': { type: 'email', name: 'Bigpond' },'mail.naver.com': { type: 'email', name: 'Naver Mail' },+ 'email.t-online.de': { type: 'email', name: 'T-online' },'mail.zoho.com': { type: 'email', name: 'Zoho' },- 'webmail.virginbroadband.com.au': { type: 'email', name: 'Virgin' },+ 'mail.163.com': { type: 'email', name: '163 Mail' },+ 'webmail.tim.it': { type: 'email', name: 'TIM' },'mail.yahoo.net': { type: 'email', name: 'Yahoo! Mail' },'mail.yahoo.com': { type: 'email', name: 'Yahoo! Mail' },'mail.yahoo.co.uk': { type: 'email', name: 'Yahoo! Mail' },@@ -2409,10 +2412,18 @@ const referrers: Record<string, { type: string; name: string }> = {'mail.iinet.net.au': { type: 'email', name: 'iiNet' },'mail.e1.ru': { type: 'email', name: 'E1.ru' },'webmail.vodafone.co.nz': { type: 'email', name: 'Vodafone' },+ 'mail.vodafone.de': { type: 'email', name: 'Vodafone' },+ 'deref-1und1-02.de': { type: 'email', name: '1und1' },+ 'webmail.dodo.com.au': { type: 'email', name: 'Dodo' },'mail.126.com': { type: 'email', name: '126 Mail' },'com.mailchimp.mailchimp': { type: 'email', name: 'Mailchimp' },'inbox.com': { type: 'email', name: 'Inbox.com' },'webmail.iprimus.com.au': { type: 'email', name: 'iPrimus' },+ 'deref-web.de': { type: 'email', name: 'Web.de' },+ '3c.web.de': { type: 'email', name: 'Web.de' },+ '3c-bap.web.de': { type: 'email', name: 'Web.de' },+ 'lightmailer-bap.web.de': { type: 'email', name: 'Web.de' },+ 'lightmailer-bs.web.de': { type: 'email', name: 'Web.de' },'mail.qq.com': { type: 'email', name: 'QQ Mail' },'exmail.qq.com': { type: 'email', name: 'QQ Mail' },'mail.qip.ru': { type: 'email', name: 'QIP' },@@ -2423,30 +2434,79 @@ const referrers: Record<string, { type: string; name: string }> = {'mail.live.com': { type: 'email', name: 'Outlook.com' },'outlook.live.com': { type: 'email', name: 'Outlook.com' },'com.microsoft.office.outlook': { type: 'email', name: 'Outlook.com' },- 'webmail.dodo.com.au': { type: 'email', name: 'Dodo' },- 'webmail.2degreesbroadband.co.nz': { type: 'email', name: '2degrees' },+ 'deref-mail.com': { type: 'email', name: 'Mail.com' },+ '3c-lxa.mail.com': { type: 'email', name: 'Mail.com' },+ 'lightmailer.mail.com': { type: 'email', name: 'Mail.com' },'mail2.daum.net': { type: 'email', name: 'Daum Mail' },'mail.daum.net': { type: 'email', name: 'Daum Mail' },+ 'upcmail.hispeed.ch': { type: 'email', name: 'UPC' },+ 'webmail.2degreesbroadband.co.nz': { type: 'email', name: '2degrees' },'post.ru': { type: 'email', name: 'Beeline' },- 'e.mail.ru': { type: 'email', name: 'Mail.ru' },- 'touch.mail.ru': { type: 'email', name: 'Mail.ru' },+ 'mail.infomaniak.com': { type: 'email', name: 'Infomaniak' },'webmail.adam.com.au': { type: 'email', name: 'Adam Internet' },'orange.fr/webmail': { type: 'email', name: 'Orange Webmail' },+ 'mail01.orange.fr': { type: 'email', name: 'Orange Webmail' },+ 'mail02.orange.fr': { type: 'email', name: 'Orange Webmail' },+ 'wmail.orange.fr': { type: 'email', name: 'Orange Webmail' },+ 'messageriepro3.orange.fr': { type: 'email', name: 'Orange Webmail' },+ 'messagerie.orange.fr': { type: 'email', name: 'Orange Webmail' },+ 'email.ionos.de': { type: 'email', name: 'Ionos' },+ 'email.ionos.es': { type: 'email', name: 'Ionos' },+ 'email.ionos.fr': { type: 'email', name: 'Ionos' },+ 'email.ionos.it': { type: 'email', name: 'Ionos' },+ 'email.ionos.ca': { type: 'email', name: 'Ionos' },+ 'email.ionos.mx': { type: 'email', name: 'Ionos' },+ 'email.ionos.com': { type: 'email', name: 'Ionos' },+ 'email.ionos.co.uk': { type: 'email', name: 'Ionos' },+ 'mailbusiness.ionos.de': { type: 'email', name: 'Ionos' },+ 'mailbusiness.ionos.es': { type: 'email', name: 'Ionos' },+ 'mailbusiness.ionos.fr': { type: 'email', name: 'Ionos' },+ 'mailbusiness.ionos.it': { type: 'email', name: 'Ionos' },+ 'mailbusiness.ionos.ca': { type: 'email', name: 'Ionos' },+ 'mailbusiness.ionos.mx': { type: 'email', name: 'Ionos' },+ 'mailbusiness.ionos.com': { type: 'email', name: 'Ionos' },+ 'mailbusiness.ionos.co.uk': { type: 'email', name: 'Ionos' },'com.earthlink.myearthlink': { type: 'email', name: 'earthlink' },+ 'rich-v01.bluewin.ch': { type: 'email', name: 'Bluewin' },+ 'rich-v02.bluewin.ch': { type: 'email', name: 'Bluewin' },+ 'email.bluewin.ch': { type: 'email', name: 'Bluewin' },'mail.aol.com': { type: 'email', name: 'AOL Mail' },'com.aol.mobile.aolapp': { type: 'email', name: 'AOL Mail' },'webmail.netspace.net.au': { type: 'email', name: 'Netspace' },'webmail.optuszoo.com.au': { type: 'email', name: 'Optus Zoo' },'webmail.optusnet.com.au': { type: 'email', name: 'Optus Zoo' },+ 'webmail.virginbroadband.com.au': { type: 'email', name: 'Virgin' },+ 'mail.proton.me': { type: 'email', name: 'Proton' },'webmail.commander.net.au': { type: 'email', name: 'Commander' },'mastermail.ru': { type: 'email', name: 'Mastermail' },'m.mastermail.ru': { type: 'email', name: 'Mastermail' },+ 'deref-gmx.de': { type: 'email', name: 'GMX' },+ 'deref-gmx.at': { type: 'email', name: 'GMX' },+ 'deref-gmx.ch': { type: 'email', name: 'GMX' },+ 'deref-gmx.fr': { type: 'email', name: 'GMX' },+ 'deref-gmx.es': { type: 'email', name: 'GMX' },+ 'deref-gmx.it': { type: 'email', name: 'GMX' },+ 'deref-gmx.com': { type: 'email', name: 'GMX' },+ 'deref-gmx.net': { type: 'email', name: 'GMX' },+ 'deref-gmx.co.uk': { type: 'email', name: 'GMX' },+ 'lightmailer.gmx.de': { type: 'email', name: 'GMX' },+ 'lightmailer.gmx.at': { type: 'email', name: 'GMX' },+ 'lightmailer.gmx.ch': { type: 'email', name: 'GMX' },+ 'lightmailer.gmx.fr': { type: 'email', name: 'GMX' },+ 'lightmailer.gmx.es': { type: 'email', name: 'GMX' },+ 'lightmailer.gmx.it': { type: 'email', name: 'GMX' },+ 'lightmailer.gmx.com': { type: 'email', name: 'GMX' },+ 'lightmailer.gmx.net': { type: 'email', name: 'GMX' },+ 'lightmailer.gmx.co.uk': { type: 'email', name: 'GMX' },+ 'lightmailer-bs.gmx.net': { type: 'email', name: 'GMX' },+ 'lightmailer-bap.gmx.net': { type: 'email', name: 'GMX' },'mail.yandex.ru': { type: 'email', name: 'Yandex' },'mail.yandex.com': { type: 'email', name: 'Yandex' },'mail.yandex.kz': { type: 'email', name: 'Yandex' },'mail.yandex.ua': { type: 'email', name: 'Yandex' },'mail.yandex.by': { type: 'email', name: 'Yandex' },- 'mail.163.com': { type: 'email', name: '163 Mail' },+ 'e.mail.ru': { type: 'email', name: 'Mail.ru' },+ 'touch.mail.ru': { type: 'email', name: 'Mail.ru' },'mail.ukr.net': { type: 'email', name: 'Ukr.net' },'mail.rambler.ru': { type: 'email', name: 'Rambler' },'mail.mynet.com': { type: 'email', name: 'Mynet Mail' },@@ -2509,9 +2569,14 @@ const referrers: Record<string, { type: string; name: string }> = {'www.googleadservices.com': { type: 'paid', name: 'Google' },'partner.googleadservices.com': { type: 'paid', name: 'Google' },'googleads.g.doubleclick.net': { type: 'paid', name: 'Google' },+ 'tdsf.doubleclick.net': { type: 'paid', name: 'Google' },'tpc.googlesyndication.com': { type: 'paid', name: 'Google' },+ 'safeframe.googlesyndication.com': { type: 'paid', name: 'Google' },'googleadservices.com': { type: 'paid', name: 'Google' },'imasdk.googleapis.com': { type: 'paid', name: 'Google' },+ 'www.adsensecustomsearchads.com': { type: 'paid', name: 'Google' },+ 'syndicatedsearch.goog': { type: 'paid', name: 'Google' },+ 'pagead2.googlesyndication.com': { type: 'paid', name: 'Google' },'eyeota.net': { type: 'paid', name: 'Eyeota' },'price.ru': { type: 'paid', name: 'Price.ru' },'v.price.ru': { type: 'paid', name: 'Price.ru' },@@ -2544,8 +2609,7 @@ const referrers: Record<string, { type: string; name: string }> = {'sonico.com': { type: 'social', name: 'Sonico.com' },'odnoklassniki.ru': { type: 'social', name: 'Odnoklassniki' },'ok.ru': { type: 'social', name: 'Odnoklassniki' },- 'tildes.net': { type: 'social', name: 'Tildes' },- 'com.talklittle.android.tildes': { type: 'social', name: 'Tildes' },+ 'github.com': { type: 'tech', name: 'GitHub' },'classmates.com': { type: 'social', name: 'Classmates' },'friendsreunited.com': { type: 'social', name: 'Friends Reunited' },'news.ycombinator.com': { type: 'social', name: 'Hacker News' },@@ -2556,9 +2620,12 @@ const referrers: Record<string, { type: string; name: string }> = {'orkut.com': { type: 'social', name: 'Orkut' },'myheritage.com': { type: 'social', name: 'MyHeritage' },'multiply.com': { type: 'social', name: 'Multiply' },- 'threads.net': { type: 'social', name: 'Threads' },- 'l.threads.net': { type: 'social', name: 'Threads' },- 'com.instagram.barcelona': { type: 'social', name: 'Threads' },+ 'facebook.com': { type: 'social', name: 'Facebook' },+ 'fb.me': { type: 'social', name: 'Facebook' },+ 'm.facebook.com': { type: 'social', name: 'Facebook' },+ 'l.facebook.com': { type: 'social', name: 'Facebook' },+ 'lm.facebook.com': { type: 'social', name: 'Facebook' },+ 'com.facebook.katana': { type: 'social', name: 'Facebook' },'myyearbook.com': { type: 'social', name: 'myYearbook' },'renren.com': { type: 'social', name: 'Renren' },'app.slack.com': { type: 'social', name: 'Slack' },@@ -2632,15 +2699,13 @@ const referrers: Record<string, { type: string; name: string }> = {'douban.com': { type: 'social', name: 'Douban' },'login.live.com': { type: 'social', name: 'Windows Live Spaces' },'blackplanet.com': { type: 'social', name: 'BlackPlanet' },+ 'lnk.bio': { type: 'social', name: 'Lnk.Bio' },'global.cyworld.com': { type: 'social', name: 'Cyworld' },'getpocket.com': { type: 'social', name: 'Pocket' },'skyrock.com': { type: 'social', name: 'Skyrock' },- 'facebook.com': { type: 'social', name: 'Facebook' },- 'fb.me': { type: 'social', name: 'Facebook' },- 'm.facebook.com': { type: 'social', name: 'Facebook' },- 'l.facebook.com': { type: 'social', name: 'Facebook' },- 'lm.facebook.com': { type: 'social', name: 'Facebook' },- 'com.facebook.katana': { type: 'social', name: 'Facebook' },+ 'threads.net': { type: 'social', name: 'Threads' },+ 'l.threads.net': { type: 'social', name: 'Threads' },+ 'com.instagram.barcelona': { type: 'social', name: 'Threads' },'web.whatsapp.com': { type: 'social', name: 'WhatsApp' },'com.whatsapp': { type: 'social', name: 'WhatsApp' },'redirect.disqus.com': { type: 'social', name: 'Disqus' },@@ -2663,7 +2728,10 @@ const referrers: Record<string, { type: string; name: string }> = {'com.laurencedawson.reddit_sync': { type: 'social', name: 'Reddit' },'com.laurencedawson.reddit_sync.pro': { type: 'social', name: 'Reddit' },'viadeo.com': { type: 'social', name: 'Viadeo' },- 'github.com': { type: 'tech', name: 'GitHub' },+ 'tildes.net': { type: 'social', name: 'Tildes' },+ 'com.talklittle.android.tildes': { type: 'social', name: 'Tildes' },+ 'l.workplace.com': { type: 'social', name: 'Workplace' },+ 'lm.workplace.com': { type: 'social', name: 'Workplace' },'stackoverflow.com': { type: 'tech', name: 'Stack Overflow' },'gaiaonline.com': { type: 'social', name: 'Gaia Online' },'stumbleupon.com': { type: 'social', name: 'StumbleUpon' },diff --git a/package.json b/package.jsonindex f6e226de8..8ce7dade7 100644--- a/package.json+++ b/package.json@@ -7,6 +7,8 @@"packageManager": "[email protected]","scripts": {"test": "vitest",+ "gen:bots": "pnpm -r --filter api gen:bots",+ "gen:referrers": "pnpm -r --filter worker gen:referrers","dock:up": "docker compose up -d","dock:down": "docker compose down","dock:ch": "docker compose exec -it op-ch clickhouse-client -d openpanel",