OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
chore(api): update bots and referrers
Carl-Gerhard Lindesvärdf9a0025c7210
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 209e74ccd..dff2255d8 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -6,6 +6,8 @@"testing": "API_PORT=3333 pnpm dev","start": "node dist/index.js","build": "rm -rf dist && tsup",+ "gen:referrers": "jiti scripts/get-referrers.ts && biome format --write src/referrers/index.ts",+ "gen:bots": "jiti scripts/get-bots.ts && biome format --write src/bots/bots.ts","typecheck": "tsc --noEmit"},"dependencies": {@@ -16,8 +18,8 @@"@fastify/websocket": "^8.3.1","@openpanel/common": "workspace:*","@openpanel/db": "workspace:*",- "@openpanel/logger": "workspace:*","@openpanel/integrations": "workspace:^",+ "@openpanel/logger": "workspace:*","@openpanel/queue": "workspace:*","@openpanel/redis": "workspace:*","@openpanel/trpc": "workspace:*",@@ -33,7 +35,6 @@"sqlstring": "^2.3.3","superjson": "^1.13.3","svix": "^1.24.0",- "ua-parser-js": "^1.0.37","url-metadata": "^4.1.0","uuid": "^9.0.1","zod": "^3.22.4"@@ -42,13 +43,14 @@"@faker-js/faker": "^9.0.1","@openpanel/sdk": "workspace:*","@openpanel/tsconfig": "workspace:*",+ "@types/js-yaml": "^4.0.9","@types/jsonwebtoken": "^9.0.6","@types/ramda": "^0.29.6","@types/request-ip": "^0.0.41","@types/sqlstring": "^2.3.2",- "@types/ua-parser-js": "^0.7.39","@types/uuid": "^9.0.8","@types/ws": "^8.5.10",+ "js-yaml": "^4.1.0","tsup": "^7.2.0","typescript": "^5.2.2"}diff --git a/apps/api/scripts/get-bots.ts b/apps/api/scripts/get-bots.tsnew file mode 100644index 000000000..f72561ffd--- /dev/null+++ b/apps/api/scripts/get-bots.ts@@ -0,0 +1,29 @@+import fs from 'node:fs';+import path from 'node:path';+import yaml from 'js-yaml';++async function main() {+ // Get document, or throw exception on error+ try {+ const data = await fetch(+ 'https://raw.githubusercontent.com/matomo-org/device-detector/master/regexes/bots.yml',+ ).then((res) => res.text());++ fs.writeFileSync(+ path.resolve(__dirname, '../src/bots/bots.ts'),+ [+ '// This file is generated by the script get-bots.ts',+ '',+ '// The data is fetch from device-detector https://raw.githubusercontent.com/matomo-org/device-detector/master/regexes/bots.yml',+ '',+ `const bots = ${JSON.stringify(yaml.load(data))} as const;`,+ 'export default bots;',+ ].join('\n'),+ 'utf-8',+ );+ } catch (e) {+ console.log(e);+ }+}++main();diff --git a/apps/api/scripts/get-organizations.ts b/apps/api/scripts/get-organizations.tsdeleted file mode 100644index 427c4a4ef..000000000--- a/apps/api/scripts/get-organizations.ts+++ /dev/null@@ -1,189 +0,0 @@-// import { clerkClient } from '@clerk/fastify';--import { db } from '@openpanel/db';--// import { db } from '@openpanel/db';--// type Fn<T = unknown> = (args: { limit: number; offset: number }) => Promise<{-// data: T[];-// totalCount: number;-// }>;--// function getAllDataByPagination<T extends Fn>(-// cb: T-// ): Promise<Awaited<ReturnType<T>>['data']> {-// const data: Awaited<ReturnType<T>>['data'] = [];-// async function getData(page = 0) {-// console.log(`getData with offset ${page * 100}`);-// const response = await cb({-// limit: 100,-// offset: page * 100,-// });-// if (response.data.length !== 0) {-// data.push(...response.data);-// await getData(page + 1);-// }-// await new Promise((resolve) => setTimeout(resolve, 100));-// }--// return getData().then(() => data);-// }--// async function main() {-// const organizations = await getAllDataByPagination(-// clerkClient.organizations.getOrganizationList.bind(-// clerkClient.organizations-// )-// );-// const users = await getAllDataByPagination(-// clerkClient.users.getUserList.bind(clerkClient.users)-// );--// console.log(`Found ${organizations.length} organizations`);-// console.log(`Found ${users.length} users`);--// for (const user of users.slice(-10)) {-// const email = user.primaryEmailAddress?.emailAddress;-// console.log('Check', email);--// try {-// if (email) {-// const exists = await db.user.findUnique({-// where: {-// id: user.id,-// },-// });--// if (exists) {-// console.log('already exists');-// } else {-// await db.user.create({-// data: {-// id: user.id,-// email: email,-// firstName: user.firstName,-// lastName: user.lastName,-// },-// });-// }-// } else {-// console.log('No email?', user);-// }-// } catch (e) {-// console.log('ERROR');-// console.log('');-// console.log('');-// console.dir(user, { depth: null });--// console.log('');-// console.log('');-// console.log('');-// }-// }--// for (const org of organizations.slice(-20)) {-// try {-// if (org.slug) {-// const exists = await db.organization.findUnique({-// where: {-// id: org.slug,-// },-// });--// if (exists) {-// console.log('already exists org');-// } else {-// const clerkOrgMembers =-// await clerkClient.organizations.getOrganizationMembershipList({-// organizationId: org.id,-// });--// const members = clerkOrgMembers.data.map((member) => {-// const user = users.find(-// (u) => u.id === member.publicUserData?.userId-// );-// return {-// userId: member.publicUserData?.userId,-// role: member.role,-// email: user!.primaryEmailAddress!.emailAddress,-// };-// });--// await db.organization.create({-// data: {-// id: org.slug,-// name: org.name,-// createdBy: {-// connect: {-// id: org.createdBy,-// },-// },-// members: {-// create: members,-// },-// },-// });--// const invites =-// await clerkClient.organizations.getOrganizationInvitationList({-// organizationId: org.id,-// status: ['pending'],-// });--// for (const invite of invites.data) {-// await db.member.create({-// data: {-// email: invite.emailAddress,-// organizationId: org.slug,-// role: invite.role,-// userId: null,-// meta: {-// access: invite.publicMetadata?.access as string[],-// invitationId: invite.id,-// },-// },-// });-// }-// }-// } else {-// console.log('org does not have any slug', org);-// }-// } catch (e) {-// console.log('ERROR');-// console.log('');-// console.log('');-// console.dir(org, { depth: null });-// console.log('');-// console.log('');-// console.log('');-// }-// }--// process.exit(0);-// }--// main();--async function main() {- const organization = await db.organization.findUnique({- where: {- id: 'openpanel-dev',- members: {- some: {- userId: 'user_2cEoI8b1SuEFbZERGEAyVvC676F',- },- },- },- include: {- members: {- select: {- role: true,- user: true,- },- },- },- });-- console.dir(organization, { depth: null });-}-main();diff --git a/apps/api/scripts/migrate-origins.ts b/apps/api/scripts/migrate-origins.tsdeleted file mode 100644index a88f5950e..000000000--- a/apps/api/scripts/migrate-origins.ts+++ /dev/null@@ -1,65 +0,0 @@-import { TABLE_NAMES, ch, chQuery } from '@openpanel/db';--async function main() {- const projects = await chQuery(- `SELECT distinct project_id FROM ${TABLE_NAMES.events} ORDER BY project_id`,- );- const withOrigin = [];-- for (const project of projects) {- try {- const [eventWithOrigin, eventWithoutOrigin] = await Promise.all([- await chQuery(- `SELECT * FROM ${TABLE_NAMES.events} WHERE origin != '' AND project_id = '${project.project_id}' ORDER BY created_at DESC LIMIT 1`,- ),- await chQuery(- `SELECT * FROM ${TABLE_NAMES.events} WHERE origin = '' AND project_id = '${project.project_id}' AND path != '' ORDER BY created_at DESC LIMIT 1`,- ),- ]);-- if (eventWithOrigin[0] && eventWithoutOrigin[0]) {- console.log(`Project ${project.project_id} as events without origin`);- console.log(`- Origin: ${eventWithOrigin[0].origin}`);- withOrigin.push(project.project_id);- const events = await chQuery(- `SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = '${project.project_id}' AND path != '' AND origin = ''`,- );- console.log(`🤠🤠🤠🤠 Will update ${events[0]?.count} events`);- await ch.command({- query: `ALTER TABLE events UPDATE origin = '${eventWithOrigin[0].origin}' WHERE project_id = '${project.project_id}' AND path != '' AND origin = '';`,- clickhouse_settings: {- wait_end_of_query: 1,- },- });- }-- if (!eventWithOrigin[0] && eventWithoutOrigin[0]) {- console.log(- `😧 Project ${project.project_id} has no events with origin (last event ${eventWithoutOrigin[0].created_at})`,- );- console.log('- NO ORIGIN');- }-- if (!eventWithOrigin[0] && !eventWithoutOrigin[0]) {- console.log(- `🔥 WARNING: Project ${project.project_id} has no events at all?!?!?!`,- );- }-- if (eventWithOrigin[0] && !eventWithoutOrigin[0]) {- console.log(- `✅ Project ${project.project_id} has all events with origin!!!`,- );- }- console.log('');- console.log('');-- await new Promise((resolve) => setTimeout(resolve, 500));- } catch (e) {- console.log('🥵 ERROR ORRROR');- console.log('Error for project', project.project_id);- }- }- process.exit(0);-}-main();diff --git a/apps/api/src/bots/bots.ts b/apps/api/src/bots/bots.tsindex 421891faf..3ce59472a 100644--- a/apps/api/src/bots/bots.ts+++ b/apps/api/src/bots/bots.ts@@ -1,4 +1,14 @@-export default [+// This file is generated by the script get-bots.ts++// The data is fetch from device-detector https://raw.githubusercontent.com/matomo-org/device-detector/master/regexes/bots.yml++const bots = [+ {+ regex: 'WireReaderBot(?:/([\\d+.]+))?',+ name: 'WireReaderBot',+ category: 'Feed Fetcher',+ url: 'https://wirereader.app/',+ },{regex: 'monitoring360bot',name: '360 Monitoring',@@ -14,40 +24,28 @@ export default [name: 'Cloudflare Health Checks',category: 'Service Agent',url: 'https://developers.cloudflare.com/health-checks/',- producer: {- name: 'CloudFlare',- url: 'https://www.cloudflare.com/',- },+ producer: { name: 'CloudFlare', url: 'https://www.cloudflare.com/' },},{regex: '360Spider',name: '360Spider',category: 'Search bot',url: 'https://www.so.com/help/help_3_2.html',- producer: {- name: 'Online Media Group, Inc.',- url: '',- },+ producer: { name: 'Online Media Group, Inc.', url: '' },},{regex: 'Aboundex',name: 'Aboundexbot',category: 'Search bot',url: 'http://www.aboundex.com/crawler/',- producer: {- name: 'Aboundex.com',- url: 'http://www.aboundex.com',- },+ producer: { name: 'Aboundex.com', url: 'http://www.aboundex.com' },},{regex: 'AcoonBot',name: 'Acoon',category: 'Search bot',url: 'http://www.acoon.de/robot.asp',- producer: {- name: 'Acoon GmbH',- url: 'http://www.acoon.de',- },+ producer: { name: 'Acoon GmbH', url: 'http://www.acoon.de' },},{regex: 'AddThis\\.com',@@ -64,69 +62,55 @@ export default [name: 'aHrefs Bot',category: 'Crawler',url: 'https://ahrefs.com/robot',- producer: {- name: 'Ahrefs Pte Ltd',- url: 'https://ahrefs.com/robot',- },+ producer: { name: 'Ahrefs Pte Ltd', url: 'https://ahrefs.com/robot' },},{regex: 'AhrefsSiteAudit/[\\d.]+',name: 'AhrefsSiteAudit',category: 'Site Monitor',url: 'https://ahrefs.com/robot/site-audit',- producer: {- name: 'Ahrefs Pte Ltd',- url: 'https://ahrefs.com/',- },+ producer: { name: 'Ahrefs Pte Ltd', url: 'https://ahrefs.com/' },},{regex: 'ia_archiver|alexabot|verifybot',name: 'Alexa Crawler',category: 'Search bot',url: 'https://support.alexa.com/hc/en-us/sections/200100794-Crawlers',- producer: {- name: 'Alexa Internet',- url: 'https://www.alexa.com',- },+ producer: { name: 'Alexa Internet', url: 'https://www.alexa.com' },},{regex: 'alexa site audit',name: 'Alexa Site Audit',category: 'Site Monitor',url: 'https://support.alexa.com/hc/en-us/articles/200450194',- producer: {- name: 'Alexa Internet',- url: 'https://www.alexa.com',- },+ producer: { name: 'Alexa Internet', url: 'https://www.alexa.com' },},{- regex: 'Amazonbot',+ regex: 'Amazonbot/[\\d.]+',name: 'Amazon Bot',category: 'Crawler',url: 'https://developer.amazon.com/support/amazonbot',- producer: {- name: 'Amazon.com, Inc.',- url: 'https://www.amazon.com/',- },+ producer: { name: 'Amazon.com, Inc.', url: 'https://www.amazon.com/' },+ },+ {+ regex: 'AmazonAdBot/[\\d.]+',+ name: 'Amazon AdBot',+ category: 'Crawler',+ url: 'https://adbot.amazon.com/',+ producer: { name: 'Amazon.com, Inc.', url: 'https://www.amazon.com/' },},{regex: 'Amazon[ -]Route ?53[ -]Health[ -]Check[ -]Service',name: 'Amazon Route53 Health Check',category: 'Service Agent',- producer: {- name: 'Amazon Web Services',- url: 'https://aws.amazon.com/',- },+ producer: { name: 'Amazon Web Services', url: 'https://aws.amazon.com/' },},{regex: 'AmorankSpider',name: 'Amorank Spider',category: 'Crawler',url: 'http://amorank.com/webcrawler.html',- producer: {- name: 'Amorank',- url: 'http://www.amorank.com',- },+ producer: { name: 'Amorank', url: 'http://www.amorank.com' },},{regex: 'ApacheBench',@@ -142,81 +126,64 @@ export default [regex: 'Applebot',name: 'Applebot',category: 'Crawler',- url: 'https://support.apple.com/en-us/HT204683',- producer: {- name: 'Apple Inc',- url: 'https://www.apple.com',- },+ url: 'https://support.apple.com/en-us/119829',+ producer: { name: 'Apple Inc', url: 'https://www.apple.com/' },+ },+ {+ regex: 'iTMS',+ name: 'iTMS',+ category: 'Crawler',+ url: 'https://support.apple.com/en-us/119829',+ producer: { name: 'Apple Inc', url: 'https://www.apple.com/' },},{regex: 'AppSignalBot',name: 'AppSignalBot',category: 'Site Monitor',url: 'https://docs.appsignal.com/uptime-monitoring/',- producer: {- name: 'AppSignal',- url: 'https://appsignal.com/',- },+ producer: { name: 'AppSignal', url: 'https://appsignal.com/' },},{regex: 'Arachni',name: 'Arachni',category: 'Security Checker',url: 'https://www.arachni-scanner.com/',- producer: {- name: 'Sarosys LLC',- url: 'https://www.sarosys.com/',- },+ producer: { name: 'Sarosys LLC', url: 'https://www.sarosys.com/' },},{regex: 'AspiegelBot',name: 'AspiegelBot',category: 'Crawler',url: 'https://aspiegel.com/',- producer: {- name: 'Huawei',- url: 'https://www.huawei.com/',- },+ producer: { name: 'Huawei', url: 'https://www.huawei.com/' },},{regex: 'Castro 2, Episode Duration Lookup',name: 'Castro 2',category: 'Service Agent',url: 'http://supertop.co/castro/',- producer: {- name: 'Supertop',- url: 'http://supertop.co',- },+ producer: { name: 'Supertop', url: 'http://supertop.co' },},{regex: 'Curious George',name: 'Analytics SEO Crawler',category: 'Crawler',url: 'http://www.analyticsseo.com/crawler',- producer: {- name: 'Analytics SEO',- url: 'http://www.analyticsseo.com',- },+ producer: { name: 'Analytics SEO', url: 'http://www.analyticsseo.com' },},{regex: 'archive\\.org_bot|special_archiver',name: 'archive.org bot',category: 'Crawler',url: 'https://archive.org/details/archive.org_bot',- producer: {- name: 'The Internet Archive',- url: 'https://archive.org',- },+ producer: { name: 'The Internet Archive', url: 'https://archive.org' },},{regex: 'Ask Jeeves/Teoma',name: 'Ask Jeeves',category: 'Search bot',url: '',- producer: {- name: 'Ask Jeeves Inc.',- url: 'http://www.ask.com',- },+ producer: { name: 'Ask Jeeves Inc.', url: 'http://www.ask.com' },},{regex: 'Backlink-Check\\.de',@@ -233,40 +200,28 @@ export default [name: 'BacklinkCrawler',category: 'Crawler',url: 'http://www.backlinktest.com/crawler.html',- producer: {- name: '2.0Promotion GbR',- url: 'http://www.backlinktest.com',- },+ producer: { name: '2.0Promotion GbR', url: 'http://www.backlinktest.com' },},{regex: 'Baidu.*spider|baidu Transcoder',name: 'Baidu Spider',category: 'Search bot',url: 'http://www.baidu.com/search/spider.htm',- producer: {- name: 'Baidu',- url: 'http://www.baidu.com',- },+ producer: { name: 'Baidu', url: 'http://www.baidu.com' },},{regex: 'BazQux',name: 'BazQux Reader',url: 'https://bazqux.com/fetcher',category: 'Feed Fetcher',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'Better Uptime Bot',name: 'Better Uptime Bot',category: 'Site Monitor',url: 'https://betteruptime.com/faq',- producer: {- name: 'Better Uptime',- url: 'https://betteruptime.com/',- },+ producer: { name: 'Better Uptime', url: 'https://betteruptime.com/' },},{regex:@@ -279,74 +234,60 @@ export default [url: 'http://www.microsoft.com',},},+ {+ regex: 'Blackbox Exporter',+ name: 'Blackbox Exporter',+ category: 'Site Monitor',+ url: 'https://github.com/prometheus/blackbox_exporter',+ producer: { name: 'Prometheus', url: 'https://prometheus.io/' },+ },{regex: 'Blekkobot',name: 'Blekkobot',category: 'Search bot',url: 'http://blekko.com/about/blekkobot',- producer: {- name: 'Blekko',- url: 'http://blekko.com',- },+ producer: { name: 'Blekko', url: 'http://blekko.com' },},{regex: 'BLEXBot',name: 'BLEXBot Crawler',category: 'Crawler',url: 'http://webmeup-crawler.com',- producer: {- name: 'WebMeUp',- url: 'http://webmeup.com',- },+ producer: { name: 'WebMeUp', url: 'http://webmeup.com' },},{regex: 'Bloglovin',name: 'Bloglovin',url: 'http://www.bloglovin.com',category: 'Feed Fetcher',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'Blogtrottr',name: 'Blogtrottr',url: '',category: 'Feed Fetcher',- producer: {- name: 'Blogtrottr Ltd',- url: 'https://blogtrottr.com/',- },+ producer: { name: 'Blogtrottr Ltd', url: 'https://blogtrottr.com/' },},{regex: 'BoardReader Blog Indexer',name: 'BoardReader Blog Indexer',category: 'Crawler',- producer: {- name: 'BoardReader',- url: 'https://boardreader.com/',- },+ producer: { name: 'BoardReader', url: 'https://boardreader.com/' },},{regex: 'BountiiBot',name: 'Bountii Bot',category: 'Search bot',url: 'http://bountii.com/contact.php',- producer: {- name: 'Bountii Inc.',- url: 'http://bountii.com',- },+ producer: { name: 'Bountii Inc.', url: 'http://bountii.com' },},{regex: 'Browsershots',name: 'Browsershots',category: 'Service Agent',url: 'http://browsershots.org/faq',- producer: {- name: 'Browsershots.org',- url: 'http://browsershots.org',- },+ producer: { name: 'Browsershots.org', url: 'http://browsershots.org' },},{regex: 'BUbiNG',@@ -363,140 +304,112 @@ export default [name: 'Butterfly Robot',category: 'Search bot',url: 'http://labs.topsy.com/butterfly',- producer: {- name: 'Topsy Labs',- url: 'http://labs.topsy.com',- },+ producer: { name: 'Topsy Labs', url: 'http://labs.topsy.com' },},{regex: 'CareerBot',name: 'CareerBot',category: 'Crawler',url: 'http://www.career-x.de/bot.html',- producer: {- name: 'career-x GmbH',- url: 'http://www.career-x.de',- },+ producer: { name: 'career-x GmbH', url: 'http://www.career-x.de' },},{regex: 'CCBot',name: 'ccBot crawler',category: 'Crawler',url: 'http://commoncrawl.org/faq/',- producer: {- name: 'reddit inc.',- url: 'http://www.reddit.com',- },+ producer: { name: 'reddit inc.', url: 'http://www.reddit.com' },},{regex: 'Cliqzbot',name: 'Cliqzbot',category: 'Crawler',url: 'http://cliqz.com/company/cliqzbot',- producer: {- name: '10betterpages GmbH',- url: 'http://cliqz.com',- },+ producer: { name: '10betterpages GmbH', url: 'http://cliqz.com' },},{regex: 'Cloudflare-AMP',name: 'CloudFlare AMP Fetcher',category: 'Crawler',url: 'https://amp.cloudflare.com/doc/fetcher.html',- producer: {- name: 'CloudFlare',- url: 'http://www.cloudflare.com',- },+ producer: { name: 'CloudFlare', url: 'http://www.cloudflare.com' },},{regex: 'Cloudflare-?Diagnostics',name: 'Cloudflare Diagnostics',category: 'Site Monitor',url: 'https://www.cloudflare.com/',- producer: {- name: 'Cloudflare',- url: 'https://www.cloudflare.com/',- },+ producer: { name: 'Cloudflare', url: 'https://www.cloudflare.com/' },},{regex: 'CloudFlare-AlwaysOnline',name: 'CloudFlare Always Online',category: 'Site Monitor',url: 'https://www.cloudflare.com/always-online',- producer: {- name: 'CloudFlare',- url: 'https://www.cloudflare.com/',- },+ producer: { name: 'CloudFlare', url: 'https://www.cloudflare.com/' },},{regex: 'Cloudflare-SSLDetector',name: 'Cloudflare SSL Detector',category: 'Site Monitor',url: 'https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/',- producer: {- name: 'CloudFlare',- url: 'https://www.cloudflare.com/',- },+ producer: { name: 'CloudFlare', url: 'https://www.cloudflare.com/' },},{regex: 'Cloudflare Custom Hostname Verification',name: 'Cloudflare Custom Hostname Verification',category: 'Service Agent',url: 'https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/',- producer: {- name: 'CloudFlare',- url: 'https://www.cloudflare.com/',- },+ producer: { name: 'CloudFlare', url: 'https://www.cloudflare.com/' },},{regex: 'Cloudflare-Traffic-Manager',name: 'Cloudflare Traffic Manager',category: 'Site Monitor',url: 'https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/',- producer: {- name: 'CloudFlare',- url: 'https://www.cloudflare.com/',- },+ producer: { name: 'CloudFlare', url: 'https://www.cloudflare.com/' },+ },+ {+ regex: 'Cloudflare-Smart-Transit',+ name: 'Cloudflare Smart Transit',+ category: 'Site Monitor',+ url: 'https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/',+ producer: { name: 'CloudFlare', url: 'https://www.cloudflare.com/' },+ },+ {+ regex: 'CloudflareObservatory',+ name: 'Cloudflare Observatory',+ category: 'Site Monitor',+ url: 'https://developers.cloudflare.com/speed/speed-test/run-speed-test',+ producer: { name: 'CloudFlare', url: 'https://www.cloudflare.com/' },},{regex: 'https://developers\\.cloudflare\\.com/security-center/',name: 'Cloudflare Security Insights',category: 'Site Monitor',url: 'https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/',- producer: {- name: 'CloudFlare',- url: 'https://www.cloudflare.com/',- },+ producer: { name: 'CloudFlare', url: 'https://www.cloudflare.com/' },},{regex: 'coccoc\\.com',name: 'Cốc Cốc Bot',url: 'https://help.coccoc.com/en/search-engine/coccoc-robots',category: 'Search bot',- producer: {- name: 'Cốc Cốc',- url: 'https://coccoc.com/',- },+ producer: { name: 'Cốc Cốc', url: 'https://coccoc.com/' },},{regex: 'collectd',name: 'Collectd',url: 'https://collectd.org/',category: 'Site Monitor',- producer: {- name: 'Collectd',- url: 'https://collectd.org/',- },+ producer: { name: 'Collectd', url: 'https://collectd.org/' },},{regex: 'CommaFeed',name: 'CommaFeed',url: 'http://www.commafeed.com',category: 'Feed Fetcher',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'CSS Certificate Spider',@@ -513,20 +426,14 @@ export default [name: 'Datadog Agent',url: 'https://github.com/DataDog/dd-agent',category: 'Site Monitor',- producer: {- name: 'Datadog',- url: 'https://www.datadoghq.com/',- },+ producer: { name: 'Datadog', url: 'https://www.datadoghq.com/' },},{regex: 'Datanyze',name: 'Datanyze',url: '',category: 'Crawler',- producer: {- name: 'Datanyze',- url: 'https://www.datanyze.com',- },+ producer: { name: 'Datanyze', url: 'https://www.datanyze.com' },},{regex: 'Dataprovider',@@ -553,20 +460,14 @@ export default [name: 'Dazoobot',category: 'Search bot',url: '',- producer: {- name: 'DAZOO.FR',- url: 'http://dazoo.fr',- },+ producer: { name: 'DAZOO.FR', url: 'http://dazoo.fr' },},{regex: 'discobot',name: 'Discobot',category: 'Search bot',url: 'http://discoveryengine.com/discobot.html',- producer: {- name: 'Discovery Engine',- url: 'http://discoveryengine.com',- },+ producer: { name: 'Discovery Engine', url: 'http://discoveryengine.com' },},{regex: 'Domain Re-Animator Bot|support@domainreanimator\\.com',@@ -583,49 +484,41 @@ export default [name: 'DotBot',category: 'Crawler',url: 'http://www.opensiteexplorer.org/dotbot',- producer: {- name: 'SEOmoz, Inc.',- url: 'http://moz.com/',- },+ producer: { name: 'SEOmoz, Inc.', url: 'http://moz.com/' },},{regex: 'DuckDuck(?:Go-Favicons-)?Bot',- name: 'DuckDuckGo Bot',+ name: 'DuckDuckBot',category: 'Search bot',- url: 'https://duckduckgo.com/duckduckbot',- producer: {- name: 'DuckDuckGo',- url: 'https://duckduckgo.com/',- },+ url: 'https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/',+ producer: { name: 'DuckDuckGo', url: 'https://duckduckgo.com/' },+ },+ {+ regex: 'DuckAssistBot',+ name: 'DuckAssistBot',+ category: 'Search bot',+ url: 'https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/',+ producer: { name: 'DuckDuckGo', url: 'https://duckduckgo.com/' },},{regex: 'EasouSpider',name: 'Easou Spider',category: 'Search bot',url: 'http://www.easou.com/search/spider.html',- producer: {- name: 'easou ICP',- url: 'http://www.easou.com',- },+ producer: { name: 'easou ICP', url: 'http://www.easou.com' },},{regex: 'eCairn-Grabber',name: 'eCairn-Grabber',category: 'Crawler',- producer: {- name: 'eCairn',- url: 'https://ecairn.com',- },+ producer: { name: 'eCairn', url: 'https://ecairn.com' },},{regex: 'EMail Exractor',name: 'EMail Exractor',category: 'Crawler',url: '',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'evc-batch',@@ -642,10 +535,7 @@ export default [name: 'ExaBot',category: 'Crawler',url: 'http://www.exabot.com/go/robot',- producer: {- name: 'Dassault Systèmes',- url: 'http://www.3ds.com',- },+ producer: { name: 'Dassault Systèmes', url: 'http://www.3ds.com' },},{regex: 'ExactSeek Crawler',@@ -662,51 +552,35 @@ export default [name: 'Ezooms',category: 'Crawler',url: '',- producer: {- name: 'SEOmoz, Inc.',- url: 'http://moz.com/',- },+ producer: { name: 'SEOmoz, Inc.', url: 'http://moz.com/' },},{- regex:- 'facebookexternalhit|facebookplatform|facebookexternalua|facebookcatalog',- name: 'Facebook External Hit',+ regex: 'facebook(?:catalog|externalhit|externalua|platform|scraper)',+ name: 'Facebook Crawler',category: 'Social Media Agent',- url: 'https://www.facebook.com/externalhit_uatext.php',- producer: {- name: 'Meta Platforms, Inc.',- url: 'https://www.meta.com/',- },+ url: 'https://developers.facebook.com/docs/sharing/webmasters/crawler/',+ producer: { name: 'Meta Platforms, Inc.', url: 'https://www.meta.com/' },},{regex: 'FacebookBot/[\\d.]+',name: 'FacebookBot',category: 'Crawler',url: 'https://developers.facebook.com/docs/sharing/bot',- producer: {- name: 'Meta Platforms, Inc.',- url: 'https://www.meta.com/',- },+ producer: { name: 'Meta Platforms, Inc.', url: 'https://www.meta.com/' },},{regex: 'Feedbin',name: 'Feedbin',url: 'http://feedbin.com/',category: 'Feed Fetcher',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'FeedBurner',name: 'FeedBurner',url: 'http://www.feedburner.com',category: 'Feed Fetcher',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'Feed Wrangler',@@ -723,40 +597,28 @@ export default [name: 'Feedly',url: 'http://www.feedly.com',category: 'Feed Fetcher',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'Feedspot',name: 'Feedspot',url: 'http://www.feedspot.com',category: 'Feed Fetcher',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'Fever/[0-9]',name: 'Fever',url: 'http://feedafever.com/',category: 'Feed Fetcher',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'FlipboardProxy|FlipboardRSS',name: 'Flipboard',url: 'http://flipboard.com/browserproxy',category: 'Feed Fetcher',- producer: {- name: 'Flipboard',- url: 'http://flipboard.com/',- },+ producer: { name: 'Flipboard', url: 'http://flipboard.com/' },},{regex: 'Findxbot',@@ -775,30 +637,21 @@ export default [name: 'Genieo Web filter',category: '',url: 'http://www.genieo.com/webfilter.html',- producer: {- name: 'Genieo',- url: 'http://www.genieo.com',- },+ producer: { name: 'Genieo', url: 'http://www.genieo.com' },},{regex: 'GigablastOpenSource',name: 'Gigablast',category: 'Search bot',url: 'https://github.com/gigablast/open-source-search-engine',- producer: {- name: 'Matt Wells',- url: 'http://www.gigablast.com/faq.html',- },+ producer: { name: 'Matt Wells', url: 'http://www.gigablast.com/faq.html' },},{regex: 'Gluten Free Crawler',name: 'Gluten Free Crawler',category: 'Crawler',url: 'http://glutenfreepleasure.com/',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'gobuster',@@ -810,206 +663,176 @@ export default [name: 'Goo',category: 'Search bot',url: 'http://search.goo.ne.jp/option/use/sub4/sub4-1',- producer: {- name: 'NTT Resonant',- url: 'http://goo.ne.jp',- },- },- {- regex: 'Storebot-Google',- name: 'Google StoreBot',- category: 'Crawler',- },- {- regex: 'Google Favicon',- name: 'Google Favicon',- category: 'Crawler',+ producer: { name: 'NTT Resonant', url: 'http://goo.ne.jp' },},+ { regex: 'Storebot-Google', name: 'Google StoreBot', category: 'Crawler' },+ { regex: 'Google Favicon', name: 'Google Favicon', category: 'Crawler' },{regex: 'Google Search Console',name: 'Google Search Console',category: 'Crawler',url: 'https://search.google.com/search-console/about',- producer: {- name: 'Google Inc.',- url: 'https://www.google.com/',- },+ producer: { name: 'Google Inc.', url: 'https://www.google.com/' },},{regex: 'Google Page Speed Insights',name: 'Google PageSpeed Insights',category: 'Site Monitor',url: 'http://developers.google.com/speed/pagespeed/insights/',- producer: {- name: 'Google Inc.',- url: 'https://www.google.com/',- },+ producer: { name: 'Google Inc.', url: 'https://www.google.com/' },},{regex: 'google_partner_monitoring',name: 'Google Partner Monitoring',category: 'Site Monitor',url: '',- producer: {- name: 'Google Inc.',- url: 'https://www.google.com/',- },+ producer: { name: 'Google Inc.', url: 'https://www.google.com/' },},{regex: 'Google-Cloud-Scheduler',name: 'Google Cloud Scheduler',category: 'Crawler',url: 'https://cloud.google.com/scheduler',- producer: {- name: 'Google Inc.',- url: 'https://www.google.com',- },+ producer: { name: 'Google Inc.', url: 'https://www.google.com' },},{regex: 'Google-Structured-Data-Testing-Tool',name: 'Google Structured Data Testing Tool',category: 'Validator',url: 'https://search.google.com/structured-data/testing-tool',- producer: {- name: 'Google Inc.',- url: 'https://www.google.com/',- },+ producer: { name: 'Google Inc.', url: 'https://www.google.com/' },},{regex: 'GoogleStackdriverMonitoring',name: 'Google Stackdriver Monitoring',category: 'Site Monitor',url: 'https://cloud.google.com/monitoring',- producer: {- name: 'Google Inc.',- url: 'https://www.google.com',- },+ producer: { name: 'Google Inc.', url: 'https://www.google.com' },},{regex: 'Google-Transparency-Report',name: 'Google Transparency Report',category: 'Site Monitor',url: 'https://transparencyreport.google.com/',- producer: {- name: 'Google Inc.',- url: 'https://www.google.com/',- },+ producer: { name: 'Google Inc.', url: 'https://www.google.com/' },+ },+ {+ regex: 'Google-CloudVertexBot',+ name: 'Google-CloudVertexBot',+ category: 'Crawler',+ url: 'https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers#google-cloudvertexbot',+ producer: { name: 'Google Inc.', url: 'https://www.google.com/' },},{regex: 'via ggpht\\.com GoogleImageProxy',name: 'Gmail Image Proxy',category: 'Crawler',url: '',- producer: {- name: 'Google Inc.',- url: 'https://www.google.com/',- },+ producer: { name: 'Google Inc.', url: 'https://www.google.com/' },},{regex: 'SeznamEmailProxy',name: 'Seznam Email Proxy',category: 'Crawler',url: '',- producer: {- name: 'Seznam.cz, a.s.',- url: 'http://www.seznam.cz/',- },+ producer: { name: 'Seznam.cz, a.s.', url: 'http://www.seznam.cz/' },},{regex: 'Seznam-Zbozi-robot',name: 'Seznam Zbozi.cz',category: 'Crawler',url: '',- producer: {- name: 'Seznam.cz, a.s.',- url: 'https://www.zbozi.cz/',- },+ producer: { name: 'Seznam.cz, a.s.', url: 'https://www.zbozi.cz/' },},{regex: 'Heurekabot-Feed',name: 'Heureka Feed',category: 'Crawler',url: 'https://sluzby.heureka.cz/napoveda/heurekabot/',- producer: {- name: 'Heureka.cz, a.s.',- url: 'https://www.heureka.cz/',- },+ producer: { name: 'Heureka.cz, a.s.', url: 'https://www.heureka.cz/' },},{regex: 'ShopAlike',name: 'ShopAlike',category: 'Crawler',url: '',- producer: {- name: 'Visual Meta',- url: 'https://www.shopalike.cz/',- },+ producer: { name: 'Visual Meta', url: 'https://www.shopalike.cz/' },+ },+ {+ regex: 'deepcrawl\\.com',+ name: 'Lumar',+ category: 'Crawler',+ url: 'https://deepcrawl.com/bot',+ producer: { name: 'Lumar', url: 'https://www.lumar.io/' },+ },+ {+ regex: 'Googlebot-News',+ name: 'Googlebot News',+ category: 'Search bot',+ url: 'https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers',+ producer: { name: 'Google Inc.', url: 'https://www.google.com/' },},{regex:- 'Adwords-(?:DisplayAds|Express|Instant)|Google Web Preview|Google[ -]Publisher[ -]Plugin|Google-(?:Ads-Conversions|Ads-Qualify|Adwords|AMPHTML|Assess|Extended|HotelAdsVerifier|InspectionTool|PageRenderer|Read-Aloud|Safety|Shopping-Quality|Site-Verification|speakr|Stale-Content-Probe|Test|Youtube-Links)|(?:AdsBot|APIs|DuplexWeb|Feedfetcher|Mediapartners)-Google(?:-Mobile)?|Google(?:AdSenseInfeed|AssociationService|bot|Other|Prober|Producer)|Google.*/\\+/web/snippet',+ 'Adwords-(?:DisplayAds|Express|Instant)|Google Web Preview|Google[ -]Publisher[ -]Plugin|Google-(?:adstxt|Ads-Conversions|Ads-Qualify|Adwords|AMPHTML|Assess|Extended|HotelAdsVerifier|InspectionTool|Lens|PageRenderer|Read-Aloud|Shopping-Quality|Site-Verification|Sites-Thumbnails|speakr|Stale-Content-Probe|Test|Youtube-Links)|(?:AdsBot|APIs|Feedfetcher|Mediapartners)-Google(?:-Mobile)?|Google(?:AdSenseInfeed|AssociationService|bot|Other|Prober|Producer|Sites)|Google.*/\\+/web/snippet',name: 'Googlebot',category: 'Search bot',url: 'https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers',- producer: {- name: 'Google Inc.',- url: 'https://www.google.com/',- },+ producer: { name: 'Google Inc.', url: 'https://www.google.com/' },},{regex: '^Google$',name: 'Googlebot',category: 'Search bot',url: 'https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers',- producer: {- name: 'Google Inc.',- url: 'https://www.google.com/',- },+ producer: { name: 'Google Inc.', url: 'https://www.google.com/' },},{- regex: 'heritrix',- name: 'Heritrix',+ regex: 'Google-Safety',+ name: 'Google-Safety',category: 'Crawler',- url: 'https://webarchive.jira.com/wiki/display/Heritrix/Heritrix',- producer: {- name: 'The Internet Archive',- url: 'https://archive.org',- },+ url: 'https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers',+ producer: { name: 'Google Inc.', url: 'https://www.google.com/' },+ },+ {+ regex: 'DuplexWeb-Google',+ name: 'DuplexWeb-Google',+ category: 'Crawler',+ url: 'https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers',+ producer: { name: 'Google Inc.', url: 'https://www.google.com/' },+ },+ {+ regex: 'Google-Area120-PrivacyPolicyFetcher',+ name: 'Google Area 120 Privacy Policy Fetcher',+ category: 'Crawler',+ url: 'https://area120.google.com/',+ producer: { name: 'Google Inc.', url: 'https://www.google.com/' },},{regex: 'HubSpot ',name: 'HubSpot',category: 'Crawler',- producer: {- name: 'HubSpot Inc.',- url: 'https://www.hubspot.com',- },+ producer: { name: 'HubSpot Inc.', url: 'https://www.hubspot.com' },},{- regex: 'vuhuvBot',- name: 'Vuhuv Bot',- category: 'Crawler',- url: 'http://vuhuv.com/bot.html',+ regex: 'vuhuv(?:Bot|RBT)',+ name: 'vuhuvBot',+ category: 'Search bot',+ url: 'https://vuhuv.com/bot.html',},{regex: 'HTTPMon/[\\d.]+',name: 'HTTPMon',category: 'Site Monitor',url: 'http://www.httpmon.com',- producer: {- name: 'towards GmbH',- url: 'http://www.towards.ch/',- },+ producer: { name: 'towards GmbH', url: 'http://www.towards.ch/' },},{regex: 'ICC-Crawler',name: 'ICC-Crawler',category: 'Crawler',url: 'http://www.nict.go.jp/en/univ-com/plan/crawl.html',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'inoreader\\.com',@@ -1031,45 +854,29 @@ export default [regex: 'ips-agent',name: 'IPS Agent',category: 'Crawler',- producer: {- name: 'VeriSign, Inc',- url: 'http://www.verisign.com/',- },+ producer: { name: 'VeriSign, Inc', url: 'http://www.verisign.com/' },},{regex: 'IP-Guide\\.com',name: 'IP-Guide Crawler',category: 'Crawler',url: '',- producer: {- name: '',- url: 'https://ip-guide.com',- },- },- {- regex: 'k6/[0-9\\.]+',- name: 'K6',- url: 'https://k6.io/',+ producer: { name: '', url: 'https://ip-guide.com' },},+ { regex: 'k6/[0-9\\.]+', name: 'K6', url: 'https://k6.io/' },{regex: 'kouio',name: 'Kouio',url: 'http://kouio.com/',category: 'Feed Fetcher',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'larbin',name: 'Larbin web crawler',category: 'Crawler',url: 'http://larbin.sourceforge.net',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: '[A-z0-9]*-Lighthouse',@@ -1086,83 +893,59 @@ export default [name: 'LastMod Bot',category: 'Site Monitor',url: 'https://last-modified.com/en/about',- producer: {- name: '',- url: 'https://last-modified.com/en',- },+ producer: { name: '', url: 'https://last-modified.com/en' },},{regex: 'linkdexbot|linkdex\\.com',name: 'Linkdex Bot',category: 'Search bot',url: 'http://www.linkdex.com/bots',- producer: {- name: 'Mojeek Ltd.',- url: 'http://www.mojeek.com',- },+ producer: { name: 'Mojeek Ltd.', url: 'http://www.mojeek.com' },},{regex: 'LinkedInBot',name: 'LinkedIn Bot',category: 'Social Media Agent',url: 'http://www.linkedin.com',- producer: {- name: 'LinkedIn',- url: 'http://www.linkedin.com',- },+ producer: { name: 'LinkedIn', url: 'http://www.linkedin.com' },},{regex: 'ltx71',name: 'LTX71',category: 'Security Checker',url: 'https://ltx71.com/',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'Mail\\.RU',name: 'Mail.Ru Bot',category: 'Search bot',url: 'http://help.mail.ru/webmaster/indexing/robots/types_robots',- producer: {- name: 'Mail.Ru Group',- url: 'http://corp.mail.ru',- },+ producer: { name: 'Mail.Ru Group', url: 'http://corp.mail.ru' },},{regex: 'magpie-crawler',name: 'Magpie-Crawler',category: 'Social Media Agent',url: 'http://www.brandwatch.com/magpie-crawler/',- producer: {- name: 'Brandwatch',- url: 'http://www.brandwatch.com',- },+ producer: { name: 'Brandwatch', url: 'http://www.brandwatch.com' },},{regex: 'MagpieRSS',name: 'MagpieRSS',url: 'http://magpierss.sourceforge.net/',category: 'Feed Parser',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'masscan-ng/[\\d.]+',name: 'masscan-ng',url: 'https://github.com/bi-zone/masscan-ng',category: 'Crawler',- producer: {- name: 'BIZON, OOO',- url: 'https://bi.zone/',- },+ producer: { name: 'BIZON, OOO', url: 'https://bi.zone/' },},{- regex: 'masscan',+ regex: '.*masscan',name: 'masscan',url: 'https://github.com/robertdavidgraham/masscan',category: 'Crawler',@@ -1171,30 +954,20 @@ export default [url: 'https://github.com/robertdavidgraham',},},- {- regex: 'Mastodon/',- name: 'Mastodon Bot',- category: 'Social Media Agent',- },+ { regex: 'Mastodon/', name: 'Mastodon Bot', category: 'Social Media Agent' },{regex: 'meanpathbot',name: 'Meanpath Bot',category: 'Search bot',url: 'http://www.meanpath.com/meanpathbot.html',- producer: {- name: 'Meanpath',- url: 'http://www.meanpath.com',- },+ producer: { name: 'Meanpath', url: 'http://www.meanpath.com' },},{regex: 'MetaJobBot',name: 'MetaJobBot',category: 'Crawler',url: 'http://www.metajob.at/the/crawler',- producer: {- name: 'MetaJob',- url: 'http://www.metajob.at',- },+ producer: { name: 'MetaJob', url: 'http://www.metajob.at' },},{regex: 'MetaInspector',@@ -1207,60 +980,42 @@ export default [name: 'Mixrank Bot',category: 'Crawler',url: 'http://mixrank.com',- producer: {- name: 'Online Media Group, Inc.',- url: '',- },+ producer: { name: 'Online Media Group, Inc.', url: '' },},{regex: 'MJ12bot',name: 'MJ12 Bot',category: 'Search bot',url: 'http://majestic12.co.uk/bot.php',- producer: {- name: 'Majestic-12',- url: 'http://majestic12.co.uk',- },+ producer: { name: 'Majestic-12', url: 'http://majestic12.co.uk' },},{regex: 'Mnogosearch',name: 'Mnogosearch',category: 'Search bot',url: 'http://www.mnogosearch.org/',- producer: {- name: 'Lavtech.Com Corp.',- url: '',- },+ producer: { name: 'Lavtech.Com Corp.', url: '' },},{regex: 'MojeekBot',name: 'MojeekBot',category: 'Search bot',url: 'http://www.mojeek.com/bot.html',- producer: {- name: 'Mojeek Ltd.',- url: 'http://www.mojeek.com',- },+ producer: { name: 'Mojeek Ltd.', url: 'http://www.mojeek.com' },},{regex: 'munin',name: 'Munin',category: 'Site Monitor',url: 'http://munin-monitoring.org/',- producer: {- name: 'Munin',- url: 'http://munin-monitoring.org/',- },+ producer: { name: 'Munin', url: 'http://munin-monitoring.org/' },},{regex: 'NalezenCzBot',name: 'NalezenCzBot',category: 'Crawler',url: 'http://www.nalezen.cz/about-crawler',- producer: {- name: 'Jaroslav Kuboš',- url: '',- },+ producer: { name: 'Jaroslav Kuboš', url: '' },},{regex: 'check_http/v',@@ -1282,123 +1037,91 @@ export default [name: 'Netcraft Survey Bot',category: 'Search bot',url: '',- producer: {- name: 'Netcraft',- url: 'http://www.netcraft.com',- },+ producer: { name: 'Netcraft', url: 'http://www.netcraft.com' },},{regex: 'netEstate NE Crawler',name: 'netEstate',category: 'Crawler',url: 'http://www.website-datenbank.de/Impressum',- producer: {- name: 'netEstate GmbH',- url: 'https://www.netestate.de/en/',- },+ producer: { name: 'netEstate GmbH', url: 'https://www.netestate.de/en/' },},{regex: 'Netvibes',name: 'Netvibes',url: 'http://www.netvibes.com/',category: 'Feed Fetcher',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'NewsBlur .*(?:Fetcher|Finder)',name: 'NewsBlur',url: 'http://www.newsblur.com',category: 'Feed Fetcher',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'NewsGatorOnline',name: 'NewsGator',url: 'http://www.newsgator.com',category: 'Feed Fetcher',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'nlcrawler',name: 'NLCrawler',category: 'Crawler',url: '',- producer: {- name: 'Northern Light',- url: 'http://northernlight.com',- },+ producer: { name: 'Northern Light', url: 'http://northernlight.com' },},{regex: 'Nmap Scripting Engine',name: 'Nmap',category: 'Security Checker',url: 'https://nmap.org/book/nse.html',- producer: {- name: 'Nmap',- url: 'https://nmap.org/',- },+ producer: { name: 'Nmap', url: 'https://nmap.org/' },},{regex: 'Nuzzel',name: 'Nuzzel',category: 'Crawler',- producer: {- name: 'Nuzzel',- url: 'https://www.nuzzel.com/',- },+ producer: { name: 'Nuzzel', url: 'https://www.nuzzel.com/' },},{- regex: 'Octopus [0-9]',- name: 'Octopus',+ regex: 'NodePing',+ name: 'NodePing',+ category: 'Site Monitor',+ url: 'https://nodeping.com',+ producer: { name: 'NodePing', url: 'https://nodeping.com' },},+ { regex: 'Octopus [0-9]', name: 'Octopus' },{regex: 'OnlineOrNot\\.com_bot',name: 'OnlineOrNot Bot',category: 'Site Monitor',url: 'https://onlineornot.com/website-monitoring',- producer: {- name: 'OnlineOrNot',- url: 'https://onlineornot.com',- },+ producer: { name: 'OnlineOrNot', url: 'https://onlineornot.com' },},{regex: 'omgili',name: 'Omgili bot',category: 'Search bot',url: 'http://www.omgili.com/Crawler.html',- producer: {- name: 'Omgili',- url: 'http://www.omgili.com',- },+ producer: { name: 'Omgili', url: 'http://www.omgili.com' },},{regex: 'OpenindexSpider',name: 'Openindex Spider',category: 'Search bot',url: 'http://www.openindex.io/en/webmasters/spider.html',- producer: {- name: 'Openindex B.V.',- url: 'http://www.openindex.io',- },+ producer: { name: 'Openindex B.V.', url: 'http://www.openindex.io' },},{regex: 'spbot',name: 'OpenLinkProfiler',category: 'Crawler',url: 'http://openlinkprofiler.org/bot',- producer: {- name: 'Axandra GmbH',- url: 'http://www.axandra.com',- },+ producer: { name: 'Axandra GmbH', url: 'http://www.axandra.com' },},{regex: 'OpenWebSpider',@@ -1415,20 +1138,14 @@ export default [name: 'Orange Bot',category: 'Search bot',url: 'http://lemoteur.orange.fr',- producer: {- name: 'Orange',- url: 'http://www.orange.fr',- },+ producer: { name: 'Orange', url: 'http://www.orange.fr' },},{regex: 'PaperLiBot',name: 'PaperLiBot',category: 'Search bot',url: 'http://support.paper.li/entries/20023257-what-is-paper-li',- producer: {- name: 'Smallrivers SA',- url: 'http://www.paper.li',- },+ producer: { name: 'Smallrivers SA', url: 'http://www.paper.li' },},{regex: 'phantomas/',@@ -1451,100 +1168,70 @@ export default [name: 'Pocket',category: 'Read-it-later Service',url: 'https://getpocket.com/pocketparser_ua',- producer: {- name: 'Read It Later, Inc.',- url: 'https://getpocket.com/',- },+ producer: { name: 'Read It Later, Inc.', url: 'https://getpocket.com/' },},{regex: 'PritTorrent',name: 'PritTorrent',category: 'Crawler',url: 'https://github.com/astro/prittorrent',- producer: {- name: 'Bitlove',- url: 'http://bitlove.org/',- },+ producer: { name: 'Bitlove', url: 'http://bitlove.org/' },},{regex: 'PRTG Network Monitor',name: 'PRTG Network Monitor',category: 'Network Monitor',url: 'https://www.paessler.com/prtg',- producer: {- name: 'Paessler AG',- url: 'https://www.paessler.com',- },+ producer: { name: 'Paessler AG', url: 'https://www.paessler.com' },},{regex: 'psbot',name: 'Picsearch bot',category: 'Search bot',url: 'http://www.picsearch.com/bot.html',- producer: {- name: 'Picsearch',- url: 'http://www.picsearch.com',- },+ producer: { name: 'Picsearch', url: 'http://www.picsearch.com' },},{regex: 'Pingdom(?:\\.com|TMS)',name: 'Pingdom Bot',category: 'Site Monitor',url: '',- producer: {- name: 'Pingdom AB',- url: 'https://www.pingdom.com',- },+ producer: { name: 'Pingdom AB', url: 'https://www.pingdom.com' },},{regex: 'Quora Link Preview',name: 'Quora Link Preview',category: 'Crawler',url: '',- producer: {- name: 'Quora',- url: 'http://www.quora.com',- },+ producer: { name: 'Quora', url: 'http://www.quora.com' },},{regex: 'Quora-Bot',name: 'Quora Bot',category: 'Crawler',url: '',- producer: {- name: 'Quora',- url: 'https://www.quora.com/',- },+ producer: { name: 'Quora', url: 'https://www.quora.com/' },},{regex: 'RamblerMail',name: 'RamblerMail Image Proxy',category: 'Crawler',url: '',- producer: {- name: 'Rambler&Co',- url: 'https://rambler-co.ru/',- },+ producer: { name: 'Rambler&Co', url: 'https://rambler-co.ru/' },},{regex: 'QuerySeekerSpider',name: 'QuerySeekerSpider',category: 'Crawler',url: 'http://queryseeker.com/bot.html',- producer: {- name: 'QueryEye Inc.',- url: 'http://queryeye.com',- },+ producer: { name: 'QueryEye Inc.', url: 'http://queryeye.com' },},{- regex: 'Qwantify',- name: 'Qwantify',+ regex: 'Qwantify|Qwantbot',+ name: 'Qwantbot',category: 'Crawler',- url: 'https://www.qwant.com/',- producer: {- name: 'Qwant Corporation',- url: 'https://www.qwant.com/',- },+ url: 'https://help.qwant.com/bot/',+ producer: { name: 'Qwant Corporation', url: 'https://www.qwant.com/' },},{regex: 'Rainmeter',@@ -1557,50 +1244,35 @@ export default [name: 'Reddit Bot',category: 'Social Media Agent',url: 'http://www.reddit.com/feedback',- producer: {- name: 'reddit inc.',- url: 'http://www.reddit.com',- },+ producer: { name: 'reddit inc.', url: 'http://www.reddit.com' },},{regex: 'Riddler',name: 'Riddler',category: 'Security search bot',url: 'https://riddler.io/about',- producer: {- name: 'F-Secure',- url: 'https://www.f-secure.com',- },+ producer: { name: 'F-Secure', url: 'https://www.f-secure.com' },},{regex: 'rogerbot',name: 'Rogerbot',category: 'Crawler',url: 'http://moz.com/help/pro/what-is-rogerbot-',- producer: {- name: 'SEOmoz, Inc.',- url: 'http://moz.com/',- },+ producer: { name: 'SEOmoz, Inc.', url: 'http://moz.com/' },},{regex: 'ROI Hunter',name: 'ROI Hunter',category: 'Crawler',url: '',- producer: {- name: 'Roihunter a.s.',- url: 'http://roihunter.com/',- },+ producer: { name: 'Roihunter a.s.', url: 'http://roihunter.com/' },},{regex: 'SafeDNSBot',name: 'SafeDNSBot',category: 'Crawler',url: 'https://www.safedns.com/searchbot',- producer: {- name: 'SafeDNS, Inc.',- url: 'https://www.safedns.com/',- },+ producer: { name: 'SafeDNS, Inc.', url: 'https://www.safedns.com/' },},{regex: 'Scrapy',@@ -1623,50 +1295,56 @@ export default [name: 'ScreenerBot',category: 'Crawler',url: 'http://www.screenerbot.com',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'SemrushBot',- name: 'Semrush Bot',+ name: 'SemrushBot',category: 'Crawler',- url: 'http://www.semrush.com/bot.html',- producer: {- name: 'SEMrush',- url: 'http://www.semrush.com',- },+ url: 'https://www.semrush.com/bot/',+ producer: { name: 'Semrush Inc.', url: 'https://www.semrush.com/' },+ },+ {+ regex: 'SerpReputationManagementAgent/[\\d.]+',+ name: 'Semrush Reputation Management',+ category: 'Service Agent',+ url: 'https://www.semrush.com/bot/',+ producer: { name: 'Semrush Inc.', url: 'https://www.semrush.com/' },+ },+ {+ regex: 'SplitSignalBot',+ name: 'SplitSignalBot',+ category: 'Crawler',+ url: 'https://www.semrush.com/bot/',+ producer: { name: 'Semrush Inc.', url: 'https://www.semrush.com/' },+ },+ {+ regex: 'SiteAuditBot/[\\d.]+',+ name: 'SiteAuditBot',+ category: 'Crawler',+ url: 'https://www.semrush.com/bot/',+ producer: { name: 'Semrush Inc.', url: 'https://www.semrush.com/' },},{regex: 'SensikaBot',name: 'Sensika Bot',category: '',url: '',- producer: {- name: 'Sensika',- url: 'http://sensika.com',- },+ producer: { name: 'Sensika', url: 'http://sensika.com' },},{regex: 'SEOENG(?:World)?Bot',name: 'SEOENGBot',category: 'Crawler',url: 'http://www.seoengine.com/seoengbot.htm',- producer: {- name: 'SEO Engine',- url: 'http://www.seoengine.com',- },+ producer: { name: 'SEO Engine', url: 'http://www.seoengine.com' },},{regex: 'SEOkicks-Robot',name: 'SEOkicks-Robot',category: 'Crawler',url: 'http://www.seokicks.de/robot.html',- producer: {- name: 'SEOkicks',- url: 'https://www.seokicks.de/',- },+ producer: { name: 'SEOkicks', url: 'https://www.seokicks.de/' },},{regex: 'seoscanners\\.net',@@ -1689,70 +1367,49 @@ export default [name: 'Seznam Bot',category: 'Search bot',url: 'http://www.mapy.cz/cz/seznambot.html',- producer: {- name: 'Seznam.cz, a.s.',- url: 'http://www.seznam.cz/',- },+ producer: { name: 'Seznam.cz, a.s.', url: 'http://www.seznam.cz/' },},{regex: 'shopify-partner-homepage-scraper',name: 'Shopify Partner',category: 'Crawler',url: 'https://www.shopify.com/partners',- producer: {- name: 'Shopify',- url: 'https://www.shopify.com/',- },+ producer: { name: 'Shopify', url: 'https://www.shopify.com/' },},{regex: 'ShopWiki',name: 'ShopWiki',category: 'Search tools',url: 'http://www.shopwiki.com/wiki/Help:Bot',- producer: {- name: 'ShopWiki Corp.',- url: 'http://www.shopwiki.com',- },+ producer: { name: 'ShopWiki Corp.', url: 'http://www.shopwiki.com' },},{regex: 'SilverReader',name: 'SilverReader',url: 'http://silverreader.com',category: 'Feed Fetcher',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'SimplePie',name: 'SimplePie',url: 'http://www.simplepie.org',category: 'Feed Parser',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'SISTRIX Crawler',name: 'SISTRIX Crawler',category: 'Crawler',url: 'http://crawler.sistrix.net',- producer: {- name: 'SISTRIX GmbH',- url: 'http://www.sistrix.de',- },+ producer: { name: 'SISTRIX GmbH', url: 'http://www.sistrix.de' },},{regex: 'compatible; (?:SISTRIX )?Optimizer',name: 'SISTRIX Optimizer',category: 'Crawler',url: 'https://optimizer.sistrix.com',- producer: {- name: 'SISTRIX GmbH',- url: 'http://www.sistrix.de',- },+ producer: { name: 'SISTRIX GmbH', url: 'http://www.sistrix.de' },},{regex: 'SiteSucker',@@ -1765,20 +1422,14 @@ export default [name: 'Sixy.ch',category: 'Site Monitor',url: 'http://sixy.ch',- producer: {- name: 'Manuel Kasper',- url: 'https://neon1.net/',- },+ producer: { name: 'Manuel Kasper', url: 'https://neon1.net/' },},{regex: 'Slackbot|Slack-ImgProxy',name: 'Slackbot',category: 'Crawler',url: 'https://api.slack.com/robots',- producer: {- name: 'Slack Technologies',- url: 'http://slack.com',- },+ producer: { name: 'Slack Technologies', url: 'http://slack.com' },},{regex:@@ -1786,40 +1437,28 @@ export default [name: 'Sogou Spider',category: 'Search bot',url: 'http://www.sogou.com/docs/help/webmasters.htm',- producer: {- name: 'Sohu, Inc.',- url: 'http://www.sogou.com',- },+ producer: { name: 'Sohu, Inc.', url: 'http://www.sogou.com' },},{regex: 'Sosospider|Sosoimagespider',name: 'Soso Spider',category: 'Search bot',url: 'http://help.soso.com/webspider.htm',- producer: {- name: 'Tencent Holdings',- url: 'http://www.soso.com',- },+ producer: { name: 'Tencent Holdings', url: 'http://www.soso.com' },},{regex: 'Sprinklr',name: 'Sprinklr',category: 'Crawler',url: '',- producer: {- name: 'Sprinklr, Inc.',- url: 'https://www.sprinklr.com/',- },+ producer: { name: 'Sprinklr, Inc.', url: 'https://www.sprinklr.com/' },},{regex: 'sqlmap/',name: 'sqlmap',category: 'Security Checker',url: 'http://sqlmap.org/',- producer: {- name: 'sqlmap',- url: 'http://sqlmap.org/',- },+ producer: { name: 'sqlmap', url: 'http://sqlmap.org/' },},{regex: 'SSL Labs',@@ -1836,20 +1475,14 @@ export default [name: 'StatusCake',category: 'Site Monitor',url: 'https://www.statuscake.com',- producer: {- name: 'StatusCake',- url: 'https://www.statuscake.com',- },+ producer: { name: 'StatusCake', url: 'https://www.statuscake.com' },},{regex: 'Superfeedr bot',name: 'Superfeedr Bot',category: 'Feed Fetcher',url: '',- producer: {- name: 'Superfeedr',- url: 'https://superfeedr.com/',- },+ producer: { name: 'Superfeedr', url: 'https://superfeedr.com/' },},{regex: 'Sparkler/[0-9]',@@ -1862,17 +1495,9 @@ export default [name: 'Spinn3r',category: 'Crawler',url: 'http://spinn3r.com/robot',- producer: {- name: 'Tailrank Inc',- url: 'http://spinn3r.com',- },- },- {- regex: 'SputnikBot',- name: 'Sputnik Bot',- category: 'Crawler',- url: '',+ producer: { name: 'Tailrank Inc', url: 'http://spinn3r.com' },},+ { regex: 'SputnikBot', name: 'Sputnik Bot', category: 'Crawler', url: '' },{regex: 'SputnikFaviconBot',name: 'Sputnik Favicon Bot',@@ -1890,10 +1515,7 @@ export default [name: 'Survey Bot',category: 'Search bot',url: 'http://www.domaintools.com/webmasters/surveybot.php',- producer: {- name: 'Domain Tools',- url: 'http://www.domaintools.com',- },+ producer: { name: 'Domain Tools', url: 'http://www.domaintools.com' },},{regex: 'TarmotGezgin',@@ -1911,30 +1533,21 @@ export default [name: 'TLSProbe',url: 'https://scan.trustnet.venafi.com/',category: 'Security search bot',- producer: {- name: 'Venafi TrustNet',- url: 'https://www.venafi.com',- },+ producer: { name: 'Venafi TrustNet', url: 'https://www.venafi.com' },},{regex: 'TinEye-bot',name: 'TinEye Crawler',category: 'Search bot',url: 'http://www.tineye.com/crawler.html',- producer: {- name: 'Idée Inc.',- url: 'http://ideeinc.com',- },+ producer: { name: 'Idée Inc.', url: 'http://ideeinc.com' },},{regex: 'Tiny Tiny RSS',name: 'Tiny Tiny RSS',url: 'http://tt-rss.org',category: 'Feed Fetcher',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'theoldreader\\.com',@@ -1943,242 +1556,170 @@ export default [url: 'https://theoldreader.com',},{- regex: 'Trackable/0.1',+ regex: 'Trackable/0\\.1',name: 'Chartable',category: 'Site Monitor',url: 'https://help.chartable.com/article/34-what-is-the-trackable-analytics-prefix',- producer: {- name: 'Chartable',- url: 'https://chartable.com',- },+ producer: { name: 'Chartable', url: 'https://chartable.com' },},{regex: 'trendictionbot',name: 'Trendiction Bot',category: 'Crawler',url: 'http://www.trendiction.de/bot',- producer: {- name: 'Talkwalker Inc.',- url: 'http://www.talkwalker.com',- },+ producer: { name: 'Talkwalker Inc.', url: 'http://www.talkwalker.com' },},{regex: 'TurnitinBot',name: 'TurnitinBot',category: 'Crawler',url: 'http://www.turnitin.com/robot/crawlerinfo.html',- producer: {- name: 'iParadigms, LLC.',- url: 'http://www.turnitin.com',- },+ producer: { name: 'iParadigms, LLC.', url: 'http://www.turnitin.com' },},{- regex: 'TweetedTimes Bot',+ regex: 'TweetedTimes',name: 'TweetedTimes Bot',category: 'Crawler',- url: 'http://tweetedtimes.com',- producer: {- name: 'TweetedTimes',- url: 'http://tweetedtimes.com/',- },+ url: 'https://tweetedtimes.com/',+ producer: { name: 'TweetedTimes', url: 'https://tweetedtimes.com/' },},{regex: 'TweetmemeBot',name: 'Tweetmeme Bot',category: 'Crawler',url: 'http://tweetmeme.com/',- producer: {- name: 'Mediasift',- url: '',- },+ producer: { name: 'Mediasift', url: '' },},{regex: 'Twingly Recon',name: 'Twingly Recon',category: 'Crawler',- producer: {- name: 'Twingly',- url: 'https://www.twingly.com',- },+ producer: { name: 'Twingly', url: 'https://www.twingly.com' },},{regex: 'Twitterbot',name: 'Twitterbot',category: 'Social Media Agent',url: 'https://dev.twitter.com/docs/cards/getting-started',- producer: {- name: 'Twitter',- url: 'http://www.twitter.com',- },+ producer: { name: 'Twitter', url: 'http://www.twitter.com' },},{regex: 'UniversalFeedParser',name: 'UniversalFeedParser',category: 'Feed Fetcher',url: 'https://github.com/kurtmckee/feedparser',- producer: {- name: 'Kurt McKee',- url: 'https://github.com/kurtmckee',- },+ producer: { name: 'Kurt McKee', url: 'https://github.com/kurtmckee' },},{regex: 'via secureurl\\.fwdcdn\\.com',name: 'UkrNet Mail Proxy',category: 'Crawler',url: '',- producer: {- name: 'UkrNet Ltd',- url: 'https://www.ukr.net/',- },+ producer: { name: 'UkrNet Ltd', url: 'https://www.ukr.net/' },},{- regex: 'Uptimebot',+ regex: 'Uptime(?:bot)?/[\\d.]+',name: 'Uptimebot',category: 'Site Monitor',- url: 'https://uptime.com/uptimebot',- producer: {- name: 'Uptime',- url: 'https://uptime.com',- },+ url: 'https://uptime.com/uptime-bot',+ producer: { name: 'Uptime', url: 'https://uptime.com/' },},{regex: 'UptimeRobot',- name: 'Uptime Robot',+ name: 'UptimeRobot',category: 'Site Monitor',- url: '',- producer: {- name: 'Uptime Robot',- url: 'http://uptimerobot.com',- },+ url: 'https://uptimerobot.com/',+ producer: { name: 'Uptime Robot', url: 'https://uptimerobot.com/' },},{regex: 'URLAppendBot',name: 'URLAppendBot',category: 'Crawler',url: 'http://www.profound.net/urlappendbot.html',- producer: {- name: 'Profound Networks',- url: 'http://www.profound.net',- },+ producer: { name: 'Profound Networks', url: 'http://www.profound.net' },},{regex: 'Vagabondo',name: 'Vagabondo',category: 'Crawler',url: '',- producer: {- name: 'WiseGuys',- url: 'http://www.wise-guys.nl/',- },+ producer: { name: 'WiseGuys', url: 'http://www.wise-guys.nl/' },},{regex: 'vkShare; ',name: 'VK Share Button',category: 'Crawler',url: 'https://dev.vk.com/en/widgets/share',- producer: {- name: 'VK',- url: 'https://vk.com/',- },+ producer: { name: 'VK', url: 'https://vk.com/' },},{regex: 'VKRobot',name: 'VK Robot',category: 'Crawler',url: 'https://dev.vk.com/en/',- producer: {- name: 'VK',- url: 'https://vk.com/',- },+ producer: { name: 'VK', url: 'https://vk.com/' },},{regex: 'VSMCrawler',name: 'Visual Site Mapper Crawler',category: 'Crawler',url: 'http://www.visualsitemapper.com/crawler',- producer: {- name: 'Alentum Software Ltd.',- url: 'http://www.alentum.com',- },+ producer: { name: 'Alentum Software Ltd.', url: 'http://www.alentum.com' },},{regex: 'Jigsaw',name: 'W3C CSS Validator',category: 'Validator',url: 'http://jigsaw.w3.org/css-validator',- producer: {- name: 'W3C',- url: 'http://www.w3.org',- },+ producer: { name: 'W3C', url: 'http://www.w3.org' },},{regex: 'W3C_I18n-Checker',name: 'W3C I18N Checker',category: 'Validator',url: 'http://validator.w3.org/i18n-checker',- producer: {- name: 'W3C',- url: 'http://www.w3.org',- },+ producer: { name: 'W3C', url: 'http://www.w3.org' },},{regex: 'W3C-checklink',name: 'W3C Link Checker',category: 'Validator',url: 'http://validator.w3.org/checklink',- producer: {- name: 'W3C',- url: 'http://www.w3.org',- },+ producer: { name: 'W3C', url: 'http://www.w3.org' },},{regex: 'W3C_Validator|Validator\\.nu',name: 'W3C Markup Validation Service',category: 'Validator',url: 'http://validator.w3.org/services',- producer: {- name: 'W3C',- url: 'http://www.w3.org',- },+ producer: { name: 'W3C', url: 'http://www.w3.org' },},{regex: 'W3C-mobileOK',name: 'W3C MobileOK Checker',category: 'Validator',url: 'http://validator.w3.org/mobile',- producer: {- name: 'W3C',- url: 'http://www.w3.org',- },+ producer: { name: 'W3C', url: 'http://www.w3.org' },},{regex: 'W3C_Unicorn',name: 'W3C Unified Validator',category: 'Validator',url: 'http://validator.w3.org/unicorn',- producer: {- name: 'W3C',- url: 'http://www.w3.org',- },+ producer: { name: 'W3C', url: 'http://www.w3.org' },},{regex: 'P3P Validator',name: 'W3C P3P Validator',category: 'Validator',url: 'https://www.w3.org/P3P/validator.html',- producer: {- name: 'W3C',- url: 'https://www.w3.org',- },+ producer: { name: 'W3C', url: 'https://www.w3.org' },},{regex: 'Wappalyzer',name: 'Wappalyzer',url: 'https://github.com/AliasIO/Wappalyzer',- producer: {- name: 'AliasIO',- url: 'https://github.com/AliasIO',- },+ producer: { name: 'AliasIO', url: 'https://github.com/AliasIO' },},{regex: 'PTST/',@@ -2191,171 +1732,155 @@ export default [name: 'WeSEE:Search',category: 'Search bot',url: 'http://www.wesee.com/bot',- producer: {- name: 'WeSEE Ltd',- url: 'http://www.wesee.com',- },+ producer: { name: 'WeSEE Ltd', url: 'http://www.wesee.com' },},{regex: 'WebbCrawler',name: 'WebbCrawler',category: 'Crawler',url: 'http://badcheese.com/crawler.html',- producer: {- name: 'Steve Webb',- url: 'http://badcheese.com',- },+ producer: { name: 'Steve Webb', url: 'http://badcheese.com' },},{regex: 'websitepulse[+ ]checker',name: 'WebSitePulse',category: 'Site Monitor',url: 'http://badcheese.com/crawler.html',- producer: {- name: 'WebSitePulse',- url: 'http://www.websitepulse.com/',- },+ producer: { name: 'WebSitePulse', url: 'http://www.websitepulse.com/' },},{regex: 'WordPress.+isitwp\\.com',name: 'IsItWP',category: 'Crawler',url: 'https://www.isitwp.com/',- producer: {- name: 'WPBeginner, LLC',- url: 'https://www.wpbeginner.com/',- },+ producer: { name: 'WPBeginner, LLC', url: 'https://www.wpbeginner.com/' },+ },+ {+ regex: 'Automattic Analytics Crawler/[\\d.]+',+ name: 'Automattic Analytics',+ category: 'Crawler',+ url: 'https://wordpress.com/crawler/',+ producer: { name: 'Wordpress.org', url: 'https://wordpress.org/' },+ },+ {+ regex: 'WordPress\\.com mShots',+ name: 'WordPress.com mShots',+ category: 'Service Agent',+ url: 'https://wordpress.org/',+ producer: { name: 'Wordpress.org', url: 'https://wordpress.org/' },+ },+ {+ regex: 'wp\\.com feedbot',+ name: 'wp.com feedbot',+ category: 'Feed Fetcher',+ url: 'https://wordpress.com/',+ producer: { name: 'Automattic, Inc.', url: 'https://automattic.com/' },},{regex: 'WordPress',name: 'WordPress',category: 'Service Agent',url: 'https://wordpress.org/',- producer: {- name: 'Wordpress.org',- url: 'https://wordpress.org/',- },+ producer: { name: 'Wordpress.org', url: 'https://wordpress.org/' },},{regex: 'Wotbox',name: 'Wotbox',category: 'Search bot',url: 'http://www.wotbox.com/bot/',- producer: {- name: 'Wotbox',- url: 'http://www.wotbox.com',- },+ producer: { name: 'Wotbox', url: 'http://www.wotbox.com' },},{regex: 'XenForo',name: 'XenForo',category: 'Service Agent',url: 'https://xenforo.com/',- producer: {- name: 'XenForo Ltd.',- url: 'https://xenforo.com/',- },+ producer: { name: 'XenForo Ltd.', url: 'https://xenforo.com/' },},{regex: 'yacybot',name: 'YaCy',category: 'Search bot',url: 'http://yacy.net/bot.html',- producer: {- name: 'YaCy',- url: 'http://yacy.net',- },+ producer: { name: 'YaCy', url: 'http://yacy.net' },},{regex: 'Yahoo! Slurp|Yahoo!-AdCrawler',name: 'Yahoo! Slurp',category: 'Search bot',url: 'http://help.yahoo.com/ysearch/slurp',- producer: {- name: 'Yahoo! Inc.',- url: 'http://www.yahoo.com',- },+ producer: { name: 'Yahoo! Inc.', url: 'http://www.yahoo.com' },},{regex: 'Yahoo Link Preview|Yahoo:LinkExpander:Slingstone',name: 'Yahoo! Link Preview',category: 'Crawler',url: 'https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html',- producer: {- name: 'Yahoo! Inc.',- url: 'http://www.yahoo.com',- },+ producer: { name: 'Yahoo! Inc.', url: 'http://www.yahoo.com' },},{regex: 'YahooMailProxy',name: 'Yahoo! Mail Proxy',category: 'Service Agent',url: 'https://help.yahoo.com/kb/yahoo-mail-proxy-SLN28749.html',- producer: {- name: 'Yahoo! Inc.',- url: 'http://www.yahoo.com',- },+ producer: { name: 'Yahoo! Inc.', url: 'http://www.yahoo.com' },},{regex: 'YahooCacheSystem',name: 'Yahoo! Cache System',category: 'Crawler',url: '',- producer: {- name: 'Yahoo! Inc.',- url: 'http://www.yahoo.com',- },+ producer: { name: 'Yahoo! Inc.', url: 'http://www.yahoo.com' },},{regex: 'Y!J-BRW',name: 'Yahoo! Japan BRW',category: 'Crawler',url: 'https://support.yahoo-net.jp/PccSearch/s/article/H000007955',- producer: {- name: 'Yahoo! Japan Corp.',- url: 'https://www.yahoo.co.jp/',- },+ producer: { name: 'Yahoo! Japan Corp.', url: 'https://www.yahoo.co.jp/' },},{regex: 'Y!J-WSC',name: 'Yahoo! Japan WSC',category: 'Crawler',url: 'https://support.yahoo-net.jp/PccSearch/s/article/H000007955',- producer: {- name: 'Yahoo! Japan Corp.',- url: 'https://www.yahoo.co.jp/',- },+ producer: { name: 'Yahoo! Japan Corp.', url: 'https://www.yahoo.co.jp/' },+ },+ {+ regex: 'Y!J-ASR',+ name: 'Yahoo! Japan ASR',+ category: 'Crawler',+ url: 'https://support.yahoo-net.jp/PccSearch/s/article/H000007955',+ producer: { name: 'Yahoo! Japan Corp.', url: 'https://www.yahoo.co.jp/' },+ },+ {+ regex: '^Y!J',+ name: 'Yahoo! Japan',+ category: 'Crawler',+ url: 'https://support.yahoo-net.jp/PccSearch/s/article/H000007955',+ producer: { name: 'Yahoo! Japan Corp.', url: 'https://www.yahoo.co.jp/' },},{regex:- 'Yandex(?:(?:\\.Gazeta |Accessibility|Mobile|MobileScreenShot|RenderResources|Screenshot|Sprav)?Bot|(?:AdNet|Antivirus|Blogs|Calendar|Catalog|Direct|Favicons|ForDomain|ImageResizer|Images|Market|Media|Metrika|News|OntoDB(?:API)?|Pagechecker|Partner|RCA|SearchShop|(?:News|Site)links|Tracker|Turbo|Verticals|Vertis|Video|Webmaster))|YaDirectFetcher',+ 'Yandex(?:(?:\\.Gazeta |Accessibility|Mobile|MobileScreenShot|RenderResources|Screenshot|Sprav)?Bot|(?:AdNet|Antivirus|Blogs|Calendar|Catalog|Direct|Favicons|ForDomain|ImageResizer|Images|Market|Media|Metrika|News|OntoDB(?:API)?|Pagechecker|Partner|RCA|SearchShop|(?:News|Site)links|Tracker|Turbo|Userproxy|Verticals|Vertis|Video|Webmaster))|YaDirectFetcher',name: 'Yandex Bot',category: 'Search bot',url: 'https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html',- producer: {- name: 'Yandex LLC',- url: 'https://yandex.com/company/',- },+ producer: { name: 'Yandex LLC', url: 'https://yandex.com/company/' },},{regex: 'Yeti|NaverJapan|AdsBot-Naver',name: 'Yeti/Naverbot',category: 'Search bot',url: 'http://help.naver.com/robots/',- producer: {- name: 'Naver',- url: 'http://www.naver.com',- },+ producer: { name: 'Naver', url: 'http://www.naver.com' },},{regex: 'YoudaoBot',name: 'Youdao Bot',category: 'Search bot',url: 'http://www.youdao.com/help/webmaster/spider',- producer: {- name: 'NetEase, Inc.',- url: 'http://corp.163.com',- },+ producer: { name: 'NetEase, Inc.', url: 'http://corp.163.com' },},{regex: 'YOURLS v[0-9]',@@ -2368,10 +1893,7 @@ export default [name: 'Yunyun Bot',category: 'Search bot',url: 'http://www.yunyun.com/SiteInfo.php?r=about',- producer: {- name: 'YunYun',- url: 'http://www.yunyun.com',- },+ producer: { name: 'YunYun', url: 'http://www.yunyun.com' },},{regex: 'zgrab',@@ -2384,140 +1906,105 @@ export default [name: 'Zookabot',category: 'Crawler',url: 'http://zookabot.com',- producer: {- name: 'Hwacha ApS',- url: 'http://hwacha.dk',- },+ producer: { name: 'Hwacha ApS', url: 'http://hwacha.dk' },},{regex: 'ZumBot',name: 'ZumBot',category: 'Search bot',url: 'http://help.zum.com/inquiry',- producer: {- name: 'ZUM internet',- url: 'http://www.zuminternet.com/',- },+ producer: { name: 'ZUM internet', url: 'http://www.zuminternet.com/' },},{regex: 'YottaaMonitor',name: 'Yottaa Site Monitor',category: 'Site Monitor',url: 'http://www.yottaa.com/products/site-monitor',- producer: {- name: 'Yottaa',- url: 'http://www.yottaa.com/',- },+ producer: { name: 'Yottaa', url: 'http://www.yottaa.com/' },},{regex: 'Yahoo Ad monitoring.*yahoo-ad-monitoring-SLN24857',name: 'Yahoo Gemini',category: 'Crawler',url: 'https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html',- producer: {- name: 'Yahoo! Inc.',- url: 'http://www.yahoo.com',- },+ producer: { name: 'Yahoo! Inc.', url: 'http://www.yahoo.com' },},{regex: '.*Java.*outbrain',name: 'Outbrain',category: 'Crawler',url: '',- producer: {- name: 'Outbrain',- url: 'http://www.outbrain.com/',- },+ producer: { name: 'Outbrain', url: 'http://www.outbrain.com/' },},{regex: 'HubPages.*crawlingpolicy',name: 'HubPages',category: 'Crawler',url: 'https://hubpages.com/help/crawlingpolicy',- producer: {- name: 'HubPages, Inc.',- url: 'https://discover.hubpages.com/',- },+ producer: { name: 'HubPages, Inc.', url: 'https://discover.hubpages.com/' },},{regex: 'Pinterest(?:bot)?/[\\d.]+.*www\\.pinterest\\.com',name: 'Pinterest',url: 'https://help.pinterest.com/en/business/article/pinterest-crawler',category: 'Crawler',- producer: {- name: 'Pinterest',- url: 'https://www.pinterest.com/',- },+ producer: { name: 'Pinterest', url: 'https://www.pinterest.com/' },},{- regex: 'Site24x7',+ regex: '.*Site24x7',name: 'Site24x7 Website Monitoring',category: 'Site Monitor',url: 'https://www.site24x7.com/site24x7-faq.html',- producer: {- name: 'Site24x7',- url: 'https://www.site24x7.com',- },+ producer: { name: 'Site24x7', url: 'https://www.site24x7.com' },+ },+ {+ regex: '.* HLB/[\\d.]+',+ name: 'Site24x7 Defacement Monitor',+ category: 'Site Monitor',+ url: 'https://support.site24x7.com/portal/en/kb/articles/default-user-agent-used-in-website-defacement-monitor',+ producer: { name: 'Site24x7', url: 'https://www.site24x7.com/' },},{regex: 's~snapchat-proxy',name: 'Snapchat Proxy',category: 'Crawler',url: 'https://www.snapchat.com',- producer: {- name: 'Snapchat Inc.',- url: 'https://www.snapchat.com',- },+ producer: { name: 'Snapchat Inc.', url: 'https://www.snapchat.com' },},{regex: 'Snap URL Preview Service',name: 'Snap URL Preview Service',category: 'Service Agent',url: 'https://developers.snap.com/robots',- producer: {- name: 'Snapchat Inc.',- url: 'https://www.snapchat.com/',- },+ producer: { name: 'Snapchat Inc.', url: 'https://www.snapchat.com/' },},{regex: 'SnapchatAds/[\\d.]+',name: 'Snapchat Ads',category: 'Crawler',url: 'https://businesshelp.snapchat.com/s/article/adsbot-crawler?language=en_US',- producer: {- name: 'Snapchat Inc.',- url: 'https://www.snapchat.com/',- },+ producer: { name: 'Snapchat Inc.', url: 'https://www.snapchat.com/' },},{regex: "Let's Encrypt validation server",name: "Let's Encrypt Validation",category: 'Service Agent',url: 'https://letsencrypt.org/how-it-works/',- producer: {- name: "Let's Encrypt",- url: 'https://letsencrypt.org',- },+ producer: { name: "Let's Encrypt", url: 'https://letsencrypt.org' },},{regex: 'GrapeshotCrawler',name: 'Grapeshot',category: 'Crawler',url: 'https://www.grapeshot.com/crawler',- producer: {- name: 'Grapeshot',- url: 'https://www.grapeshot.com',- },+ producer: { name: 'Grapeshot', url: 'https://www.grapeshot.com' },},{regex: 'www\\.monitor\\.us',name: 'Monitor.Us',category: 'Site Monitor',url: 'http://www.monitor.us',- producer: {- name: 'Monitor.Us',- url: 'http://www.monitor.us',- },+ producer: { name: 'Monitor.Us', url: 'http://www.monitor.us' },},{regex: 'Catchpoint',@@ -2534,84 +2021,32 @@ export default [name: 'BitlyBot',category: 'Crawler',url: 'https://bitly.com',- producer: {- name: 'Bitly, Inc.',- url: 'https://bitly.com',- },- },- {- regex: 'Zao/',- name: 'Zao',- category: 'Crawler',- },- {- regex: 'lycos',- name: 'Lycos',- },- {- regex: 'Slurp',- name: 'Inktomi Slurp',- },- {- regex: 'Speedy Spider',- name: 'Speedy',- },- {- regex: 'ScoutJet',- name: 'ScoutJet',- },- {- regex: 'nrsbot|netresearch',- name: 'NetResearchServer',- },- {- regex: 'scooter',- name: 'Scooter',- },- {- regex: 'gigabot',- name: 'Gigabot',- },- {- regex: 'charlotte',- name: 'Charlotte',- },- {- regex: 'Pompos',- name: 'Pompos',- },- {- regex: 'ichiro',- name: 'ichiro',- },+ producer: { name: 'Bitly, Inc.', url: 'https://bitly.com' },+ },+ { regex: 'Zao/', name: 'Zao', category: 'Crawler' },+ { regex: 'lycos', name: 'Lycos' },+ { regex: 'Slurp', name: 'Inktomi Slurp' },+ { regex: 'Speedy Spider', name: 'Speedy' },+ { regex: 'ScoutJet', name: 'ScoutJet' },+ { regex: 'nrsbot|netresearch', name: 'NetResearchServer' },+ { regex: 'scooter', name: 'Scooter' },+ { regex: 'gigabot', name: 'Gigabot' },+ { regex: 'charlotte', name: 'Charlotte' },+ { regex: 'Pompos', name: 'Pompos' },+ { regex: 'ichiro', name: 'ichiro' },{regex: 'PagePeeker',name: 'PagePeeker',+ category: 'Crawler',+ url: 'https://pagepeeker.com/robots/',+ producer: { name: 'PAGEPEEKER SRL', url: 'https://pagepeeker.com/' },},- {- regex: 'WebThumbnail',- name: 'WebThumbnail',- },- {- regex: 'Willow Internet Crawler',- name: 'Willow Internet Crawler',- },- {- regex: 'EmailWolf',- name: 'EmailWolf',- },- {- regex: 'NetLyzer FastProbe',- name: 'NetLyzer FastProbe',- },- {- regex: 'AdMantX.*admantx\\.com',- name: 'ADMantX',- },- {- regex: 'Server Density Service Monitoring',- name: 'Server Density',- },+ { regex: 'WebThumbnail', name: 'WebThumbnail' },+ { regex: 'Willow Internet Crawler', name: 'Willow Internet Crawler' },+ { regex: 'EmailWolf', name: 'EmailWolf' },+ { regex: 'NetLyzer FastProbe', name: 'NetLyzer FastProbe' },+ { regex: 'AdMantX.*admantx\\.com', name: 'ADMantX' },+ { regex: 'Server Density Service Monitoring', name: 'Server Density' },{regex: 'RSSRadio \\(Push Notification Scanner;support@dorada\\.co\\.uk\\)',name: 'RSSRadio Bot',@@ -2619,33 +2054,20 @@ export default [{regex: '^sentry',name: 'Sentry Bot',- producer: {- name: 'Sentry',- url: 'https://sentry.io',- },+ producer: { name: 'Sentry', url: 'https://sentry.io' },},{regex: '^Spotify/[\\d.]+$',name: 'Spotify',- producer: {- name: 'Spotify',- url: 'https://www.spotify.com',- },- },- {- regex: 'The Knowledge AI',- name: 'The Knowledge AI',- category: 'Crawler',+ producer: { name: 'Spotify', url: 'https://www.spotify.com' },},+ { regex: 'The Knowledge AI', name: 'The Knowledge AI', category: 'Crawler' },{regex: 'Embedly',name: 'Embedly',category: 'Crawler',url: 'https://support.embed.ly/hc/en-us',- producer: {- name: 'A Medium, Corp.',- url: 'https://medium.com/',- },+ producer: { name: 'A Medium, Corp.', url: 'https://medium.com/' },},{regex: 'BrandVerity',@@ -2662,50 +2084,28 @@ export default [name: 'Kaspersky',category: 'Security Checker',url: 'https://www.kaspersky.com/',- producer: {- name: 'AO Kaspersky Lab',- url: 'https://www.kaspersky.com/',- },+ producer: { name: 'AO Kaspersky Lab', url: 'https://www.kaspersky.com/' },},{regex: 'eZ Publish Link Validator',name: 'eZ Publish Link Validator',category: 'Crawler',url: 'https://ez.no/',- producer: {- name: 'eZ Systems AS',- url: 'https://ez.no/',- },+ producer: { name: 'eZ Systems AS', url: 'https://ez.no/' },},{regex: 'woorankreview',name: 'WooRank',category: 'Search bot',url: 'https://www.woorank.com/',- producer: {- name: 'WooRank sprl',- url: 'https://www.woorank.com/',- },+ producer: { name: 'WooRank sprl', url: 'https://www.woorank.com/' },},{regex: 'by Siteimprove\\.com',name: 'Siteimprove',category: 'Search bot',url: 'https://siteimprove.com/',- producer: {- name: 'Siteimprove GmbH',- url: 'https://siteimprove.com/',- },- },- {- regex: 'Image size by Siteimprove\\.com',- name: 'Siteimprove',- category: 'Search bot',- url: 'https://siteimprove.com/',- producer: {- name: 'Siteimprove GmbH',- url: 'https://siteimprove.com/',- },+ producer: { name: 'Siteimprove GmbH', url: 'https://siteimprove.com/' },},{regex: 'CATExplorador',@@ -2722,70 +2122,49 @@ export default [name: 'Buck',category: 'Search bot',url: 'https://hypefactors.com/',- producer: {- name: 'Hypefactors A/S',- url: 'https://hypefactors.com/',- },+ producer: { name: 'Hypefactors A/S', url: 'https://hypefactors.com/' },},{regex: 'tracemyfile',name: 'TraceMyFile',category: 'Search bot',url: 'https://www.tracemyfile.com/',- producer: {- name: 'Idee Inc.',- url: 'http://ideeinc.com/',- },+ producer: { name: 'Idee Inc.', url: 'http://ideeinc.com/' },},{regex: 'zelist\\.ro feed parser',name: 'Ze List',url: 'https://www.zelist.ro/',category: 'Feed Fetcher',- producer: {- name: 'Treeworks SRL',- url: 'https://www.tree.ro/',- },+ producer: { name: 'Treeworks SRL', url: 'https://www.tree.ro/' },},{regex: 'weborama-fetcher',name: 'Weborama',category: 'Search bot',url: 'https://weborama.com/',- producer: {- name: 'Weborama SA',- url: 'https://weborama.com/',- },+ producer: { name: 'Weborama SA', url: 'https://weborama.com/' },},{regex: 'BoardReader Favicon Fetcher',name: 'BoardReader',category: 'Search bot',url: 'https://boardreader.com/',- producer: {- name: 'Effyis Inc',- url: 'https://boardreader.com/',- },+ producer: { name: 'Effyis Inc', url: 'https://boardreader.com/' },},{regex: 'IDG/IT',name: 'IDG/IT',category: 'Search bot',url: 'https://spaziodati.eu/',- producer: {- name: 'SpazioDati S.r.l.',- url: 'https://spaziodati.eu/',- },+ producer: { name: 'SpazioDati S.r.l.', url: 'https://spaziodati.eu/' },},{regex: 'Bytespider',name: 'Bytespider',category: 'Search bot',url: 'https://bytedance.com/',- producer: {- name: 'ByteDance Ltd.',- url: 'https://bytedance.com/',- },+ producer: { name: 'ByteDance Ltd.', url: 'https://bytedance.com/' },},{regex: 'WikiDo',@@ -2832,10 +2211,7 @@ export default [name: 'SMTBot',category: 'Search bot',url: 'https://www.similartech.com/smtbot',- producer: {- name: 'SimilarTech Ltd.',- url: 'https://www.similartech.com/',- },+ producer: { name: 'SimilarTech Ltd.', url: 'https://www.similartech.com/' },},{regex: 'LCC',@@ -2852,20 +2228,14 @@ export default [name: 'Startpagina Linkchecker',category: 'Search bot',url: 'https://www.startpagina.nl/linkchecker',- producer: {- name: 'Startpagina B.V.',- url: 'https://www.startpagina.nl/',- },+ producer: { name: 'Startpagina B.V.', url: 'https://www.startpagina.nl/' },},{regex: 'MoodleBot-Linkchecker',name: 'MoodleBot Linkchecker',category: 'Search bot',url: 'hhttps://docs.moodle.org/en/Usage',- producer: {- name: 'Moodle Pty Ltd',- url: 'https://moodle.org/',- },+ producer: { name: 'Moodle Pty Ltd', url: 'https://moodle.org/' },},{regex: 'GTmetrix',@@ -2877,6 +2247,13 @@ export default [url: 'https://www.carbon60.com/',},},+ {+ regex: 'CyberFind ?Crawler',+ name: 'CyberFind Crawler',+ category: 'Crawler',+ url: 'https://www.cyberfind.net/bot.html',+ producer: { name: 'Find.tf', url: 'https://find.tf/' },+ },{regex: 'Nutch',name: 'Nutch-based Bot',@@ -2905,11 +2282,7 @@ export default [category: 'Service bot',url: 'https://www.grammarly.com',},- {- regex: 'Robozilla',- name: 'Robozilla',- category: 'Crawler',- },+ { regex: 'Robozilla', name: 'Robozilla', category: 'Crawler' },{regex: 'Domains Project',name: 'Domains Project',@@ -2968,20 +2341,14 @@ export default [name: 'Adbeat',category: 'Crawler',url: 'https://www.adbeat.com/operation_policy',- producer: {- name: 'PPC Labs LLC',- url: 'https://www.adbeat.com/',- },+ producer: { name: 'PPC Labs LLC', url: 'https://www.adbeat.com/' },},{- regex: 'BW/[\\d.]+',+ regex: '(?:BuiltWith|BW)/[\\d.]+',name: 'BuiltWith',category: 'Crawler',url: 'https://builtwith.com/biup',- producer: {- name: 'BuiltWith Pty Ltd',- url: 'https://builtwith.com/',- },+ producer: { name: 'BuiltWith Pty Ltd', url: 'https://builtwith.com/' },},{regex: 'https://whatis\\.contentkingapp\\.com',@@ -2998,10 +2365,7 @@ export default [name: 'MicroAdBot',category: 'Crawler',url: 'https://www.microad.co.jp/',- producer: {- name: 'MicroAd, Inc.',- url: 'https://www.microad.co.jp/',- },+ producer: { name: 'MicroAd, Inc.', url: 'https://www.microad.co.jp/' },},{regex: 'PingAdmin\\.Ru',@@ -3030,10 +2394,7 @@ export default [name: 'parse.ly',category: 'Crawler',url: 'https://www.parse.ly/help/integration/crawler',- producer: {- name: 'Parsely, Inc.',- url: 'https://www.parse.ly/',- },+ producer: { name: 'Parsely, Inc.', url: 'https://www.parse.ly/' },},{regex: 'Nimbostratus-Bot',@@ -3051,17 +2412,15 @@ export default [regex: 'Project-Resonance',name: 'Project Resonance',category: 'Crawler',- url: 'http://project-resonance.com',+ url: 'https://project-resonance.com/',+ producer: { name: 'RedHunt Labs Limited', url: 'https://redhuntlabs.com/' },},{regex: 'DataXu/[\\d.]+',name: 'DataXu',category: 'Service Agent',url: 'https://advertising.roku.com/dataxu',- producer: {- name: 'Roku, Inc.',- url: 'https://roku.com',- },+ producer: { name: 'Roku, Inc.', url: 'https://roku.com' },},{regex: 'Cocolyzebot',@@ -3078,20 +2437,14 @@ export default [name: 'VeryHip',category: 'Crawler',url: 'https://veryhip.com/',- producer: {- name: 'VeryHip',- url: 'https://veryhip.com/',- },+ producer: { name: 'VeryHip', url: 'https://veryhip.com/' },},{regex: 'LinkpadBot',name: 'LinkpadBot',category: 'Crawler',url: 'https://www.linkpad.org/',- producer: {- name: 'Solomono LLC',- url: 'https://www.linkpad.org/',- },+ producer: { name: 'Solomono LLC', url: 'https://www.linkpad.org/' },},{regex: 'MuscatFerret',@@ -3114,40 +2467,28 @@ export default [name: 'ArchiveBox',url: 'https://archivebox.io/',category: 'Crawler',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'Choosito',name: 'Choosito',url: 'https://www.choosito.com/',category: 'Crawler',- producer: {- name: 'Choosito! Inc.',- url: 'https://www.choosito.com/',- },+ producer: { name: 'Choosito! Inc.', url: 'https://www.choosito.com/' },},{regex: 'datagnionbot',name: 'datagnionbot',url: 'https://www.datagnion.com/bot.html',category: 'Crawler',- producer: {- name: 'DATAGNION GMBH',- url: 'https://www.datagnion.com/',- },+ producer: { name: 'DATAGNION GMBH', url: 'https://www.datagnion.com/' },},{regex: 'WhatCMS',name: 'WhatCMS',url: 'https://whatcms.org/',category: 'Crawler',- producer: {- name: 'Nineteen Ten LLC',- url: 'https://whatcms.org/',- },+ producer: { name: 'Nineteen Ten LLC', url: 'https://whatcms.org/' },},{regex: 'httpx',@@ -3155,8 +2496,18 @@ export default [url: 'https://github.com/projectdiscovery/httpx',category: 'Crawler',producer: {- name: '',- url: '',+ name: 'ProjectDiscovery, Inc.',+ url: 'https://projectdiscovery.io/',+ },+ },+ {+ regex: '.*\\.oast\\.',+ name: 'Interactsh',+ category: 'Security Checker',+ url: 'https://github.com/projectdiscovery/interactsh',+ producer: {+ name: 'ProjectDiscovery, Inc.',+ url: 'https://projectdiscovery.io/',},},{@@ -3164,10 +2515,7 @@ export default [name: 'Expanse',category: 'Security Checker',url: 'https://expanse.co/',- producer: {- name: 'Expanse Inc.',- url: 'https://expanse.co/',- },+ producer: { name: 'Expanse Inc.', url: 'https://expanse.co/' },},{regex: 'HuaweiWebCatBot',@@ -3184,20 +2532,14 @@ export default [name: 'Hatena Favicon',category: 'Crawler',url: 'https://www.hatena.ne.jp/faq/',- producer: {- name: 'Hatena Co., Ltd.',- url: 'https://www.hatena.ne.jp',- },+ producer: { name: 'Hatena Co., Ltd.', url: 'https://www.hatena.ne.jp' },},{regex: 'Hatena-?Bookmark',name: 'Hatena Bookmark',category: 'Crawler',url: 'https://www.hatena.ne.jp/faq/',- producer: {- name: 'Hatena Co., Ltd.',- url: 'https://www.hatena.ne.jp',- },+ producer: { name: 'Hatena Co., Ltd.', url: 'https://www.hatena.ne.jp' },},{regex: 'RyowlEngine/[\\d.]+',@@ -3288,10 +2630,7 @@ export default [name: 'Cookiebot',category: 'Crawler',url: 'https://support.cookiebot.com/hc/en-us/articles/360014264140-Scanner-User-Agent',- producer: {- name: 'Cybot A/S',- url: 'https://www.cybot.com/',- },+ producer: { name: 'Cybot A/S', url: 'https://www.cybot.com/' },},{regex: 'NetSystemsResearch',@@ -3308,10 +2647,7 @@ export default [name: 'CensysInspect',category: 'Security Checker',url: 'https://about.censys.io/',- producer: {- name: 'Censys, Inc.',- url: 'https://censys.io/',- },+ producer: { name: 'Censys, Inc.', url: 'https://censys.io/' },},{regex: 'gdnplus\\.com',@@ -3340,20 +2676,14 @@ export default [name: 'MTRobot',category: 'Crawler',url: 'https://metrics-tools.de/robot.html',- producer: {- name: 'Metrics Tools',- url: 'https://metrics-tools.de/',- },+ producer: { name: 'Metrics Tools', url: 'https://metrics-tools.de/' },},{regex: 'serpstatbot/[\\d.]+',name: 'serpstatbot',category: 'Crawler',url: 'http://serpstatbot.com/',- producer: {- name: 'Netpeak Ltd',- url: 'https://netpeak.net/',- },+ producer: { name: 'Netpeak Ltd', url: 'https://netpeak.net/' },},{regex: 'colly',@@ -3378,10 +2708,7 @@ export default [name: 'LeakIX',category: 'Security Checker',url: 'https://leakix.net/',- producer: {- name: 'BaDaaS SRL',- url: 'https://leakix.net/',- },+ producer: { name: 'BaDaaS SRL', url: 'https://leakix.net/' },},{regex: 'MegaIndex\\.ru/[\\d.]+',@@ -3394,10 +2721,7 @@ export default [name: 'Seekport',category: 'Crawler',url: 'https://bot.seekport.com/',- producer: {- name: 'SISTRIX GmbH',- url: 'https://www.sistrix.de/',- },+ producer: { name: 'SISTRIX GmbH', url: 'https://www.sistrix.de/' },},{regex: 'seolyt/[\\d.]+',@@ -3410,30 +2734,28 @@ export default [name: 'YaK',category: 'Crawler',url: 'https://www.linkfluence.com/',- producer: {- name: 'Linkfluence SAS',- url: 'https://www.linkfluence.com/',- },+ producer: { name: 'Linkfluence SAS', url: 'https://www.linkfluence.com/' },},{regex: 'KomodiaBot/[\\d.]+',name: 'KomodiaBot',category: 'Crawler',url: 'http://www.komodia.com/newwiki/index.php/URL_server_crawler',- producer: {- name: 'Komodia Inc.',- url: 'https://www.komodia.com/',- },+ producer: { name: 'Komodia Inc.', url: 'https://www.komodia.com/' },+ },+ {+ regex: 'KStandBot/[\\d.]+',+ name: 'KStandBot',+ category: 'Crawler',+ url: 'https://url-classification.io/wiki/index.php?title=URL_server_crawler',+ producer: { name: 'Komodia Inc.', url: 'https://www.komodia.com/' },},{regex: 'Neevabot/[\\d.]+',name: 'Neevabot',category: 'Search bot',url: 'https://neeva.com/neevabot',- producer: {- name: 'Neeva Inc.',- url: 'https://neeva.com/',- },+ producer: { name: 'Neeva Inc.', url: 'https://neeva.com/' },},{regex: 'LinkPreview/[\\d.]+',@@ -3452,10 +2774,7 @@ export default [name: 'RocketMonitorBot',category: 'Site Monitor',url: 'https://www.radiomast.io/docs/stream-monitoring/technical_details.html',- producer: {- name: 'Radio Mast, Inc.',- url: 'https://www.radiomast.io/',- },+ producer: { name: 'Radio Mast, Inc.', url: 'https://www.radiomast.io/' },},{regex: 'SitemapParser-VIPnytt/[\\d.]+',@@ -3475,11 +2794,7 @@ export default [category: 'Site Monitor',url: 'https://www.dotcom-monitor.com',},- {- regex: 'ThinkChaos/',- name: 'ThinkChaos',- category: 'Crawler',- },+ { regex: 'ThinkChaos/', name: 'ThinkChaos', category: 'Crawler' },{regex: 'DataForSeoBot',name: 'DataForSeoBot',@@ -3519,10 +2834,7 @@ export default [name: 'IONOS Crawler',category: 'Crawler',url: 'https://www.ionos.de/terms-gtc/faq-crawler-en/',- producer: {- name: 'IONOS SE',- url: 'https://www.ionos.de/',- },+ producer: { name: 'IONOS SE', url: 'https://www.ionos.de/' },},{regex: 'Crawldad',@@ -3577,33 +2889,24 @@ export default [name: 'Onalytica',category: 'Crawler',url: 'https://www.airslate.com/bot/explore/onalytica-bot',- producer: {- name: 'airSlate, Inc.',- url: 'https://www.airslate.com/',- },+ producer: { name: 'airSlate, Inc.', url: 'https://www.airslate.com/' },},{regex: 'deepnoc',name: 'deepnoc',category: 'Crawler',url: 'https://deepnoc.com/bot',- producer: {- name: 'deepnoc, GmbH',- url: 'https://deepnoc.com/',- },+ producer: { name: 'deepnoc, GmbH', url: 'https://deepnoc.com/' },},{regex: 'Newslitbot/[\\d.]+',name: 'Newslitbot',category: 'Crawler',url: 'https://www.newslit.co/',- producer: {- name: 'Newslit, LLC.',- url: 'https://www.newslit.co/',- },+ producer: { name: 'Newslit, LLC.', url: 'https://www.newslit.co/' },},{- regex: 'um-LN/[\\d.]+',+ regex: 'um-(?:ANS|CC|FC|IC|LN)/[\\d.]+',name: 'uMBot',category: 'Crawler',url: 'https://www.ubermetrics-technologies.com/',@@ -3623,10 +2926,7 @@ export default [name: 'Infegy',category: 'Crawler',url: 'https://infegy.com/',- producer: {- name: 'Infegy, Inc.',- url: 'https://infegy.com/',- },+ producer: { name: 'Infegy, Inc.', url: 'https://infegy.com/' },},{regex: 'HTTP Banner Detection \\(https://security\\.ipip\\.net\\)',@@ -3653,20 +2953,14 @@ export default [name: 'WebPros',category: 'Crawler',url: 'https://webpros.com/',- producer: {- name: 'WebPros Holdco B.V.',- url: 'https://webpros.com/',- },+ producer: { name: 'WebPros Holdco B.V.', url: 'https://webpros.com/' },},{regex: 'ELB-HealthChecker',name: 'Amazon ELB',category: 'Site Monitor',url: 'https://aws.amazon.com/elasticloadbalancing/',- producer: {- name: 'Amazon.com, Inc.',- url: 'https://www.amazon.com/',- },+ producer: { name: 'Amazon.com, Inc.', url: 'https://www.amazon.com/' },},{regex: 'Wheregoes\\.com Redirect Checker/[\\d.]+',@@ -3721,30 +3015,21 @@ export default [name: 'adstxtlab.com',category: 'Crawler',url: 'https://adstxtlab.com/validator.php',- producer: {- name: 'Jaohawi AB',- url: 'https://adstxtlab.com/',- },+ producer: { name: 'Jaohawi AB', url: 'https://adstxtlab.com/' },},{regex: 'Iframely/[\\d.]+',name: 'Iframely',category: 'Crawler',url: 'https://iframely.com/',- producer: {- name: 'Itteco Software, Corp.',- url: 'https://iframely.com/',- },+ producer: { name: 'Itteco Software, Corp.', url: 'https://iframely.com/' },},{regex: 'DomainStatsBot/[\\d.]+',name: 'DomainStatsBot',category: 'Crawler',url: 'https://domainstats.com/pages/our-bot',- producer: {- name: 'Domainstats Ltd',- url: 'https://domainstats.com/',- },+ producer: { name: 'Domainstats Ltd', url: 'https://domainstats.com/' },},{regex: 'aiHitBot/[\\d.]+',@@ -3758,16 +3043,8 @@ export default [category: 'Crawler',url: 'https://domaincrawler.com/about-us/',},- {- regex: 'DNSResearchBot',- name: 'DNSResearchBot',- category: 'Crawler',- },- {- regex: 'GitCrawlerBot',- name: 'GitCrawlerBot',- category: 'Crawler',- },+ { regex: 'DNSResearchBot', name: 'DNSResearchBot', category: 'Crawler' },+ { regex: 'GitCrawlerBot', name: 'GitCrawlerBot', category: 'Crawler' },{regex: 'AdAuth/[\\d.]+',name: 'AdAuth',@@ -3797,10 +3074,7 @@ export default [name: 'PayPal IPN',category: 'Service Agent',url: 'https://developer.paypal.com/api/nvp-soap/ipn/IPNIntro/',- producer: {- name: 'PayPal, Inc.',- url: 'https://www.paypal.com/',- },+ producer: { name: 'PayPal, Inc.', url: 'https://www.paypal.com/' },},{regex: 'MaCoCu',@@ -3812,6 +3086,16 @@ export default [url: 'https://www.ijs.si/ijsw/JSI',},},+ {+ regex: 'CLASSLA',+ name: 'CLASSLA-web',+ category: 'Crawler',+ url: 'https://www.clarin.si/info/classla-web-crawler/',+ producer: {+ name: 'Jožef Stefan Institute',+ url: 'https://www.ijs.si/ijsw/JSI',+ },+ },{regex: 'dnt-policy@eff\\.org',name: 'EFF Do Not Track Verifier',@@ -3827,10 +3111,7 @@ export default [name: 'InfoTigerBot',category: 'Crawler',url: 'https://infotiger.com/bot',- producer: {- name: 'Infotiger UG',- url: 'https://infotiger.com/',- },+ producer: { name: 'Infotiger UG', url: 'https://infotiger.com/' },},{regex: '(?:Birdcrawlerbot|CrawlaDeBot)',@@ -3857,10 +3138,7 @@ export default [name: 'Zaldamo',category: 'Crawler',url: 'https://www.zaldamo.com/search.html',- producer: {- name: 'Zaldamo, LLC.',- url: 'https://www.zaldamo.com/',- },+ producer: { name: 'Zaldamo, LLC.', url: 'https://www.zaldamo.com/' },},{regex: 'AFB/[\\d.]+',@@ -3879,10 +3157,7 @@ export default [name: 'LinkWalker',category: 'Crawler',url: 'https://www.phishlabs.com/',- producer: {- name: 'PhishLabs, Inc.',- url: 'https://www.phishlabs.com/',- },+ producer: { name: 'PhishLabs, Inc.', url: 'https://www.phishlabs.com/' },},{regex: 'RenovateBot/[\\d.]+',@@ -3905,30 +3180,14 @@ export default [name: 'NETZZAPPEN',category: 'Crawler',url: 'https://www.netzzappen.com/',- producer: {- name: 'Marc Huemer',- url: 'https://www.netzzappen.com/',- },- },- {- regex: 'SerpReputationManagementAgent/[\\d.]+',- name: 'SEMrush Reputation Management',- category: 'Service Agent',- url: 'https://www.semrush.com/bot/',- producer: {- name: 'SEMrush',- url: 'https://www.semrush.com/',- },+ producer: { name: 'Marc Huemer', url: 'https://www.netzzappen.com/' },},{regex: 'panscient\\.com',name: 'Panscient',category: 'Crawler',url: 'https://www.panscient.com/faq.htm',- producer: {- name: 'Panscient, Inc.',- url: 'https://www.panscient.com/',- },+ producer: { name: 'Panscient, Inc.', url: 'https://www.panscient.com/' },},{regex: 'research@pdrlabs\\.net',@@ -3945,110 +3204,77 @@ export default [name: 'NiceCrawler',category: 'Crawler',url: 'https://www.nicecrawler.com/',- producer: {- name: 'Intelium Corp.',- url: 'https://www.intelium.com/',- },+ producer: { name: 'Intelium Corp.', url: 'https://www.intelium.com/' },},{regex: 't3versionsBot/[\\d.]+',name: 't3versions',category: 'Crawler',url: 'https://www.t3versions.com/bot',- producer: {- name: 'Torben Hansen',- url: 'https://www.t3versions.com/',- },+ producer: { name: 'Torben Hansen', url: 'https://www.t3versions.com/' },},{regex: 'Crawlson/[\\d.]+',name: 'Crawlson',category: 'Crawler',url: 'https://www.crawlson.com/about',- producer: {- name: 'Crawlson',- url: 'https://www.crawlson.com/',- },+ producer: { name: 'Crawlson', url: 'https://www.crawlson.com/' },},{regex: 'tchelebi/[\\d.]+',name: 'tchelebi',category: 'Crawler',url: 'https://tchelebi.io/',- producer: {- name: 'NormShield, Inc.',- url: 'https://blackkite.com/',- },+ producer: { name: 'NormShield, Inc.', url: 'https://blackkite.com/' },},{regex: 'JobboerseBot',name: 'JobboerseBot',category: 'Crawler',url: 'https://www.xing.com/jobs',- producer: {- name: 'New Work SE',- url: 'https://www.xing.com/',- },+ producer: { name: 'New Work SE', url: 'https://www.xing.com/' },},{- regex: 'RepoLookoutBot/[\\d.]+',+ regex: 'RepoLookoutBot/v?[\\d.]+',name: 'Repo Lookout',category: 'Security Checker',url: 'https://www.repo-lookout.org/',- producer: {- name: 'Crissy Field GmbH',- url: 'https://www.crissyfield.de/',- },+ producer: { name: 'Crissy Field GmbH', url: 'https://www.crissyfield.de/' },},{regex: 'PATHspider',name: 'PATHspider',category: 'Security Checker',url: 'https://pathspider.net/',- producer: {- name: 'MAMI Project',- url: 'https://mami-project.eu/',- },+ producer: { name: 'MAMI Project', url: 'https://mami-project.eu/' },},{regex: 'everyfeed-spider/[\\d.]+',name: 'Everyfeed',url: 'https://web.archive.org/web/20050930235914/http://www.everyfeed.com/',category: 'Feed Fetcher',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'Exchange check',name: 'Exchange check',category: 'Security Checker',url: 'https://github.com/GossiTheDog/scanning',- producer: {- name: 'Kevin Beaumont',- url: 'https://doublepulsar.com/',- },+ producer: { name: 'Kevin Beaumont', url: 'https://doublepulsar.com/' },},{regex: 'Sublinq',name: 'Sublinq',category: 'Crawler',url: 'https://web.archive.org/web/20220626191617/https://sublinq.com/',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'Gregarius/[\\d.]+',name: 'Gregarius',category: 'Feed Fetcher',url: 'https://web.archive.org/web/20100614011837/http://devlog.gregarius.net/docs/ua/',- producer: {- name: '',- url: '',- },+ producer: { name: '', url: '' },},{regex: 'COMODO DCV',@@ -4061,14 +3287,11 @@ export default [},},{- regex: 'Sectigo DCV',+ regex: 'Sectigo DCV|acme\\.sectigo\\.com',name: 'Sectigo DCV',category: 'Service Agent',url: 'https://sectigo.com/',- producer: {- name: 'Sectigo Limited',- url: 'https://sectigo.com/',- },+ producer: { name: 'Sectigo Limited', url: 'https://sectigo.com/' },},{regex:@@ -4076,40 +3299,28 @@ export default [name: 'KlarnaBot',category: 'Crawler',url: 'https://docs.klarna.com/klarna-bot/',- producer: {- name: 'Klarna Bank AB',- url: 'https://www.klarna.com/',- },+ producer: { name: 'Klarna Bank AB', url: 'https://www.klarna.com/' },},{regex: 'Taboolabot/[\\d.]+',name: 'Taboolabot',category: 'Crawler',url: 'https://help.taboola.com/hc/en-us/articles/115002347594-The-Taboola-Crawler',- producer: {- name: 'Taboola, Inc.',- url: 'https://www.taboola.com/',- },+ producer: { name: 'Taboola, Inc.', url: 'https://www.taboola.com/' },},{regex: 'Asana/[\\d.]+',name: 'Asana',category: 'Crawler',url: 'https://asana.com/',- producer: {- name: 'Asana, Inc.',- url: 'https://asana.com/',- },+ producer: { name: 'Asana, Inc.', url: 'https://asana.com/' },},{regex: 'Chrome Privacy Preserving Prefetch Proxy',name: 'Chrome Privacy Preserving Prefetch Proxy',category: 'Service Agent',url: 'https://developer.chrome.com/blog/private-prefetch-proxy/',- producer: {- name: 'Google Inc.',- url: 'https://www.google.com/',- },+ producer: { name: 'Google Inc.', url: 'https://www.google.com/' },},{regex: 'URLinspectorBot/[\\d.]+',@@ -4126,30 +3337,21 @@ export default [name: 'Entfer',category: 'Crawler',url: 'https://entfer.com/',- producer: {- name: 'Entfer Ltd.',- url: 'https://entfer.com/',- },+ producer: { name: 'Entfer Ltd.', url: 'https://entfer.com/' },},{regex: 'TagInspector/[\\d.]+',name: 'Tag Inspector',category: 'Crawler',url: 'https://taginspector.com/',- producer: {- name: 'InfoTrust, LLC',- url: 'https://infotrust.com/',- },+ producer: { name: 'InfoTrust, LLC', url: 'https://infotrust.com/' },},{regex: 'pageburst',name: 'Pageburst',category: 'Crawler',url: 'https://pageburstls.elsevier.com/',- producer: {- name: 'Elsevier Ltd',- url: 'https://www.elsevier.com/',- },+ producer: { name: 'Elsevier Ltd', url: 'https://www.elsevier.com/' },},{regex: '.+diffbot',@@ -4166,20 +3368,14 @@ export default [name: 'Disqus',category: 'Crawler',url: 'https://help.disqus.com/en/articles/1765357-ads-txt-implementation-guide',- producer: {- name: 'Disqus, Inc.',- url: 'https://disqus.com/',- },+ producer: { name: 'Disqus, Inc.', url: 'https://disqus.com/' },},{regex: 'startmebot/[\\d.]+',name: 'start.me',category: 'Crawler',url: 'https://about.start.me/',- producer: {- name: 'start.me BV',- url: 'https://about.start.me/',- },+ producer: { name: 'start.me BV', url: 'https://about.start.me/' },},{regex: '2ip bot/[\\d.]+',@@ -4198,10 +3394,7 @@ export default [name: 'XoviBot',category: 'Crawler',url: 'https://www.xovibot.net',- producer: {- name: 'Xovi GmbH',- url: 'http://www.xovi.de',- },+ producer: { name: 'Xovi GmbH', url: 'http://www.xovi.de' },},{regex: 'Overcast/[\\d.]+ Podcast Sync',@@ -4272,10 +3465,7 @@ export default [name: 'Morningscore Bot',category: 'Crawler',url: 'https://morningscore.io/',- producer: {- name: 'Morningscore',- url: 'https://morningscore.io/',- },+ producer: { name: 'Morningscore', url: 'https://morningscore.io/' },},{regex: 'Uptime-Kuma/[\\d.]+',@@ -4283,15 +3473,26 @@ export default [category: 'Site Monitor',url: 'https://github.com/louislam/uptime-kuma',},+ {+ regex: 'OAI-SearchBot',+ name: 'OAI-SearchBot',+ category: 'Crawler',+ url: 'https://platform.openai.com/docs/bots',+ producer: { name: 'OpenAI OpCo, LLC', url: 'https://openai.com/' },+ },+ {+ regex: 'GPTBot/[\\d.]+',+ name: 'GPTBot',+ category: 'Crawler',+ url: 'https://platform.openai.com/docs/bots',+ producer: { name: 'OpenAI OpCo, LLC', url: 'https://openai.com/' },+ },{regex: 'ChatGPT-User',- name: 'ChatGPT',+ name: 'ChatGPT-User',category: 'Crawler',- url: 'https://platform.openai.com/docs/plugins/bot',- producer: {- name: 'OpenAI OpCo, LLC',- url: 'https://openai.com/',- },+ url: 'https://platform.openai.com/docs/bots',+ producer: { name: 'OpenAI OpCo, LLC', url: 'https://openai.com/' },},{regex: 'BrightEdge Crawler/[\\d.]+',@@ -4314,30 +3515,7 @@ export default [name: 'Cyberscan',category: 'Security Checker',url: 'https://www.cyberscan.io/',- producer: {- name: 'DGC Verwaltungs GmbH',- url: 'https://dgc.org/',- },- },- {- regex: 'deepcrawl\\.com',- name: 'Lumar',- category: 'Crawler',- url: 'https://deepcrawl.com/bot',- producer: {- name: 'Lumar',- url: 'https://www.lumar.io/',- },- },- {- regex: 'RepoLookoutBot',- name: 'Repo Lookout',- category: 'Crawler',- url: 'https://www.repo-lookout.org/',- producer: {- name: 'Crissy Field GmbH',- url: 'https://www.crissyfield.de/',- },+ producer: { name: 'DGC Verwaltungs GmbH', url: 'https://dgc.org/' },},{regex: 'researchscan\\.comsys\\.rwth-aachen\\.de',@@ -4354,40 +3532,21 @@ export default [name: 'Scraping Robot',category: 'Crawler',url: 'https://scrapingrobot.com/',- producer: {- name: 'Sprious LLC',- url: 'https://sprious.com/',- },- },- {- regex: 'GPTBot/[\\d.]+',- name: 'GPTBot',- category: 'Crawler',- url: 'https://platform.openai.com/docs/gptbot',- producer: {- name: 'OpenAI OpCo, LLC',- url: 'https://openai.com/',- },+ producer: { name: 'Sprious LLC', url: 'https://sprious.com/' },},{regex: 'Ant(?:\\.com beta|Bot)(?:/([\\d+.]+))?',name: 'Ant',category: 'Crawler',url: 'https://www.ant.com/',- producer: {- name: 'Ant.com Ltd.',- url: 'https://www.ant.com/',- },+ producer: { name: 'Ant.com Ltd.', url: 'https://www.ant.com/' },},{regex: 'WebwikiBot/[\\d.]+',name: 'Webwiki',category: 'Crawler',url: 'https://www.webwiki.com/',- producer: {- name: 'webwiki GmbH',- url: 'https://www.webwiki.com/',- },+ producer: { name: 'webwiki GmbH', url: 'https://www.webwiki.com/' },},{regex: 'phpMyAdmin',@@ -4400,30 +3559,21 @@ export default [name: 'Matomo',category: 'Service Agent',url: 'https://github.com/matomo-org/matomo',- producer: {- name: 'InnoCraft Ltd',- url: 'https://matomo.org/',- },+ producer: { name: 'InnoCraft Ltd', url: 'https://matomo.org/' },},{regex: 'Prometheus/[\\d.]+',name: 'Prometheus',category: 'Service Agent',url: 'https://github.com/prometheus/prometheus',- producer: {- name: 'The Linux Foundation',- url: 'https://www.cncf.io/',- },+ producer: { name: 'The Linux Foundation', url: 'https://www.cncf.io/' },},{regex: 'ArchiveTeam ArchiveBot',name: 'ArchiveBot',category: 'Crawler',url: 'https://wiki.archiveteam.org/index.php?title=ArchiveBot',- producer: {- name: 'ArchiveTeam',- url: 'https://wiki.archiveteam.org/',- },+ producer: { name: 'ArchiveTeam', url: 'https://wiki.archiveteam.org/' },},{regex: 'MADBbot/[\\d.]+',@@ -4441,7 +3591,7 @@ export default [},},{- regex: '(?:Owler@ows\\.eu|OWLer)/[\\d.]+',+ regex: 'owler',name: 'OWLer',category: 'Crawler',url: 'https://openwebsearch.eu/owler/',@@ -4455,20 +3605,14 @@ export default [name: 'BBC Page Monitor',category: 'Site Monitor',url: 'https://confluence.dev.bbc.co.uk/display/men/Page+Monitor',- producer: {- name: 'BBC',- url: 'https://www.bbc.com/',- },+ producer: { name: 'BBC', url: 'https://www.bbc.com/' },},{regex: 'BBC-Forge-URL-Monitor-Twisted',name: 'BBC Forge URL Monitor',category: 'Site Monitor',url: 'https://www.bbc.com/',- producer: {- name: 'BBC',- url: 'https://www.bbc.com/',- },+ producer: { name: 'BBC', url: 'https://www.bbc.com/' },},{regex: 'ClaudeBot',@@ -4481,19 +3625,14 @@ export default [name: 'ImageSift',category: 'Crawler',url: 'https://imagesift.com/',- producer: {- name: 'Castle Global, Inc.',- url: 'https://thehive.ai/',- },+ producer: { name: 'Castle Global, Inc.', url: 'https://thehive.ai/' },},{regex: 'TactiScout',name: 'TactiScout',category: 'Crawler',url: 'https://find-it.world/TempCrawl/Crawltheque.php',- producer: {- name: 'Tactikast',- },+ producer: { name: 'Tactikast' },},{regex: 'Brightbot ([\\d+.]+)',@@ -4510,20 +3649,14 @@ export default [name: 'DaspeedBot',category: 'Crawler',url: 'https://daspeed.io/',- producer: {- name: 'DAWAP SARL',- url: 'https://dawap.fr/',- },+ producer: { name: 'DAWAP SARL', url: 'https://dawap.fr/' },},{regex: 'StractBot(?:/([\\d+.]+))?',name: 'Stract',category: 'Crawler',url: 'https://stract.com/webmasters',- producer: {- name: 'Stract',- url: 'https://github.com/StractOrg/stract/',- },+ producer: { name: 'Stract', url: 'https://github.com/StractOrg/stract/' },},{regex: 'GeedoBot(?:/([\\d+.]+))?',@@ -4531,35 +3664,32 @@ export default [category: 'Crawler',url: 'https://geedo.com/bot/',},+ {+ regex: 'GeedoProductSearch',+ name: 'GeedoProductSearch',+ category: 'Crawler',+ url: 'https://geedo.com/product-search/',+ },{regex: 'BackupLand(?:/([\\d+.]+))?',name: 'BackupLand',category: 'Crawler',url: 'https://go.backupland.com/',- producer: {- name: 'ООО «КВАРТА»',- url: 'https://go.backupland.com/',- },+ producer: { name: 'ООО «КВАРТА»', url: 'https://go.backupland.com/' },},{regex: 'Konturbot(?:/([\\d+.]+))?',name: 'Konturbot',category: 'Crawler',url: 'https://kontur.ru/',- producer: {- name: 'АО «ПФ «СКБ Контур»',- url: 'https://kontur.ru/',- },+ producer: { name: 'АО «ПФ «СКБ Контур»', url: 'https://kontur.ru/' },},{regex: 'keys-so-bot',name: 'Keys.so',category: 'Crawler',url: 'https://www.keys.so/',- producer: {- name: 'ООО «МОДЕСКО»',- url: 'https://www.modesco.ru/',- },+ producer: { name: 'ООО «МОДЕСКО»', url: 'https://www.modesco.ru/' },},{regex: 'LetsearchBot(?:/([\\d+.]+))?',@@ -4588,40 +3718,28 @@ export default [name: 'Spawning AI',category: 'Crawler',url: 'https://spawning.ai/',- producer: {- name: 'Spawning, Inc',- url: 'https://spawning.ai/',- },+ producer: { name: 'Spawning, Inc', url: 'https://spawning.ai/' },},{regex: 'domain research project',name: 'Domain Research Project',category: 'Crawler',url: 'https://trentwil.es/domains.html',- producer: {- name: 'Trent Wiles',- url: 'https://trentwil.es/',- },+ producer: { name: 'Trent Wiles', url: 'https://trentwil.es/' },},{regex: 'getodin\\.com',name: 'Odin',category: 'Security Checker',url: 'https://docs.getodin.com/',- producer: {- name: 'Cyble Inc.',- url: 'https://cyble.com/',- },+ producer: { name: 'Cyble Inc.', url: 'https://cyble.com/' },},{regex: 'YouBot',name: 'YouBot',category: 'Crawler',url: 'https://about.you.com/youbot/',- producer: {- name: 'SuSea, Inc.',- url: 'https://you.com/',- },+ producer: { name: 'SuSea, Inc.', url: 'https://you.com/' },},{regex: 'SiteScoreBot',@@ -4634,20 +3752,14 @@ export default [name: 'Monitor Backlinks',category: 'Crawler',url: 'https://www.seoptimer.com/monitor-backlinks/',- producer: {- name: 'SEOptimer',- url: 'https://www.seoptimer.com/',- },+ producer: { name: 'SEOptimer', url: 'https://www.seoptimer.com/' },},{regex: 'mariadb-mysql-kbs-bot',name: 'MariaDB/MySQL Knowledge Base',category: 'Crawler',url: 'https://github.com/williamdes/mariadb-mysql-kbs',- producer: {- name: 'WDES SAS',- url: 'https://wdes.fr/en/',- },+ producer: { name: 'WDES SAS', url: 'https://wdes.fr/en/' },},{regex: 'GitHubCopilotChat',@@ -4655,25 +3767,31 @@ export default [category: 'Crawler',url: 'https://github.com/aaamoon/copilot-gpt4-service',},+ {+ regex: '^pdrl\\.fm',+ name: 'Podroll Analyzer',+ category: 'Crawler',+ url: 'https://podroll.fm',+ },+ {+ regex: 'PodUptime/',+ name: 'PodUptime',+ category: 'Site Monitor',+ url: 'https://poduptime.com',+ },{regex: 'anthropic-ai',name: 'Anthropic AI',category: 'Crawler',url: 'https://www.anthropic.com/',- producer: {- name: 'Anthropic, PBC',- url: 'https://www.anthropic.com/',- },+ producer: { name: 'Anthropic, PBC', url: 'https://www.anthropic.com/' },},{regex: 'NetpeakCheckerBot/[\\d.]+',name: 'Netpeak Checker',category: 'Crawler',url: 'https://netpeaksoftware.com/checker',- producer: {- name: 'Netpeak LTD',- url: 'https://netpeaksoftware.com/',- },+ producer: { name: 'Netpeak LTD', url: 'https://netpeaksoftware.com/' },},{regex: 'SandobaCrawler/[\\d.]+',@@ -4690,10 +3808,7 @@ export default [name: 'Sirdata',category: 'Crawler',url: 'https://semantic-api.docs.sirdata.net/contextual-api/contextual-api/introduction',- producer: {- name: 'Sirdata SAS',- url: 'https://www.sirdata.com/',- },+ producer: { name: 'Sirdata SAS', url: 'https://www.sirdata.com/' },},{regex: 'CheckMarkNetwork/[\\d.]+',@@ -4710,10 +3825,7 @@ export default [name: 'Cohere AI',category: 'Crawler',url: 'https://cohere.com/',- producer: {- name: 'Cohere, Inc.',- url: 'https://cohere.com/',- },+ producer: { name: 'Cohere, Inc.', url: 'https://cohere.com/' },},{regex: 'PerplexityBot/[\\d.]+',@@ -4740,10 +3852,7 @@ export default [name: 'Montastic Monitor',category: 'Site Monitor',url: 'https://www.montastic.com/',- producer: {- name: 'Metadot, Corp.',- url: 'https://www.metadot.com/',- },+ producer: { name: 'Metadot, Corp.', url: 'https://www.metadot.com/' },},{regex: 'Ruby, Twurly v[\\d.]+',@@ -4761,11 +3870,7 @@ export default [url: 'https://www.mixnode.com/',},},- {- regex: 'CSSCheck/[\\d.]+',- name: 'CSSCheck',- category: 'Validator',- },+ { regex: 'CSSCheck/[\\d.]+', name: 'CSSCheck', category: 'Validator' },{regex: 'MicrosoftPreview/[\\d.]+',name: 'Microsoft Preview',@@ -4791,10 +3896,7 @@ export default [name: 'TinEye',category: 'Crawler',url: 'https://tineye.com/',- producer: {- name: 'Idée, Inc.',- url: 'https://tineye.com/',- },+ producer: { name: 'Idée, Inc.', url: 'https://tineye.com/' },},{regex: 'e~arsnova-filter-system',@@ -4811,10 +3913,7 @@ export default [name: 'Botify',category: 'Crawler',url: 'https://www.botify.com/',- producer: {- name: 'BOTIFY SAS',- url: 'https://www.botify.com/',- },+ producer: { name: 'BOTIFY SAS', url: 'https://www.botify.com/' },},{regex: 'adscanner',@@ -4831,10 +3930,7 @@ export default [name: 'WebCEO',category: 'Crawler',url: 'https://www.webceo.com/',- producer: {- name: 'WebCEO, LLC',- url: 'https://www.webceo.com/',- },+ producer: { name: 'WebCEO, LLC', url: 'https://www.webceo.com/' },},{regex: 'NetTrack',@@ -4847,20 +3943,14 @@ export default [name: 'htmlyse',category: 'Crawler',url: 'https://www.htmlyse.com/',- producer: {- name: 'Vistex LTD',- url: 'https://www.htmlyse.com/',- },+ producer: { name: 'Vistex LTD', url: 'https://www.htmlyse.com/' },},{regex: 'TrendsmapResolver/[\\d.]+',name: 'Trendsmap',category: 'Crawler',url: 'https://www.trendsmap.com/',- producer: {- name: 'Trendsmap Pty Ltd',- url: 'https://www.trendsmap.com/',- },+ producer: { name: 'Trendsmap Pty Ltd', url: 'https://www.trendsmap.com/' },},{regex: 'Shareaholic(?:bot)?/[\\d.]+',@@ -4873,29 +3963,713 @@ export default [},},{- regex: 'keycdn-tools',+ regex: 'keycdn-tools:',name: 'KeyCDN Tools',category: 'Service Agent',url: 'https://tools.keycdn.com/geo',},+ {+ regex: 'keycdn-tools/',+ name: 'KeyCDN Tools',+ category: 'Service Agent',+ url: 'https://tools.keycdn.com/',+ producer: { name: 'proinity LLC', url: 'https://www.keycdn.com/' },+ },{regex: 'Arquivo-web-crawler',name: 'Arquivo.pt',category: 'Crawler',url: 'https://sobre.arquivo.pt/en/help/crawling-and-archiving-web-content/',+ producer: { name: 'FCT|FCCN', url: 'https://www.fct.pt/' },+ },+ {+ regex: 'WhatsMyIP\\.org',+ name: 'WhatsMyIP.org',+ category: 'Service Agent',+ url: 'https://www.whatsmyip.org/ua/',+ },+ {+ regex: 'SenutoBot/[\\d.]+',+ name: 'Senuto',+ category: 'Crawler',+ url: 'https://www.senuto.com/',+ producer: { name: 'Senuto Sp. z o.o.', url: 'https://www.senuto.com/' },+ },+ {+ regex: 'spaziodati',+ name: 'SpazioDati',+ category: 'Crawler',+ url: 'https://www.spaziodati.eu/',+ producer: { name: 'SpazioDati s.r.l.', url: 'https://www.spaziodati.eu/' },+ },+ {+ regex: 'GozleBot',+ name: 'Gozle',+ category: 'Crawler',+ url: 'https://gozle.com.tm/en/blog/post/1',+ producer: { name: 'Doly Horjun HJ', url: 'https://gozle.com.tm/' },+ },+ {+ regex: 'Quantcastbot/[\\d.]+',+ name: 'Quantcast',+ category: 'Crawler',+ url: 'https://www.quantcast.com/bot/',+ producer: { name: 'Quantcast Corp.', url: 'https://www.quantcast.com/' },+ },+ {+ regex: 'FontRadar',+ name: 'FontRadar',+ category: 'Crawler',+ url: 'https://www.fontradar.com/',+ producer: { name: 'EMDASH SAS', url: 'https://www.fontradar.com/' },+ },+ {+ regex: 'ViberUrlDownloader',+ name: 'Viber Url Downloader',+ category: 'Service Agent',+ url: 'https://www.viber.com/',+ producer: { name: 'Viber Media S.à r.l.', url: 'https://www.viber.com/' },+ },+ {+ regex: '^Zeno$',+ name: 'Zeno',+ category: 'Crawler',+ url: 'https://github.com/internetarchive/Zeno',+ producer: { name: 'The Internet Archive', url: 'https://archive.org/' },+ },+ {+ regex: 'Barracuda Sentinel',+ name: 'Barracuda Sentinel',+ category: 'Service Agent',+ url: 'https://sentinel.barracudanetworks.com/',+ producer: {+ name: 'Barracuda Networks, Inc.',+ url: 'https://www.barracudanetworks.com/',+ },+ },+ {+ regex: 'RuxitSynthetic/[\\d.]+',+ name: 'RuxitSynthetic',+ category: 'Site Monitor',+ url: 'https://community.dynatrace.com/t5/Troubleshooting/Basic-Commands-for-Synthetic/ta-p/198164',+ producer: { name: 'Dynatrace LLC', url: 'https://www.dynatrace.com/' },+ },+ {+ regex: 'DynatraceSynthetic/[\\d.]+',+ name: 'DynatraceSynthetic',+ category: 'Site Monitor',+ url: 'https://community.dynatrace.com/t5/Troubleshooting/Basic-Commands-for-Synthetic/ta-p/198164',+ producer: { name: 'Dynatrace LLC', url: 'https://www.dynatrace.com/' },+ },+ {+ regex: 'sitebulb',+ name: 'Sitebulb',+ category: 'Crawler',+ url: 'https://sitebulb.com/',+ producer: { name: 'Sitebulb Limited', url: 'https://sitebulb.com/' },+ },+ {+ regex: 'Monsidobot/[\\d.]+',+ name: 'Monsidobot',+ category: 'Crawler',+ url: 'https://monsido.com/bot-html',+ producer: { name: 'Monsido LLC', url: 'https://monsido.com/' },+ },+ {+ regex: 'AccompanyBot',+ name: 'AccompanyBot',+ category: 'Crawler',+ url: 'https://www.accompany.com/',+ producer: { name: 'Accompani, Inc', url: 'https://www.accompany.com/' },+ },+ {+ regex: 'Ghost Inspector',+ name: 'Ghost Inspector',+ category: 'Site Monitor',+ url: 'https://docs.ghostinspector.com/faq/#how-do-i-detect-ghost-inspector-test-runner-traffic-on-my-site',+ producer: {+ name: 'Ghost Inspector, Inc.',+ url: 'https://www.ghostinspector.com/',+ },+ },+ {+ regex: 'Cypress/[\\d.]+',+ name: 'Cypress',+ category: 'Site Monitor',+ url: 'https://github.com/cypress-io/cypress',+ producer: { name: 'Cypress.io, Inc.', url: 'https://www.cypress.io/' },+ },+ {+ regex: 'Google-Apps-Script',+ name: 'Google Apps Script',+ category: 'Service Agent',+ url: 'https://www.google.com/script/start/',+ },+ {+ regex: 'SiteOne-Crawler/[\\d.]+',+ name: 'SiteOne Crawler',+ category: 'Crawler',+ url: 'https://crawler.siteone.io/bot/',+ producer: { name: 'SiteOne s.r.o.', url: 'https://www.siteone.io/' },+ },+ {+ regex: 'Detectify',+ name: 'Detectify',+ category: 'Security Checker',+ url: 'https://support.detectify.com/support/solutions/articles/48001049001-how-to-allow-detectify-to-access-your-site',+ producer: { name: 'Detectify AB', url: 'https://detectify.com/' },+ },+ {+ regex: 'DomCopBot',+ name: 'DomCop Bot',+ category: 'Crawler',+ url: 'https://www.domcop.com/bot',+ producer: {+ name: 'Axeman Technology Solutions LLP',+ url: 'https://axemantech.com/',+ },+ },+ {+ regex: 'Paqlebot/[\\d.]+',+ name: 'Paqlebot',+ category: 'Crawler',+ url: 'https://www.paqle.dk/about/paqlebot',+ producer: { name: 'Paqle A/S', url: 'https://www.paqle.dk/' },+ },+ {+ regex: 'Wibybot',+ name: 'Wibybot',+ category: 'Crawler',+ url: 'https://www.wiby.me/',+ },+ {+ regex: 'Synapse',+ name: 'Synapse',+ category: 'Crawler',+ url: 'https://github.com/matrix-org/synapse',+ },+ {+ regex: 'OSZKbot/[\\d.]+',+ name: 'OSZKbot',+ category: 'Crawler',+ url: 'http://mekosztaly.oszk.hu/mia/',producer: {- name: 'FCT|FCCN',- url: 'https://www.fct.pt/',+ name: 'National Szechenyi Library',+ url: 'https://webarchivum.oszk.hu/',},},+ {+ regex: 'ZoomBot',+ name: 'ZoomBot',+ category: 'Crawler',+ url: 'https://suite.seozoom.it/bot.html',+ producer: { name: 'SEO Cube S.r.l.', url: 'https://www.seocube.it/' },+ },+ {+ regex: 'RavenCrawler/[\\d.]+',+ name: 'RavenCrawler',+ category: 'Crawler',+ url: 'https://raventools.com/site-auditor/',+ producer: { name: 'TapClicks, Inc.', url: 'https://www.tapclicks.com/' },+ },+ {+ regex: 'KadoBot',+ name: 'KadoBot',+ category: 'Crawler',+ url: 'https://www.kadolijst.nl/bot',+ producer: { name: 'Kadolijst', url: 'https://www.kadolijst.nl/' },+ },+ {+ regex: 'Dubbotbot/[\\d.]+',+ name: 'Dubbotbot',+ category: 'Crawler',+ url: 'https://help.dubbot.com/en/articles/6746594-example-custom-user-agent',+ producer: { name: 'DubBot', url: 'https://dubbot.com/' },+ },+ {+ regex: 'Swiftbot/[\\d.]+',+ name: 'Swiftbot',+ category: 'Crawler',+ url: 'https://swiftype.com/swiftbot',+ producer: { name: 'Elasticsearch, B.V.', url: 'https://www.elastic.co/' },+ },+ {+ regex: 'EyeMonIT',+ name: 'EyeMonit',+ category: 'Site Monitor',+ url: 'https://eyemonit.com/',+ producer: { name: 'EyeMonit', url: 'https://eyemonit.com/' },+ },+ {+ regex: 'ThousandEyes',+ name: 'ThousandEyes',+ category: 'Site Monitor',+ url: 'https://www.thousandeyes.com/',+ producer: { name: 'Cisco Systems, Inc.', url: 'https://www.cisco.com/' },+ },+ { regex: 'OmtrBot/[\\d.]+', name: 'OmtrBot', category: 'Site Monitor' },+ { regex: 'WebMon/[\\d.]+', name: 'WebMon', category: 'Site Monitor' },+ {+ regex: 'AdsTxtCrawlerTP/[\\d.]+',+ name: 'AdsTxtCrawlerTP',+ category: 'Crawler',+ },+ {+ regex: 'fragFINN',+ name: 'fragFINN',+ category: 'Crawler',+ url: 'https://www.fragfinn.de/',+ producer: { name: 'fragFINN e.V.', url: 'https://www.fragfinn.de/' },+ },+ {+ regex: 'Clickagy',+ name: 'Clickagy',+ category: 'Crawler',+ url: 'https://www.clickagy.com/',+ producer: { name: 'Clickagy, LLC', url: 'https://www.clickagy.com/' },+ },+ {+ regex: 'kiwitcms-gitops/[\\d.]+',+ name: 'Kiwi TCMS GitOps',+ category: 'Service Agent',+ url: 'https://kiwitcms.org',+ producer: {+ name: 'Open Technologies Bulgaria, Ltd.',+ url: 'https://kiwitcms.org',+ },+ },+ {+ regex: 'webtru_crawler',+ name: 'webtru',+ category: 'Crawler',+ url: 'https://webtru.io/',+ producer: { name: 'DataSign Inc.', url: 'https://datasign.jp/' },+ },+ {+ regex: 'URLSuMaBot',+ name: 'URLSuMaBot',+ category: 'Crawler',+ url: 'https://www.urlsuma.de/',+ },+ {+ regex: '360JK yunjiankong',+ name: '360JK',+ category: 'Site Monitor',+ url: 'http://jk.cloud.360.cn/',+ producer: {+ name: '360 Security Technology Inc.',+ url: 'https://www.360.cn/',+ },+ },+ {+ regex: 'UCSBNetworkMeasurement',+ name: 'UCSB Network Measurement',+ category: 'Crawler',+ url: 'https://www.it.ucsb.edu/',+ producer: {+ name: 'University of California, Santa Barbara',+ url: 'https://www.it.ucsb.edu/',+ },+ },+ {+ regex: 'Plesk screenshot bot',+ name: 'Plesk Screenshot Service',+ category: 'Service Agent',+ url: 'https://support.plesk.com/hc/en-us/articles/13302778306199-What-is-Plesk-Screenshot-Service',+ producer: {+ name: 'Plesk International GmbH',+ url: 'https://www.plesk.com/',+ },+ },+ {+ regex: 'Who\\.is',+ name: 'Who.is Bot',+ category: 'Crawler',+ url: 'https://who.is/',+ },+ {+ regex: 'Probely',+ name: 'Probely',+ category: 'Security Checker',+ url: 'https://probely.com/sos/',+ producer: {+ name: 'Probely - Soluções de Cibersegurança, S.A.',+ url: 'https://probely.com/',+ },+ },+ {+ regex: 'Uptimia(?:/[\\d.]+)?',+ name: 'Uptimia',+ category: 'Site Monitor',+ url: 'https://www.uptimia.com/',+ producer: { name: 'JJ Online GmbH', url: 'https://www.uptimia.com/' },+ },+ {+ regex: '2GDPR/[\\d.]+',+ name: '2GDPR',+ category: 'Service Agent',+ url: 'https://2gdpr.com/tos',+ producer: { name: '2GDPR', url: 'https://2gdpr.com/' },+ },+ {+ regex: 'abuse\\.xmco\\.fr',+ name: 'Serenety',+ category: 'Security Checker',+ url: 'https://abuse.xmco.fr/',+ producer: { name: 'XMCO, SASU', url: 'https://www.xmco.fr/' },+ },+ {+ regex: 'CheckHost',+ name: 'CheckHost',+ category: 'Site Monitor',+ url: 'https://check-host.net/',+ producer: { name: 'CheckHost', url: 'https://check-host.net/' },+ },+ {+ regex: 'LAC_IAHarvester/[\\d.]+',+ name: 'LAC IA Harvester',+ category: 'Crawler',+ url: 'https://library-archives.canada.ca/eng/services/government-canada/web-social-media-preservation-program/Pages/web-archive.aspx',+ producer: {+ name: 'Library and Archives Canada',+ url: 'https://library-archives.canada.ca/',+ },+ },+ {+ regex: 'InsytfulBot/[\\d.]+',+ name: 'InsytfulBot',+ category: 'Crawler',+ url: 'https://www.insytful.com/',+ producer: { name: 'Zengenti Limited', url: 'https://www.zengenti.com/' },+ },+ {+ regex: 'statista\\.com',+ name: 'Statista',+ category: 'Crawler',+ url: 'https://www.statista.com/',+ producer: { name: 'Statista, Inc.', url: 'https://www.statista.com/' },+ },+ {+ regex: 'SubstackContentFetch/[\\d.]+',+ name: 'Substack Content Fetch',+ category: 'Crawler',+ url: 'https://substack.com/',+ producer: { name: 'Substack, Inc.', url: 'https://substack.com/' },+ },+ {+ regex: '^ds9',+ name: 'Deep SEARCH 9',+ category: 'Crawler',+ url: 'https://www.copyright.com/blog/ccc-expands-corporate-solutions-offering-with-new-technology/',+ producer: {+ name: 'Copyright Clearance Center, Inc.',+ url: 'https://www.copyright.com/',+ },+ },+ {+ regex: 'LiveJournal\\.com',+ name: 'LiveJournal',+ url: 'https://www.livejournal.com/',+ category: 'Feed Fetcher',+ producer: { name: 'ООО "СИМ"', url: 'https://www.livejournal.com/' },+ },+ {+ regex: 'bitdiscovery',+ name: 'Tenable.asm',+ category: 'Security Checker',+ url: 'https://bitdiscovery.com/',+ producer: { name: 'Tenable, Inc.', url: 'https://www.tenable.com/' },+ },+ {+ regex: 'Castopod/[\\d.]+',+ name: 'Castopod',+ category: 'Crawler',+ url: 'https://www.castopod.org/',+ },+ {+ regex: 'Elastic/Synthetics',+ name: 'Elastic Synthetics',+ category: 'Site Monitor',+ url: 'https://github.com/elastic/synthetics',+ producer: { name: 'Elasticsearch B.V.', url: 'https://www.elastic.co/' },+ },+ {+ regex: 'WDG_Validator/[\\d.]+',+ name: 'WDG HTML Validator',+ category: 'Validator',+ url: 'http://www.htmlhelp.com/tools/validator/',+ },+ {+ regex: '[email protected]',+ name: 'Aegis',+ category: 'Crawler',+ url: 'https://web.archive.org/web/20180910002802/http://www.aegis.network/',+ },+ {+ regex: 'CrawlyProjectCrawler/[\\d.]+',+ name: 'Crawly Project',+ category: 'Crawler',+ url: 'https://web.archive.org/web/20240326141952/https://crawlyproject.digitaldragon.dev/',+ },+ {+ regex: 'BDFetch',+ name: 'BDFetch',+ category: 'Crawler',+ url: 'https://web.archive.org/web/20130821043949/http://www.branddimensions.com/',+ },+ {+ regex: 'PunkMap',+ name: 'Punk Map',+ category: 'Security Checker',+ url: 'https://github.com/openeasm/punkmap',+ },+ {+ regex: 'GenomeCrawlerd/[\\d.]+',+ name: 'Deepfield Genome',+ category: 'Crawler',+ url: 'https://www.nokia.com/networks/ip-networks/deepfield/genome/',+ producer: { name: 'Nokia Corporation', url: 'https://www.nokia.com/' },+ },+ {+ regex: 'Gaisbot/[\\d.]+',+ name: 'Gaisbot',+ category: 'Crawler',+ url: 'https://web.archive.org/web/20090604121511/https://gais.cs.ccu.edu.tw/robot.php',+ },+ {+ regex: 'FAST-WebCrawler/[\\d.]+',+ name: 'AlltheWeb',+ category: 'Crawler',+ url: 'https://web.archive.org/web/20041020050801/http://www.alltheweb.com/help/webmaster/crawler',+ },+ {+ regex: 'ducks\\.party',+ name: 'ducks.party',+ category: 'Security Checker',+ url: 'https://ducks.party/',+ },+ {+ regex: 'DepSpid/[\\d.]+',+ name: 'DepSpid',+ category: 'Crawler',+ url: 'https://web.archive.org/web/20080321224033/http://about.depspid.net/',+ },+ {+ regex: 'Website-info\\.net',+ name: 'Website-info',+ category: 'Crawler',+ url: 'https://website-info.net/robot',+ producer: { name: 'Meins und Vogel GmbH', url: 'https://muv.com/' },+ },+ {+ regex: 'RedekenBot',+ name: 'RedekenBot',+ category: 'Crawler',+ url: 'https://www.redeken.com/en/help/bot.html',+ producer: { name: 'Redeken', url: 'https://www.redeken.com/' },+ },+ {+ regex: 'semaltbot',+ name: 'semaltbot',+ category: 'Crawler',+ url: 'https://semalt.net/',+ producer: { name: 'Semalt LP', url: 'https://semalt.net/' },+ },+ {+ regex: 'MakeMerryBot',+ name: 'MakeMerryBot',+ category: 'Crawler',+ url: 'https://makemerry.app/bots',+ },+ {+ regex: 'Timpibot',+ name: 'Timpibot',+ category: 'Crawler',+ url: 'https://timpi.io/',+ producer: { name: 'Timpi Inc.', url: 'https://timpi.io/' },+ },+ {+ regex: 'Validbot',+ name: 'ValidBot',+ category: 'Crawler',+ url: 'https://www.validbot.com/',+ producer: { name: 'Jake Olefsky LLC', url: 'https://www.validbot.com/' },+ },+ {+ regex: 'NPBot',+ name: 'NameProtectBot',+ category: 'Crawler',+ url: 'https://www.cscglobal.com/cscglobal/home/',+ producer: { name: 'NameProtect, Inc.', url: 'https://www.cscglobal.com/' },+ },+ {+ regex: 'domaincodex\\.com',+ name: 'Domain Codex',+ category: 'Crawler',+ url: 'https://www.domaincodex.com/',+ producer: {+ name: 'Erie Data Systems, LLC',+ url: 'https://www.eriedatasys.com/',+ },+ },+ {+ regex: 'Swisscows Favicons',+ name: 'Swisscows Favicons',+ category: 'Crawler',+ url: 'https://swisscows.com/',+ producer: { name: 'Swisscows AG', url: 'https://swisscows.com/' },+ },+ {+ regex: 'leak\\.info',+ name: 'leak.info',+ category: 'Crawler',+ url: 'http://www.leak.info/',+ },+ {+ regex: 'workona',+ name: 'Workona',+ category: 'Crawler',+ url: 'https://workona.com/',+ producer: { name: 'Workona, Inc.', url: 'https://workona.com/' },+ },+ {+ regex: 'Bloglines',+ name: 'Bloglines',+ category: 'Crawler',+ url: 'https://web.archive.org/web/20140309033202/http://www.bloglines.com/',+ producer: { name: 'Reply!, Inc.', url: 'https://www.reply.com/' },+ },+ {+ regex: 'heritrix',+ name: 'Heritrix',+ category: 'Crawler',+ url: 'https://webarchive.jira.com/wiki/display/Heritrix/Heritrix',+ producer: { name: 'The Internet Archive', url: 'https://archive.org' },+ },+ {+ regex: 'search\\.marginalia\\.nu',+ name: 'Marginalia',+ category: 'Crawler',+ url: 'https://www.marginalia.nu/marginalia-search/for-webmasters/',+ producer: { name: 'Marginalia', url: 'https://www.marginalia.nu/' },+ },+ {+ regex: 'vu-server-health-scanner/[\\d.]+',+ name: 'VU Server Health Scanner',+ category: 'Security Checker',+ url: 'https://130.37.198.75/index.html',+ producer: { name: 'VU Amsterdam', url: 'https://vu.nl/en' },+ },+ {+ regex: 'Functionize',+ name: 'Functionize',+ category: 'Crawler',+ url: 'https://www.functionize.com/',+ producer: {+ name: 'Functionize, Inc.',+ url: 'https://www.functionize.com/',+ },+ },+ {+ regex: 'Prerender',+ name: 'Prerender',+ category: 'Crawler',+ url: 'https://docs.prerender.io/docs/33-overview-of-prerender-crawlers',+ producer: { name: 'saas.group Inc.', url: 'https://saas.group/' },+ },+ {+ regex: 'bl\\.uk_ldfc_bot',+ name: 'The British Library Legal Deposit Bot',+ category: 'Crawler',+ url: 'https://www.bl.uk/',+ producer: { name: 'The British Library', url: 'https://www.bl.uk/' },+ },+ {+ regex: 'Miniature\\.io',+ name: 'Miniature.io',+ category: 'Service Agent',+ url: 'https://miniature.io/',+ producer: { name: 'LCX Ventures Ltd', url: 'https://www.lcxventures.com/' },+ },+ {+ regex: 'Convertify',+ name: 'Convertify',+ category: 'Service Agent',+ url: 'https://www.convertify.app/',+ producer: { name: 'Convertify', url: 'https://www.convertify.app/' },+ },+ {+ regex: 'ZoteroTranslationServer',+ name: 'Zotero Translation Server',+ category: 'Service Agent',+ url: 'https://github.com/wikimedia/mediawiki-services-zotero',+ producer: {+ name: 'The Wikimedia Foundation, Inc.',+ url: 'https://www.wikimedia.org/',+ },+ },+ {+ regex: 'MuckRack',+ name: 'MuckRack',+ category: 'Crawler',+ url: 'https://muckrack.com/',+ producer: { name: 'Muck Rack, LLC', url: 'https://muckrack.com/' },+ },+ {+ regex: 'Golfe',+ name: 'Golfe',+ category: 'Crawler',+ url: 'http://www.goo-olfe.ae/bot.html',+ },+ {+ regex: 'SpiderLing',+ name: 'SpiderLing',+ category: 'Crawler',+ url: 'https://nlp.fi.muni.cz/projects/biwec/',+ producer: {+ name: 'Natural Language Processing Centre',+ url: 'https://nlp.fi.muni.cz/',+ },+ },+ {+ regex: 'Bravebot',+ name: 'Bravebot',+ category: 'Search bot',+ url: 'https://search.brave.com/help/brave-search-crawler',+ producer: { name: 'Brave Software, Inc.', url: 'https://brave.com/' },+ },+ {+ regex: '1001FirmsBot',+ name: '1001FirmsBot',+ category: 'Crawler',+ url: 'https://www.1001firms.com/1001firmsbot.php',+ },+ {+ regex: 'SteamChatURLLookup',+ name: 'Steam Chat URL Lookup',+ category: 'Service Agent',+ url: 'https://help.steampowered.com/en/faqs/view/595C-42F4-3B66-E02F',+ producer: {+ name: 'Valve Corporation',+ url: 'https://www.valvesoftware.com/',+ },+ },+ {+ regex: 'ohdear\\.app',+ name: 'Oh Dear',+ category: 'Site Monitor',+ url: 'https://ohdear.app/docs/faq/what-is-the-oh-dear-crawler-doing-in-my-logs',+ producer: { name: 'Immutable, SNC', url: 'https://ohdear.app/' },+ },{regex:- 'nuhk|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr\\.com|tweetedtimes\\.com|teoma|oegp|http%20client|htdig|mogimogi|larbin|scrubby|searchsight|semanticdiscovery|snappy|vortex(?!(?: Build|Plus))|zeal(?!ot)|dataparksearch|findlinks|BrowserMob|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|7Siters|centuryb\\.o\\.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|cortex|Re-re Studio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle Client|Hello,? world|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|daumoa,damoa,daum,daumos,duamoa,duam,duamos|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M CODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux Gnu \\(cow\\)|Test Certificate Info|iplabel|Magellan|TheSafex?Internetx?Search|kirkland-signature|^xenu|^ZmEu|^(?:chrome|firefox|Zeus)$',+ 'nuhk|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr\\.com|teoma|oegp|http%20client|htdig|mogimogi|larbin|scrubby|searchsight|semanticdiscovery|snappy|vortex(?!(?: Build|Plus| CM62| HD65))|zeal(?!ot)|dataparksearch|findlinks|BrowserMob|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|7Siters|centuryb\\.o\\.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|cortex|Re-re Studio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle Client|Hello,? world|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M CODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux Gnu \\(cow\\)|Test Certificate Info|iplabel|Magellan|TheSafex?Internetx?Search|Searcherx?web|kirkland-signature|LinkChain|survey-security-dot-txt|infrawatch|Time/|r00ts3c-owned-you|nvdorz|Root Slut|NiggaBalls|BotPoke|GlobalWebSearch|xx032_bo9vs83_2a|sslshed|geckotrail|Wordup|^xenu|^(?:chrome|firefox|Abcd|Dark|KvshClient|Node.js|Report Runner|url|Zeus|ZmEu)$',name: 'Generic Bot',},{regex:- '[a-z0-9_-]*(?:(?<!cu|power[ _]|m[ _])bot(?![ _]TAB|[ _]?5[0-9]|[ _]Senior|[ _]Junior)|analyzer|appengine|archiver|checker|collector|crawl|crawler|fetcher|indexer|monitor|project(?!or)|research|resolver|robots|scraper|spider|transcoder|uptime|user[ _]?agent|validator)(?:[^a-z]|$)',+ '[a-z0-9_-]*(?:(?<!cu|power[ _]|m[ _])bot(?![ _]TAB|[ _]?5[0-9]|[ _]Senior|[ _]Junior)|analyzer|appengine|archiver?|checker|collector|crawl|crawler|(?<!node-|uclient-|Mikrotik/\\d\\.[x\\d] |electron-)fetch(?:er)?|indexer|inspector|monitor|(?<!Microsoft |banshee-)project(?!or)|(?<!Google Wap |Blue |SpeedMode; )proxy|(?<!P)research|resolver|robots|(?<!Cam)scanner|scraper|script|searcher|(?<!-)security|spider(?! 8)|study|transcoder|uptime|user[ _]?agent|validator)(?:[^a-z]|$)',name: 'Generic Bot',},-];+] as const;+export default bots;diff --git a/apps/worker/package.json b/apps/worker/package.jsonindex 1b118e921..fdb6ae33b 100644--- a/apps/worker/package.json+++ b/apps/worker/package.json@@ -22,7 +22,6 @@"prom-client": "^15.1.3","ramda": "^0.29.1","sqlstring": "^2.3.3",- "ua-parser-js": "^1.0.37","uuid": "^9.0.1"},"devDependencies": {@@ -30,7 +29,6 @@"@types/express": "^4.17.21","@types/ramda": "^0.29.6","@types/sqlstring": "^2.3.2",- "@types/ua-parser-js": "^0.7.39","@types/uuid": "^9.0.8","tsup": "^7.2.0","typescript": "^5.2.2"diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex c76df094f..8616eb002 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -121,6 +121,9 @@ importers:'@openpanel/tsconfig':specifier: workspace:*version: link:../../tooling/typescript+ '@types/js-yaml':+ specifier: ^4.0.9+ version: 4.0.9'@types/jsonwebtoken':specifier: ^9.0.6version: 9.0.6@@ -139,6 +142,9 @@ importers:'@types/ws':specifier: ^8.5.10version: 8.5.10+ js-yaml:+ specifier: ^4.1.0+ version: 4.1.0tsup:specifier: ^7.2.0version: 7.3.0([email protected])@@ -8444,7 +8450,6 @@ packages:/@types/[email protected]:resolution: {integrity: sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==}- dev: false/@types/[email protected]:resolution: {integrity: sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==}@@ -9059,7 +9064,6 @@ packages:resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==}- dev: falseresolution: {integrity: sha512-xcLxITLe2HYa1cnYnwCjkOO1PqUHQpozB8x9AR0OgWN2woOBi5kSDVxKfd0b7sb1hw5qFeJhXm9H1nu3xSfLeQ==}@@ -13946,7 +13950,6 @@ packages:hasBin: truedependencies:argparse: 2.0.1- dev: falseresolution: {integrity: sha512-rS46PvsjYmdmuz1OAWXY/1kCYG7pnf1TBqeTiOJr1iDz7s5DLxxC9n/ZMknLDxzYzNVfI7R95MH10emSSG1Wuw==}