Skip to content

Open-source SEO, GEO, and marketing skills for AI agents.

fix(seo): fetch robots rules with crawler user agent (#125)

Deepanshu Palf98476822da8
diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json
index ec93a27e..eca7d79b 100644
--- a/.claude-plugin/marketplace.json
+++ b/.claude-plugin/marketplace.json
@@ -5,14 +5,14 @@
},
"metadata": {
"description": "The official ads, analytics, SEO, and GEO plugin from NotFair.",
- "version": "0.27.10"
+ "version": "0.27.11"
},
"plugins": [
{
"name": "notfair",
"source": "./",
"description": "Operate Google, Meta, X, LinkedIn, Reddit, and TikTok Ads; WordPress and GoHighLevel; analyze Search Console and GA4; and run open-source SEO, GEO, and content workflows.",
- "version": "0.27.10",
+ "version": "0.27.11",
"author": {
"name": "NotFair",
"url": "https://notfair.co"
diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json
index 4791a444..add61cad 100644
--- a/.claude-plugin/plugin.json
+++ b/.claude-plugin/plugin.json
@@ -1,6 +1,6 @@
{
"name": "notfair",
- "version": "0.27.10",
+ "version": "0.27.11",
"description": "The official ads, analytics, SEO, and GEO plugin from NotFair. Operate Google, Meta, X, LinkedIn, Reddit, and TikTok Ads; WordPress and GoHighLevel; analyze Google Search Console and GA4; and run open-source search and content workflows from any compatible agent.",
"author": {
"name": "NotFair",
diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json
index 3c70b304..20d04d45 100644
--- a/.codex-plugin/plugin.json
+++ b/.codex-plugin/plugin.json
@@ -1,6 +1,6 @@
{
"name": "notfair",
- "version": "0.27.10",
+ "version": "0.27.11",
"description": "The official ads, analytics, SEO, and GEO plugin from NotFair.",
"author": {
"name": "NotFair",
diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json
index a37619df..1ec848c2 100644
--- a/.cursor-plugin/plugin.json
+++ b/.cursor-plugin/plugin.json
@@ -1,7 +1,7 @@
{
"name": "notfair",
"displayName": "NotFair",
- "version": "0.27.10",
+ "version": "0.27.11",
"description": "One NotFair OAuth MCP for Google, Meta, X, LinkedIn, Reddit, and TikTok Ads, GA4, Search Console, GoHighLevel, and WordPress, plus open-source SEO, GEO, and content skills.",
"author": {
"name": "Notfair",
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5b680da0..05a4a59a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,12 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
## [Unreleased]
+## [0.27.11] — 2026-09-21
+
+### Fixed
+
+- The broken-link checker now fetches `robots.txt` with its declared crawler user agent, avoiding false fetch failures on sites that block Python's default urllib agent while preserving fail-open behavior when robots rules are unavailable.
+
## [0.27.10] — 2026-09-21
### Fixed
diff --git a/VERSION b/VERSION
index f49ede62..adb8d6a4 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-0.27.10
+0.27.11
diff --git a/gemini-extension.json b/gemini-extension.json
index 542f378b..e2e2341b 100644
--- a/gemini-extension.json
+++ b/gemini-extension.json
@@ -1,6 +1,6 @@
{
"name": "notfair",
- "version": "0.27.10",
+ "version": "0.27.11",
"description": "Operate ads and analytics, and run open-source SEO, GEO, and content workflows with NotFair.",
"contextFileName": "AGENTS.md",
"mcpServers": {
diff --git a/plugin.json b/plugin.json
index 3adbd453..ca9ffe2b 100644
--- a/plugin.json
+++ b/plugin.json
@@ -1,7 +1,7 @@
{
"$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
"name": "notfair",
- "version": "0.27.10",
+ "version": "0.27.11",
"description": "Operate Google, Meta, X, LinkedIn, Reddit, and TikTok Ads plus GA4, Search Console, GoHighLevel, and WordPress through one OAuth MCP, and run open-source SEO, GEO, and content workflows.",
"author": {
"name": "NotFair",
diff --git a/seo/broken-link-checker/scripts/checker.py b/seo/broken-link-checker/scripts/checker.py
index 99671432..de5795d2 100644
--- a/seo/broken-link-checker/scripts/checker.py
+++ b/seo/broken-link-checker/scripts/checker.py
@@ -72,8 +72,15 @@ def crawl(start_url, max_pages=50):
# Robots.txt check
rp = urllib.robotparser.RobotFileParser()
try:
- rp.set_url(urllib.parse.urljoin(start_url, '/robots.txt'))
- rp.read()
+ robots_url = urllib.parse.urljoin(start_url, '/robots.txt')
+ robots_request = urllib.request.Request(
+ robots_url,
+ headers={'User-Agent': 'NotFairBrokenLinkChecker/1.0'},
+ )
+ with urllib.request.urlopen(robots_request, timeout=10) as response:
+ robots_text = response.read().decode('utf-8', errors='ignore')
+ rp.set_url(robots_url)
+ rp.parse(robots_text.splitlines())
except Exception as e:
print(f"Warning: Could not read robots.txt: {e}", file=sys.stderr)
# Default to allowing if robots.txt is missing or unreachable
diff --git a/server.json b/server.json
index ecd09247..f4e4180a 100644
--- a/server.json
+++ b/server.json
@@ -8,7 +8,7 @@
"url": "https://github.com/nowork-studio/notfair-plugin",
"source": "github"
},
- "version": "0.27.10",
+ "version": "0.27.11",
"remotes": [
{
"type": "streamable-http",
diff --git a/test/unit/test_broken_link_checker.py b/test/unit/test_broken_link_checker.py
index 4df0ad7d..46f82050 100644
--- a/test/unit/test_broken_link_checker.py
+++ b/test/unit/test_broken_link_checker.py
@@ -5,6 +5,7 @@ import socket
import urllib.request
import importlib.util
import sys
+from unittest.mock import patch
from pathlib import Path
# Fix for hyphen in directory name
@@ -108,6 +109,36 @@ class TestBrokenLinkChecker(unittest.TestCase):
# Should NOT contain external-mock since it's valid
self.assertNotIn(f"{self.base_url}/external-mock", targets)
+ def test_robots_request_uses_crawler_user_agent(self):
+ seen = []
+
+ original_do_get = self.httpd.RequestHandlerClass.do_GET
+
+ def record_get(handler):
+ if handler.path == '/robots.txt':
+ seen.append(handler.headers.get('User-Agent'))
+ return original_do_get(handler)
+
+ with patch.object(self.httpd.RequestHandlerClass, 'do_GET', record_get):
+ crawl(self.base_url, max_pages=1)
+
+ self.assertEqual(seen, ['NotFairBrokenLinkChecker/1.0'])
+
+ def test_robots_fetch_failure_is_fail_open(self):
+ real_urlopen = urllib.request.urlopen
+
+ def fail_robots(request, *args, **kwargs):
+ url = request.full_url if hasattr(request, 'full_url') else request
+ if str(url).endswith('/robots.txt'):
+ raise urllib.error.URLError('robots unavailable')
+ return real_urlopen(request, *args, **kwargs)
+
+ with patch.object(urllib.request, 'urlopen', side_effect=fail_robots):
+ broken = crawl(self.base_url, max_pages=1)
+
+ targets = [entry.get('target') for entry in broken]
+ self.assertIn(f"{self.base_url}/broken", targets)
+
def test_robots_txt_respect(self):
# We need to check if /private is skipped.
# crawl() prints to stderr when skipping.