Open-source SEO, GEO, and marketing skills for AI agents.
fix(seo): fetch robots rules with crawler user agent (#125)
Deepanshu Palf98476822da8
diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.jsonindex ec93a27e..eca7d79b 100644--- a/.claude-plugin/marketplace.json+++ b/.claude-plugin/marketplace.json@@ -5,14 +5,14 @@},"metadata": {"description": "The official ads, analytics, SEO, and GEO plugin from NotFair.",- "version": "0.27.10"+ "version": "0.27.11"},"plugins": [{"name": "notfair","source": "./","description": "Operate Google, Meta, X, LinkedIn, Reddit, and TikTok Ads; WordPress and GoHighLevel; analyze Search Console and GA4; and run open-source SEO, GEO, and content workflows.",- "version": "0.27.10",+ "version": "0.27.11","author": {"name": "NotFair","url": "https://notfair.co"diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.jsonindex 4791a444..add61cad 100644--- a/.claude-plugin/plugin.json+++ b/.claude-plugin/plugin.json@@ -1,6 +1,6 @@{"name": "notfair",- "version": "0.27.10",+ "version": "0.27.11","description": "The official ads, analytics, SEO, and GEO plugin from NotFair. Operate Google, Meta, X, LinkedIn, Reddit, and TikTok Ads; WordPress and GoHighLevel; analyze Google Search Console and GA4; and run open-source search and content workflows from any compatible agent.","author": {"name": "NotFair",diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.jsonindex 3c70b304..20d04d45 100644--- a/.codex-plugin/plugin.json+++ b/.codex-plugin/plugin.json@@ -1,6 +1,6 @@{"name": "notfair",- "version": "0.27.10",+ "version": "0.27.11","description": "The official ads, analytics, SEO, and GEO plugin from NotFair.","author": {"name": "NotFair",diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.jsonindex a37619df..1ec848c2 100644--- a/.cursor-plugin/plugin.json+++ b/.cursor-plugin/plugin.json@@ -1,7 +1,7 @@{"name": "notfair","displayName": "NotFair",- "version": "0.27.10",+ "version": "0.27.11","description": "One NotFair OAuth MCP for Google, Meta, X, LinkedIn, Reddit, and TikTok Ads, GA4, Search Console, GoHighLevel, and WordPress, plus open-source SEO, GEO, and content skills.","author": {"name": "Notfair",diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 5b680da0..05a4a59a 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -9,6 +9,12 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).## [Unreleased]+## [0.27.11] — 2026-09-21++### Fixed++- The broken-link checker now fetches `robots.txt` with its declared crawler user agent, avoiding false fetch failures on sites that block Python's default urllib agent while preserving fail-open behavior when robots rules are unavailable.+## [0.27.10] — 2026-09-21### Fixeddiff --git a/VERSION b/VERSIONindex f49ede62..adb8d6a4 100644--- a/VERSION+++ b/VERSION@@ -1 +1 @@-0.27.10+0.27.11diff --git a/gemini-extension.json b/gemini-extension.jsonindex 542f378b..e2e2341b 100644--- a/gemini-extension.json+++ b/gemini-extension.json@@ -1,6 +1,6 @@{"name": "notfair",- "version": "0.27.10",+ "version": "0.27.11","description": "Operate ads and analytics, and run open-source SEO, GEO, and content workflows with NotFair.","contextFileName": "AGENTS.md","mcpServers": {diff --git a/plugin.json b/plugin.jsonindex 3adbd453..ca9ffe2b 100644--- a/plugin.json+++ b/plugin.json@@ -1,7 +1,7 @@{"$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json","name": "notfair",- "version": "0.27.10",+ "version": "0.27.11","description": "Operate Google, Meta, X, LinkedIn, Reddit, and TikTok Ads plus GA4, Search Console, GoHighLevel, and WordPress through one OAuth MCP, and run open-source SEO, GEO, and content workflows.","author": {"name": "NotFair",diff --git a/seo/broken-link-checker/scripts/checker.py b/seo/broken-link-checker/scripts/checker.pyindex 99671432..de5795d2 100644--- a/seo/broken-link-checker/scripts/checker.py+++ b/seo/broken-link-checker/scripts/checker.py@@ -72,8 +72,15 @@ def crawl(start_url, max_pages=50):# Robots.txt checkrp = urllib.robotparser.RobotFileParser()try:- rp.set_url(urllib.parse.urljoin(start_url, '/robots.txt'))- rp.read()+ robots_url = urllib.parse.urljoin(start_url, '/robots.txt')+ robots_request = urllib.request.Request(+ robots_url,+ headers={'User-Agent': 'NotFairBrokenLinkChecker/1.0'},+ )+ with urllib.request.urlopen(robots_request, timeout=10) as response:+ robots_text = response.read().decode('utf-8', errors='ignore')+ rp.set_url(robots_url)+ rp.parse(robots_text.splitlines())except Exception as e:print(f"Warning: Could not read robots.txt: {e}", file=sys.stderr)# Default to allowing if robots.txt is missing or unreachablediff --git a/server.json b/server.jsonindex ecd09247..f4e4180a 100644--- a/server.json+++ b/server.json@@ -8,7 +8,7 @@"url": "https://github.com/nowork-studio/notfair-plugin","source": "github"},- "version": "0.27.10",+ "version": "0.27.11","remotes": [{"type": "streamable-http",diff --git a/test/unit/test_broken_link_checker.py b/test/unit/test_broken_link_checker.pyindex 4df0ad7d..46f82050 100644--- a/test/unit/test_broken_link_checker.py+++ b/test/unit/test_broken_link_checker.py@@ -5,6 +5,7 @@ import socketimport urllib.requestimport importlib.utilimport sys+from unittest.mock import patchfrom pathlib import Path# Fix for hyphen in directory name@@ -108,6 +109,36 @@ class TestBrokenLinkChecker(unittest.TestCase):# Should NOT contain external-mock since it's validself.assertNotIn(f"{self.base_url}/external-mock", targets)+ def test_robots_request_uses_crawler_user_agent(self):+ seen = []++ original_do_get = self.httpd.RequestHandlerClass.do_GET++ def record_get(handler):+ if handler.path == '/robots.txt':+ seen.append(handler.headers.get('User-Agent'))+ return original_do_get(handler)++ with patch.object(self.httpd.RequestHandlerClass, 'do_GET', record_get):+ crawl(self.base_url, max_pages=1)++ self.assertEqual(seen, ['NotFairBrokenLinkChecker/1.0'])++ def test_robots_fetch_failure_is_fail_open(self):+ real_urlopen = urllib.request.urlopen++ def fail_robots(request, *args, **kwargs):+ url = request.full_url if hasattr(request, 'full_url') else request+ if str(url).endswith('/robots.txt'):+ raise urllib.error.URLError('robots unavailable')+ return real_urlopen(request, *args, **kwargs)++ with patch.object(urllib.request, 'urlopen', side_effect=fail_robots):+ broken = crawl(self.base_url, max_pages=1)++ targets = [entry.get('target') for entry in broken]+ self.assertIn(f"{self.base_url}/broken", targets)+def test_robots_txt_respect(self):# We need to check if /private is skipped.# crawl() prints to stderr when skipping.