Skip to content

Universal SEO skill for Claude Code. 26 sub-skills + 19 sub-agents covering technical SEO, E-E-A-T, schema, GEO/AEO, agent readiness (Lighthouse Agentic Browsing, WebMCP, llms.txt), backlinks, local SEO, e-commerce, international SEO, Google APIs, and PDF/Excel reporting. 9 optional extensions for live SEO data.

fix(backlinks): match the exact target link, report page nofollow, GET when HEAD fails

Hwee-Boon Yara4d83b3ef8e5
diff --git a/scripts/verify_backlinks.py b/scripts/verify_backlinks.py
index 7ca1e6d..758f32a 100644
--- a/scripts/verify_backlinks.py
+++ b/scripts/verify_backlinks.py
@@ -4,7 +4,8 @@ Backlink verification crawler for Claude SEO.
Verifies whether known backlinks still exist by fetching source pages and
checking if the target URL appears in their outbound links. Uses HTTP HEAD
-for fast existence checks and full GET + HTML parsing for link verification.
+for fast existence checks (falling back to GET when a server rejects HEAD) and
+full GET + HTML parsing for link verification.
Usage:
python verify_backlinks.py --target https://example.com --links links.json --json
@@ -14,6 +15,7 @@ Usage:
import argparse
import json
+import re
import sys
import time
from urllib.parse import urlparse
@@ -69,7 +71,7 @@ def _head_check(url: str, timeout: int = 15) -> dict:
return {
"status_code": resp.status_code,
"exists": resp.status_code == 200,
- "redirect_url": str(resp.url) if str(resp.url) != url else None,
+ "redirect_url": str(resp.url) if resp.history else None,
"error": None,
}
except URLSafetyError as e:
@@ -78,6 +80,7 @@ def _head_check(url: str, timeout: int = 15) -> dict:
"exists": False,
"redirect_url": None,
"error": f"blocked by SSRF protection: {e}",
+ "blocked": True,
}
except requests.exceptions.Timeout:
return {"status_code": None, "exists": False, "redirect_url": None, "error": "timeout"}
@@ -92,6 +95,68 @@ def _normalize_url(url: str) -> str:
return f"{parsed.scheme}://{parsed.netloc}{path}"
+# Directives whose value follows a colon, so "max-snippet: 10" is not a
+# user-agent scope like "otherbot: nofollow".
+_VALUED_ROBOTS_DIRECTIVES = {
+ "max-snippet", "max-image-preview", "max-video-preview", "unavailable_after",
+}
+# A crawler name has no spaces, unlike the pieces of a comma-split date in
+# "unavailable_after: Wed, 03 Nov 2026 15:00:00 GMT".
+_CRAWLER_NAME = re.compile(r"[a-z][a-z0-9_.-]*")
+
+
+def _robots_nofollow(value: str) -> bool:
+ """True if a robots directive list has nofollow (or none) for all crawlers.
+
+ Directives after a user-agent prefix ("otherbot: noindex, nofollow") apply
+ only to that crawler and are ignored.
+ """
+ scoped = False
+ for token in value.split(","):
+ token = token.strip().lower()
+ name, sep, rest = token.partition(":")
+ name = name.strip()
+ if sep and name not in _VALUED_ROBOTS_DIRECTIVES and _CRAWLER_NAME.fullmatch(name):
+ scoped = True
+ token = rest.strip()
+ if not scoped and token in ("nofollow", "none"):
+ return True
+ return False
+
+
+def _page_nofollow(meta_robots, headers: dict) -> bool:
+ """True if meta robots or X-Robots-Tag tells crawlers not to follow page links."""
+ if meta_robots and _robots_nofollow(meta_robots):
+ return True
+ return any(
+ name.lower() == "x-robots-tag" and _robots_nofollow(value or "")
+ for name, value in (headers or {}).items()
+ )
+
+
+def _apply_get_status(result: dict, page_data: dict) -> bool:
+ """Record a GET outcome on result; True only for a usable 200 response."""
+ result["http_status"] = page_data.get("status_code")
+ if page_data.get("error"):
+ result["status"] = "error"
+ result["error"] = page_data["error"]
+ return False
+ if page_data.get("redirect_chain"):
+ result["redirect_url"] = page_data.get("url")
+ if result["http_status"] == 404:
+ result["status"] = "lost"
+ return False
+ if result["http_status"] != 200:
+ result["status"] = "error"
+ result["error"] = f"HTTP {result['http_status']}"
+ return False
+ return True
+
+
+# Lower rank wins when several links on the page point at the target.
+_MATCH_RANK = {"exact_url": 0, "domain_match": 1, "subdomain_match": 2}
+
+
def verify_single_backlink(source_url: str, target_url: str,
head_only: bool = False, timeout: int = 30) -> dict:
"""
@@ -115,6 +180,8 @@ def verify_single_backlink(source_url: str, target_url: str,
"anchor_text": None,
"rel_attributes": [],
"link_context": None,
+ "page_nofollow": False,
+ "redirect_url": None,
"error": None,
}
@@ -127,26 +194,31 @@ def verify_single_backlink(source_url: str, target_url: str,
source_domain = urlparse(source_url).netloc
_polite_delay(source_domain)
- # Step 1: HEAD check
+ # Step 1: HEAD check. Redirects are followed, so a 3xx never surfaces
+ # here; a changed final URL is reported as redirect_url instead.
head_result = _head_check(source_url, timeout=min(timeout, 15))
result["http_status"] = head_result["status_code"]
- if not head_result["exists"]:
- if head_result["status_code"] == 404:
- result["status"] = "lost"
- elif head_result["status_code"] and 300 <= head_result["status_code"] < 400:
- result["status"] = "moved"
- result["redirect_url"] = head_result.get("redirect_url")
- elif head_result["error"]:
- result["status"] = "error"
- result["error"] = head_result["error"]
- else:
- result["status"] = "error"
- result["http_status"] = head_result["status_code"]
+ if head_result.get("blocked"):
+ result["status"] = "error"
+ result["error"] = head_result["error"]
return result
+ if head_result["status_code"] == 404:
+ result["status"] = "lost"
+ return result
+
+ if head_result["exists"]:
+ result["redirect_url"] = head_result["redirect_url"]
+
if head_only:
- result["status"] = "exists"
+ # Many servers answer HEAD with 403/405 or fail it outright; only a
+ # GET can tell whether the page exists.
+ if not head_result["exists"]:
+ _polite_delay(source_domain)
+ if not _apply_get_status(result, fetch_page(source_url, timeout=timeout)):
+ return result
+ result["status"] = "moved" if result["redirect_url"] else "exists"
result["target_found"] = None # Unknown without full check
return result
@@ -154,10 +226,7 @@ def verify_single_backlink(source_url: str, target_url: str,
_polite_delay(source_domain)
page_data = fetch_page(source_url, timeout=timeout)
- if page_data.get("error"):
- result["status"] = "error"
- result["error"] = page_data["error"]
- result["http_status"] = page_data.get("status_code")
+ if not _apply_get_status(result, page_data):
return result
if not page_data.get("content"):
@@ -165,8 +234,6 @@ def verify_single_backlink(source_url: str, target_url: str,
result["error"] = "Page returned no content"
return result
- result["http_status"] = page_data.get("status_code", 200)
-
# Step 3: Parse HTML and find target link
parsed = parse_html(page_data["content"], base_url=source_url)
all_links = parsed.get("links", {})
@@ -178,6 +245,8 @@ def verify_single_backlink(source_url: str, target_url: str,
raw_target_host = urlparse(target_url).netloc.lower()
target_domain = raw_target_host[4:] if raw_target_host.startswith("www.") else raw_target_host
+ best_link = None
+ best_match_type = None
for link in all_page_links:
link_href = link.get("href", "")
if not link_href:
@@ -197,14 +266,27 @@ def verify_single_backlink(source_url: str, target_url: str,
else:
continue
+ # The first link of the best match type wins, so an earlier nav or
+ # footer link to the same domain cannot mask the exact link.
+ if best_match_type is None or _MATCH_RANK[match_type] < _MATCH_RANK[best_match_type]:
+ best_link, best_match_type = link, match_type
+ if match_type == "exact_url":
+ break
+
+ if best_link is not None:
result["target_found"] = True
- result["match_type"] = match_type
- result["anchor_text"] = link.get("text", "").strip()[:200]
- rel = link.get("rel", "")
+ result["match_type"] = best_match_type
+ result["anchor_text"] = best_link.get("text", "").strip()[:200]
+ rel = best_link.get("rel", "")
if rel:
result["rel_attributes"] = rel.split() if isinstance(rel, str) else rel
else:
result["rel_attributes"] = ["follow"] # No rel = dofollow
+ # Page-level nofollow is reported separately; rel_attributes stays
+ # the link's own markup.
+ result["page_nofollow"] = _page_nofollow(
+ parsed.get("meta_robots"), page_data.get("headers", {})
+ )
result["status"] = "verified"
return result
@@ -384,7 +466,9 @@ def main():
anchor_display = f" [{anchor[:30]}]" if anchor else ""
rel = r.get("rel_attributes", [])
rel_display = f" rel={','.join(rel)}" if rel and rel != ["follow"] else ""
- print(f" [{status:13s}] {r.get('source_url', '?')}{anchor_display}{rel_display}")
+ page_display = " page=nofollow" if r.get("page_nofollow") else ""
+ print(f" [{status:13s}] {r.get('source_url', '?')}{anchor_display}{rel_display}"
+ f"{page_display}")
if __name__ == "__main__":
diff --git a/tests/test_verify_backlinks.py b/tests/test_verify_backlinks.py
new file mode 100644
index 0000000..5f973d9
--- /dev/null
+++ b/tests/test_verify_backlinks.py
@@ -0,0 +1,225 @@
+"""
+Tests for link matching and status handling in scripts/verify_backlinks.py.
+
+Network calls are mocked: `safe_requests_head` and `fetch_page` are replaced
+with fakes so each test controls the HEAD status, the GET status, headers and
+the HTML. The real `parse_html` runs on the fake HTML.
+"""
+import sys
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import verify_backlinks # noqa: E402
+
+SOURCE = "https://blog.example.org/post"
+TARGET = "https://target.example/tool"
+
+
+class FakeHeadResponse:
+ def __init__(self, status_code, url=SOURCE, redirected=False):
+ self.status_code = status_code
+ self.url = url
+ self.history = [object()] if redirected else []
+
+
+def _page(html, status_code=200, headers=None, url=SOURCE):
+ return {
+ "url": url,
+ "status_code": status_code,
+ "content": html,
+ "headers": headers or {},
+ "redirect_chain": [],
+ "error": None,
+ }
+
+
+def fake_network(monkeypatch):
+ """Install fakes; tests set `state["head"]` and `state["page"]`."""
+ state = {"head": FakeHeadResponse(200), "page": None, "get_calls": 0}
+
+ def fake_head(url, **kwargs):
+ head = state["head"]
+ if isinstance(head, Exception):
+ raise head
+ return head
+
+ def fake_fetch(url, **kwargs):
+ state["get_calls"] += 1
+ return state["page"]
+
+ monkeypatch.setattr(verify_backlinks, "DOMAIN_DELAY", 0)
+ monkeypatch.setattr(verify_backlinks, "validate_url", lambda url: True)
+ monkeypatch.setattr(verify_backlinks, "safe_requests_head", fake_head)
+ monkeypatch.setattr(verify_backlinks, "fetch_page", fake_fetch)
+ return state
+
+
+def _html(body, head=""):
+ return f"<html><head>{head}</head><body>{body}</body></html>"
+
+
+def test_exact_link_wins_over_earlier_same_domain_link(fake_network):
+ fake_network["page"] = _page(_html(
+ '<nav><a href="https://target.example/" rel="nofollow">Home</a></nav>'
+ '<p>Try <a href="https://target.example/tool">the tool</a>.</p>'
+ ))
+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)
+ assert result["status"] == "verified"
+ assert result["match_type"] == "exact_url"
+ assert result["anchor_text"] == "the tool"
+ assert result["rel_attributes"] == ["follow"]
+
+
+def test_domain_match_still_reported_when_no_exact_link(fake_network):
+ fake_network["page"] = _page(_html(
+ '<a href="https://sub.target.example/x">sub</a>'
+ '<a href="https://target.example/other" rel="sponsored">other</a>'
+ ))
+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)
+ assert result["match_type"] == "domain_match"
+ assert result["anchor_text"] == "other"
+ assert result["rel_attributes"] == ["sponsored"]
+
+
+def test_link_without_rel_and_no_page_directive_is_not_page_nofollow(fake_network):
+ fake_network["page"] = _page(_html('<a href="https://target.example/tool">t</a>'))
+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)
+ assert result["rel_attributes"] == ["follow"]
+ assert result["page_nofollow"] is False
+
+
[email protected]("content", ["noindex, nofollow", "NOFOLLOW", "none"])
+def test_meta_robots_nofollow_is_reported(fake_network, content):
+ fake_network["page"] = _page(_html(
+ '<a href="https://target.example/tool">t</a>',
+ head=f'<meta name="robots" content="{content}">',
+ ))
+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)
+ assert result["status"] == "verified"
+ assert result["page_nofollow"] is True
+ assert result["rel_attributes"] == ["follow"]
+
+
[email protected]("value", ["nofollow", "noarchive, NoFollow", "none"])
+def test_x_robots_tag_nofollow_is_reported(fake_network, value):
+ fake_network["page"] = _page(
+ _html('<a href="https://target.example/tool">t</a>'),
+ headers={"x-robots-tag": value},
+ )
+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)
+ assert result["page_nofollow"] is True
+
+
[email protected]("value", ["otherbot: nofollow", "otherbot: noindex, nofollow"])
+def test_x_robots_tag_scoped_to_another_crawler_is_ignored(fake_network, value):
+ fake_network["page"] = _page(
+ _html('<a href="https://target.example/tool">t</a>'),
+ headers={"X-Robots-Tag": value},
+ )
+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)
+ assert result["page_nofollow"] is False
+
+
+def test_x_robots_tag_valued_directive_is_not_a_crawler_scope(fake_network):
+ fake_network["page"] = _page(
+ _html('<a href="https://target.example/tool">t</a>'),
+ headers={"X-Robots-Tag": "max-snippet: 50, nofollow"},
+ )
+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)
+ assert result["page_nofollow"] is True
+
+
+def test_x_robots_tag_date_with_commas_is_not_a_crawler_scope(fake_network):
+ fake_network["page"] = _page(
+ _html('<a href="https://target.example/tool">t</a>'),
+ headers={"X-Robots-Tag": "unavailable_after: Wed, 03 Nov 2026 15:00:00 GMT, nofollow"},
+ )
+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)
+ assert result["page_nofollow"] is True
+
+
[email protected]("head_status", [403, 405, 501])
+def test_head_rejected_falls_back_to_get(fake_network, head_status):
+ fake_network["head"] = FakeHeadResponse(head_status)
+ fake_network["page"] = _page(_html('<a href="https://target.example/tool">t</a>'))
+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)
+ assert result["status"] == "verified"
+ assert result["http_status"] == 200
+ assert fake_network["get_calls"] == 1
+
+
+def test_head_request_error_falls_back_to_get(fake_network):
+ fake_network["head"] = verify_backlinks.requests.exceptions.ConnectionError("reset")
+ fake_network["page"] = _page(_html('<a href="https://target.example/tool">t</a>'))
+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)
+ assert result["status"] == "verified"
+
+
+def test_head_ssrf_block_does_not_fall_back_to_get(fake_network):
+ fake_network["head"] = verify_backlinks.URLSafetyError("private address")
+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)
+ assert result["status"] == "error"
+ assert "SSRF" in result["error"]
+ assert fake_network["get_calls"] == 0
+
+
+def test_head_404_is_lost_without_get(fake_network):
+ fake_network["head"] = FakeHeadResponse(404)
+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)
+ assert result["status"] == "lost"
+ assert fake_network["get_calls"] == 0
+
+
+def test_get_fallback_404_is_lost(fake_network):
+ fake_network["head"] = FakeHeadResponse(405)
+ fake_network["page"] = _page("Not found", status_code=404)
+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)
+ assert result["status"] == "lost"
+ assert result["http_status"] == 404
+
+
+def test_get_fallback_still_blocked_is_error(fake_network):
+ fake_network["head"] = FakeHeadResponse(403)
+ fake_network["page"] = _page(_html("Access denied"), status_code=403)
+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)
+ assert result["status"] == "error"
+ assert result["http_status"] == 403
+ assert result["target_found"] is False
+
+
+def test_head_only_falls_back_to_get_for_existence(fake_network):
+ fake_network["head"] = FakeHeadResponse(405)
+ fake_network["page"] = _page(_html("hello"))
+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET, head_only=True)
+ assert result["status"] == "exists"
+ assert result["http_status"] == 200
+
+
+def test_head_only_redirect_is_moved(fake_network):
+ fake_network["head"] = FakeHeadResponse(
+ 200, url="https://blog.example.org/new-post", redirected=True
+ )
+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET, head_only=True)
+ assert result["status"] == "moved"
+ assert result["redirect_url"] == "https://blog.example.org/new-post"
+
+
+def test_head_only_url_normalization_alone_is_not_moved(fake_network):
+ fake_network["head"] = FakeHeadResponse(200, url=SOURCE + "/")
+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET, head_only=True)
+ assert result["status"] == "exists"
+ assert result["redirect_url"] is None
+
+
+def test_full_check_on_redirected_page_reports_redirect_url(fake_network):
+ final = "https://blog.example.org/new-post"
+ fake_network["head"] = FakeHeadResponse(200, url=final, redirected=True)
+ fake_network["page"] = _page(_html('<a href="https://target.example/tool">t</a>'), url=final)
+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)
+ assert result["status"] == "verified"
+ assert result["redirect_url"] == final