Universal SEO skill for Claude Code. 26 sub-skills + 19 sub-agents covering technical SEO, E-E-A-T, schema, GEO/AEO, agent readiness (Lighthouse Agentic Browsing, WebMCP, llms.txt), backlinks, local SEO, e-commerce, international SEO, Google APIs, and PDF/Excel reporting. 9 optional extensions for live SEO data.
fix(backlinks): match the exact target link, report page nofollow, GET when HEAD fails
Hwee-Boon Yara4d83b3ef8e5
diff --git a/scripts/verify_backlinks.py b/scripts/verify_backlinks.pyindex 7ca1e6d..758f32a 100644--- a/scripts/verify_backlinks.py+++ b/scripts/verify_backlinks.py@@ -4,7 +4,8 @@ Backlink verification crawler for Claude SEO.Verifies whether known backlinks still exist by fetching source pages andchecking if the target URL appears in their outbound links. Uses HTTP HEAD-for fast existence checks and full GET + HTML parsing for link verification.+for fast existence checks (falling back to GET when a server rejects HEAD) and+full GET + HTML parsing for link verification.Usage:python verify_backlinks.py --target https://example.com --links links.json --json@@ -14,6 +15,7 @@ Usage:import argparseimport json+import reimport sysimport timefrom urllib.parse import urlparse@@ -69,7 +71,7 @@ def _head_check(url: str, timeout: int = 15) -> dict:return {"status_code": resp.status_code,"exists": resp.status_code == 200,- "redirect_url": str(resp.url) if str(resp.url) != url else None,+ "redirect_url": str(resp.url) if resp.history else None,"error": None,}except URLSafetyError as e:@@ -78,6 +80,7 @@ def _head_check(url: str, timeout: int = 15) -> dict:"exists": False,"redirect_url": None,"error": f"blocked by SSRF protection: {e}",+ "blocked": True,}except requests.exceptions.Timeout:return {"status_code": None, "exists": False, "redirect_url": None, "error": "timeout"}@@ -92,6 +95,68 @@ def _normalize_url(url: str) -> str:return f"{parsed.scheme}://{parsed.netloc}{path}"+# Directives whose value follows a colon, so "max-snippet: 10" is not a+# user-agent scope like "otherbot: nofollow".+_VALUED_ROBOTS_DIRECTIVES = {+ "max-snippet", "max-image-preview", "max-video-preview", "unavailable_after",+}+# A crawler name has no spaces, unlike the pieces of a comma-split date in+# "unavailable_after: Wed, 03 Nov 2026 15:00:00 GMT".+_CRAWLER_NAME = re.compile(r"[a-z][a-z0-9_.-]*")+++def _robots_nofollow(value: str) -> bool:+ """True if a robots directive list has nofollow (or none) for all crawlers.++ Directives after a user-agent prefix ("otherbot: noindex, nofollow") apply+ only to that crawler and are ignored.+ """+ scoped = False+ for token in value.split(","):+ token = token.strip().lower()+ name, sep, rest = token.partition(":")+ name = name.strip()+ if sep and name not in _VALUED_ROBOTS_DIRECTIVES and _CRAWLER_NAME.fullmatch(name):+ scoped = True+ token = rest.strip()+ if not scoped and token in ("nofollow", "none"):+ return True+ return False+++def _page_nofollow(meta_robots, headers: dict) -> bool:+ """True if meta robots or X-Robots-Tag tells crawlers not to follow page links."""+ if meta_robots and _robots_nofollow(meta_robots):+ return True+ return any(+ name.lower() == "x-robots-tag" and _robots_nofollow(value or "")+ for name, value in (headers or {}).items()+ )+++def _apply_get_status(result: dict, page_data: dict) -> bool:+ """Record a GET outcome on result; True only for a usable 200 response."""+ result["http_status"] = page_data.get("status_code")+ if page_data.get("error"):+ result["status"] = "error"+ result["error"] = page_data["error"]+ return False+ if page_data.get("redirect_chain"):+ result["redirect_url"] = page_data.get("url")+ if result["http_status"] == 404:+ result["status"] = "lost"+ return False+ if result["http_status"] != 200:+ result["status"] = "error"+ result["error"] = f"HTTP {result['http_status']}"+ return False+ return True+++# Lower rank wins when several links on the page point at the target.+_MATCH_RANK = {"exact_url": 0, "domain_match": 1, "subdomain_match": 2}++def verify_single_backlink(source_url: str, target_url: str,head_only: bool = False, timeout: int = 30) -> dict:"""@@ -115,6 +180,8 @@ def verify_single_backlink(source_url: str, target_url: str,"anchor_text": None,"rel_attributes": [],"link_context": None,+ "page_nofollow": False,+ "redirect_url": None,"error": None,}@@ -127,26 +194,31 @@ def verify_single_backlink(source_url: str, target_url: str,source_domain = urlparse(source_url).netloc_polite_delay(source_domain)- # Step 1: HEAD check+ # Step 1: HEAD check. Redirects are followed, so a 3xx never surfaces+ # here; a changed final URL is reported as redirect_url instead.head_result = _head_check(source_url, timeout=min(timeout, 15))result["http_status"] = head_result["status_code"]- if not head_result["exists"]:- if head_result["status_code"] == 404:- result["status"] = "lost"- elif head_result["status_code"] and 300 <= head_result["status_code"] < 400:- result["status"] = "moved"- result["redirect_url"] = head_result.get("redirect_url")- elif head_result["error"]:- result["status"] = "error"- result["error"] = head_result["error"]- else:- result["status"] = "error"- result["http_status"] = head_result["status_code"]+ if head_result.get("blocked"):+ result["status"] = "error"+ result["error"] = head_result["error"]return result+ if head_result["status_code"] == 404:+ result["status"] = "lost"+ return result++ if head_result["exists"]:+ result["redirect_url"] = head_result["redirect_url"]+if head_only:- result["status"] = "exists"+ # Many servers answer HEAD with 403/405 or fail it outright; only a+ # GET can tell whether the page exists.+ if not head_result["exists"]:+ _polite_delay(source_domain)+ if not _apply_get_status(result, fetch_page(source_url, timeout=timeout)):+ return result+ result["status"] = "moved" if result["redirect_url"] else "exists"result["target_found"] = None # Unknown without full checkreturn result@@ -154,10 +226,7 @@ def verify_single_backlink(source_url: str, target_url: str,_polite_delay(source_domain)page_data = fetch_page(source_url, timeout=timeout)- if page_data.get("error"):- result["status"] = "error"- result["error"] = page_data["error"]- result["http_status"] = page_data.get("status_code")+ if not _apply_get_status(result, page_data):return resultif not page_data.get("content"):@@ -165,8 +234,6 @@ def verify_single_backlink(source_url: str, target_url: str,result["error"] = "Page returned no content"return result- result["http_status"] = page_data.get("status_code", 200)-# Step 3: Parse HTML and find target linkparsed = parse_html(page_data["content"], base_url=source_url)all_links = parsed.get("links", {})@@ -178,6 +245,8 @@ def verify_single_backlink(source_url: str, target_url: str,raw_target_host = urlparse(target_url).netloc.lower()target_domain = raw_target_host[4:] if raw_target_host.startswith("www.") else raw_target_host+ best_link = None+ best_match_type = Nonefor link in all_page_links:link_href = link.get("href", "")if not link_href:@@ -197,14 +266,27 @@ def verify_single_backlink(source_url: str, target_url: str,else:continue+ # The first link of the best match type wins, so an earlier nav or+ # footer link to the same domain cannot mask the exact link.+ if best_match_type is None or _MATCH_RANK[match_type] < _MATCH_RANK[best_match_type]:+ best_link, best_match_type = link, match_type+ if match_type == "exact_url":+ break++ if best_link is not None:result["target_found"] = True- result["match_type"] = match_type- result["anchor_text"] = link.get("text", "").strip()[:200]- rel = link.get("rel", "")+ result["match_type"] = best_match_type+ result["anchor_text"] = best_link.get("text", "").strip()[:200]+ rel = best_link.get("rel", "")if rel:result["rel_attributes"] = rel.split() if isinstance(rel, str) else relelse:result["rel_attributes"] = ["follow"] # No rel = dofollow+ # Page-level nofollow is reported separately; rel_attributes stays+ # the link's own markup.+ result["page_nofollow"] = _page_nofollow(+ parsed.get("meta_robots"), page_data.get("headers", {})+ )result["status"] = "verified"return result@@ -384,7 +466,9 @@ def main():anchor_display = f" [{anchor[:30]}]" if anchor else ""rel = r.get("rel_attributes", [])rel_display = f" rel={','.join(rel)}" if rel and rel != ["follow"] else ""- print(f" [{status:13s}] {r.get('source_url', '?')}{anchor_display}{rel_display}")+ page_display = " page=nofollow" if r.get("page_nofollow") else ""+ print(f" [{status:13s}] {r.get('source_url', '?')}{anchor_display}{rel_display}"+ f"{page_display}")if __name__ == "__main__":diff --git a/tests/test_verify_backlinks.py b/tests/test_verify_backlinks.pynew file mode 100644index 0000000..5f973d9--- /dev/null+++ b/tests/test_verify_backlinks.py@@ -0,0 +1,225 @@+"""+Tests for link matching and status handling in scripts/verify_backlinks.py.++Network calls are mocked: `safe_requests_head` and `fetch_page` are replaced+with fakes so each test controls the HEAD status, the GET status, headers and+the HTML. The real `parse_html` runs on the fake HTML.+"""+import sys+from pathlib import Path++import pytest++REPO_ROOT = Path(__file__).resolve().parent.parent+sys.path.insert(0, str(REPO_ROOT / "scripts"))++import verify_backlinks # noqa: E402++SOURCE = "https://blog.example.org/post"+TARGET = "https://target.example/tool"+++class FakeHeadResponse:+ def __init__(self, status_code, url=SOURCE, redirected=False):+ self.status_code = status_code+ self.url = url+ self.history = [object()] if redirected else []+++def _page(html, status_code=200, headers=None, url=SOURCE):+ return {+ "url": url,+ "status_code": status_code,+ "content": html,+ "headers": headers or {},+ "redirect_chain": [],+ "error": None,+ }+++def fake_network(monkeypatch):+ """Install fakes; tests set `state["head"]` and `state["page"]`."""+ state = {"head": FakeHeadResponse(200), "page": None, "get_calls": 0}++ def fake_head(url, **kwargs):+ head = state["head"]+ if isinstance(head, Exception):+ raise head+ return head++ def fake_fetch(url, **kwargs):+ state["get_calls"] += 1+ return state["page"]++ monkeypatch.setattr(verify_backlinks, "DOMAIN_DELAY", 0)+ monkeypatch.setattr(verify_backlinks, "validate_url", lambda url: True)+ monkeypatch.setattr(verify_backlinks, "safe_requests_head", fake_head)+ monkeypatch.setattr(verify_backlinks, "fetch_page", fake_fetch)+ return state+++def _html(body, head=""):+ return f"<html><head>{head}</head><body>{body}</body></html>"+++def test_exact_link_wins_over_earlier_same_domain_link(fake_network):+ fake_network["page"] = _page(_html(+ '<nav><a href="https://target.example/" rel="nofollow">Home</a></nav>'+ '<p>Try <a href="https://target.example/tool">the tool</a>.</p>'+ ))+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)+ assert result["status"] == "verified"+ assert result["match_type"] == "exact_url"+ assert result["anchor_text"] == "the tool"+ assert result["rel_attributes"] == ["follow"]+++def test_domain_match_still_reported_when_no_exact_link(fake_network):+ fake_network["page"] = _page(_html(+ '<a href="https://sub.target.example/x">sub</a>'+ '<a href="https://target.example/other" rel="sponsored">other</a>'+ ))+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)+ assert result["match_type"] == "domain_match"+ assert result["anchor_text"] == "other"+ assert result["rel_attributes"] == ["sponsored"]+++def test_link_without_rel_and_no_page_directive_is_not_page_nofollow(fake_network):+ fake_network["page"] = _page(_html('<a href="https://target.example/tool">t</a>'))+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)+ assert result["rel_attributes"] == ["follow"]+ assert result["page_nofollow"] is False++[email protected]("content", ["noindex, nofollow", "NOFOLLOW", "none"])+def test_meta_robots_nofollow_is_reported(fake_network, content):+ fake_network["page"] = _page(_html(+ '<a href="https://target.example/tool">t</a>',+ head=f'<meta name="robots" content="{content}">',+ ))+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)+ assert result["status"] == "verified"+ assert result["page_nofollow"] is True+ assert result["rel_attributes"] == ["follow"]++[email protected]("value", ["nofollow", "noarchive, NoFollow", "none"])+def test_x_robots_tag_nofollow_is_reported(fake_network, value):+ fake_network["page"] = _page(+ _html('<a href="https://target.example/tool">t</a>'),+ headers={"x-robots-tag": value},+ )+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)+ assert result["page_nofollow"] is True++[email protected]("value", ["otherbot: nofollow", "otherbot: noindex, nofollow"])+def test_x_robots_tag_scoped_to_another_crawler_is_ignored(fake_network, value):+ fake_network["page"] = _page(+ _html('<a href="https://target.example/tool">t</a>'),+ headers={"X-Robots-Tag": value},+ )+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)+ assert result["page_nofollow"] is False+++def test_x_robots_tag_valued_directive_is_not_a_crawler_scope(fake_network):+ fake_network["page"] = _page(+ _html('<a href="https://target.example/tool">t</a>'),+ headers={"X-Robots-Tag": "max-snippet: 50, nofollow"},+ )+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)+ assert result["page_nofollow"] is True+++def test_x_robots_tag_date_with_commas_is_not_a_crawler_scope(fake_network):+ fake_network["page"] = _page(+ _html('<a href="https://target.example/tool">t</a>'),+ headers={"X-Robots-Tag": "unavailable_after: Wed, 03 Nov 2026 15:00:00 GMT, nofollow"},+ )+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)+ assert result["page_nofollow"] is True++[email protected]("head_status", [403, 405, 501])+def test_head_rejected_falls_back_to_get(fake_network, head_status):+ fake_network["head"] = FakeHeadResponse(head_status)+ fake_network["page"] = _page(_html('<a href="https://target.example/tool">t</a>'))+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)+ assert result["status"] == "verified"+ assert result["http_status"] == 200+ assert fake_network["get_calls"] == 1+++def test_head_request_error_falls_back_to_get(fake_network):+ fake_network["head"] = verify_backlinks.requests.exceptions.ConnectionError("reset")+ fake_network["page"] = _page(_html('<a href="https://target.example/tool">t</a>'))+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)+ assert result["status"] == "verified"+++def test_head_ssrf_block_does_not_fall_back_to_get(fake_network):+ fake_network["head"] = verify_backlinks.URLSafetyError("private address")+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)+ assert result["status"] == "error"+ assert "SSRF" in result["error"]+ assert fake_network["get_calls"] == 0+++def test_head_404_is_lost_without_get(fake_network):+ fake_network["head"] = FakeHeadResponse(404)+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)+ assert result["status"] == "lost"+ assert fake_network["get_calls"] == 0+++def test_get_fallback_404_is_lost(fake_network):+ fake_network["head"] = FakeHeadResponse(405)+ fake_network["page"] = _page("Not found", status_code=404)+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)+ assert result["status"] == "lost"+ assert result["http_status"] == 404+++def test_get_fallback_still_blocked_is_error(fake_network):+ fake_network["head"] = FakeHeadResponse(403)+ fake_network["page"] = _page(_html("Access denied"), status_code=403)+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)+ assert result["status"] == "error"+ assert result["http_status"] == 403+ assert result["target_found"] is False+++def test_head_only_falls_back_to_get_for_existence(fake_network):+ fake_network["head"] = FakeHeadResponse(405)+ fake_network["page"] = _page(_html("hello"))+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET, head_only=True)+ assert result["status"] == "exists"+ assert result["http_status"] == 200+++def test_head_only_redirect_is_moved(fake_network):+ fake_network["head"] = FakeHeadResponse(+ 200, url="https://blog.example.org/new-post", redirected=True+ )+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET, head_only=True)+ assert result["status"] == "moved"+ assert result["redirect_url"] == "https://blog.example.org/new-post"+++def test_head_only_url_normalization_alone_is_not_moved(fake_network):+ fake_network["head"] = FakeHeadResponse(200, url=SOURCE + "/")+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET, head_only=True)+ assert result["status"] == "exists"+ assert result["redirect_url"] is None+++def test_full_check_on_redirected_page_reports_redirect_url(fake_network):+ final = "https://blog.example.org/new-post"+ fake_network["head"] = FakeHeadResponse(200, url=final, redirected=True)+ fake_network["page"] = _page(_html('<a href="https://target.example/tool">t</a>'), url=final)+ result = verify_backlinks.verify_single_backlink(SOURCE, TARGET)+ assert result["status"] == "verified"+ assert result["redirect_url"] == final