Skip to content

Open-source SEO, GEO, and marketing skills for AI agents.

fix: decompose page-level traffic drops into query sub-issues with regression classifier (#58)

Tong Chen516ecd439342
diff --git a/openclaw/bin/weekly_review.py b/openclaw/bin/weekly_review.py
index 247a7148..b5ccac3d 100755
--- a/openclaw/bin/weekly_review.py
+++ b/openclaw/bin/weekly_review.py
@@ -297,7 +297,9 @@ def best_practice_alignment_for_issue(issue: dict[str, Any]) -> dict[str, Any]:
area = SEO_BEST_PRACTICE_AREAS[area_key]
notes = [area["why"]]
- if action_type in {"meta_tags", "snippet_content_packaging"}:
+ if action_type == "content_refresh":
+ notes.append("Content has lost ranking position; refresh with current data, intent alignment, and competitive comparison before republishing.")
+ elif action_type in {"meta_tags", "snippet_content_packaging"}:
notes.append("Validate the query/page/SERP fit before treating low CTR as a title or meta-description problem.")
elif action_type in {"query_intent_mapping", "local_intent_ownership"}:
notes.append("Resolve search intent ownership before changing content, redirects, canonicals, or metadata.")
@@ -416,6 +418,61 @@ def goal_alignment_for_query(query: str | None, brand_terms: list[str] | None) -
return 1.0, []
+def classify_regression_pattern(top_query: dict[str, Any]) -> tuple[str, str, list[str]]:
+ """Classify a losing query into an action type and best-practice area.
+
+ Examines which metric moved first to determine the root cause:
+
+ - Position drop + impression drop + CTR ~stable → ranking loss → content_refresh
+ - CTR drop + position stable + impressions ~stable → SERP displacement → snippet_content_packaging
+ - Impression drop + position stable + CTR stable → demand fade → query_intent_mapping
+ - Mixed/ambiguous → page_improvement (defer to operator)
+ """
+ pos_now = top_query.get("position_now")
+ pos_prev = top_query.get("position_prev")
+ pos_delta = None
+ if pos_now is not None and pos_prev is not None:
+ pos_delta = pos_now - pos_prev # positive = rank dropped (higher number = worse)
+
+ ctr_now = top_query.get("ctr_now") or 0
+ ctr_prev = top_query.get("ctr_prev") or 0
+ ctr_delta = ctr_now - ctr_prev
+
+ impr_now = top_query.get("impressions_now", 0)
+ impr_prev = top_query.get("impressions_prev", 0)
+ impr_delta = impr_now - impr_prev
+ impr_decline_pct = (impr_delta / max(impr_prev, 1)) * 100 if impr_prev > 0 else 0
+
+ notes = []
+
+ # Rank position drop is the strongest signal
+ if pos_delta is not None and pos_delta >= 1.5:
+ notes.append(f"Position dropped {pos_delta:+.1f} positions; likely ranking loss or competitive displacement")
+ return "content_refresh", "content_usefulness_trust", notes
+
+ if pos_delta is not None and pos_delta >= 0.5:
+ notes.append(f"Position drifted {pos_delta:+.1f} positions; mild ranking regression")
+ return "content_refresh", "content_usefulness_trust", notes
+
+ # CTR drop with stable position → SERP displacement
+ if ctr_delta < -2.0 and impr_decline_pct > -30:
+ notes.append(f"CTR dropped {ctr_delta:+.1f}% while position held; likely SERP feature displacement")
+ return "snippet_content_packaging", "on_page_relevance_serp_packaging", notes
+
+ if ctr_delta < -1.0 and pos_delta is not None and abs(pos_delta) < 0.5:
+ notes.append(f"Mild CTR decline {ctr_delta:+.1f}% with stable position; possible SERP layout change")
+ return "snippet_content_packaging", "on_page_relevance_serp_packaging", notes
+
+ # Impression drop without position or CTR change → demand fade / intent shift
+ if impr_decline_pct < -40 and ctr_delta > -1.0 and (pos_delta is None or abs(pos_delta) < 0.5):
+ notes.append(f"Impressions declined {impr_decline_pct:.0f}% without ranking or CTR change; possible intent shift or demand fade")
+ return "query_intent_mapping", "demand_intent_targeting", notes
+
+ # Mixed signals — keep as page_improvement
+ notes.append("Mixed or ambiguous regression signals; operator investigation required")
+ return "page_improvement", "content_usefulness_trust", notes
+
+
def decline_issue(page: dict[str, Any]) -> dict[str, Any]:
raw_target = page.get("page")
context = url_context(raw_target)
@@ -429,6 +486,31 @@ def decline_issue(page: dict[str, Any]) -> dict[str, Any]:
action_type = "page_improvement"
title_target = raw_target
notes = []
+
+ # Decompose with top-losing-queries from query-page GSC data
+ top_losing_queries = page.get("top_losing_queries", [])
+ if top_losing_queries:
+ top_losing_queries = [q for q in top_losing_queries if q.get("absolute_click_loss", 0) > 0]
+
+ primary_query: str | None = None
+ if top_losing_queries:
+ top = top_losing_queries[0]
+ total_lost = top.get("absolute_click_loss", 0)
+ share_of_loss = total_lost / max(lost_clicks, 1)
+ top_name = top.get("query", "")
+
+ if share_of_loss >= 0.3:
+ # Dominant losing query — reclassify action type based on regression pattern
+ primary_query = top_name
+ classified_type, classified_area, classify_notes = classify_regression_pattern(top)
+ action_type = classified_type
+ notes.extend(classify_notes)
+ notes.append(f"Top query '{primary_query}' drives {share_of_loss:.0%} of lost clicks ({total_lost:g} of {lost_clicks:g})")
+ else:
+ # Distributed across queries — keep page_improvement but attach context
+ notes.append(f"Top query '{top_name}' drives {share_of_loss:.0%} of lost clicks; decline is distributed")
+ notes.append("Distributed decline; diagnose SERP changes and content freshness across multiple queries")
+
if context["has_tracking_params"]:
action_type = "canonical_or_tracking_investigation"
actionability_score = 0.45
@@ -439,6 +521,25 @@ def decline_issue(page: dict[str, Any]) -> dict[str, Any]:
url_quality = float(context["url_quality_score"])
score = impact_score * confidence * actionability_score * url_quality
+
+ extra_kwargs: dict[str, Any] = {
+ "target": raw_target,
+ "base_priority": impact_score,
+ "expected_click_delta": round(-lost_clicks, 1),
+ "priority_score": round(score, 3),
+ "score_components": {
+ "expected_impact": round(impact_score, 3),
+ "confidence_score": confidence,
+ "goal_alignment_score": 1.0,
+ "actionability_score": actionability_score,
+ "url_quality_score": url_quality,
+ },
+ "operator_judgment_notes": notes,
+ "top_losing_queries": top_losing_queries,
+ }
+ if primary_query:
+ extra_kwargs["primary_query"] = primary_query
+
return make_issue(
f"Traffic dropped on {title_target}",
severity_from_score(score),
@@ -448,18 +549,7 @@ def decline_issue(page: dict[str, Any]) -> dict[str, Any]:
f"Current clicks: {clicks_now:g} | previous clicks: {clicks_prev:g} | lost clicks: {lost_clicks:g}",
],
action_type,
- target=raw_target,
- base_priority=impact_score,
- expected_click_delta=round(-lost_clicks, 1),
- priority_score=round(score, 3),
- score_components={
- "expected_impact": round(impact_score, 3),
- "confidence_score": confidence,
- "goal_alignment_score": 1.0,
- "actionability_score": actionability_score,
- "url_quality_score": url_quality,
- },
- operator_judgment_notes=notes,
+ **extra_kwargs,
)
@@ -650,7 +740,19 @@ def derive_candidate_issues(data: dict[str, Any]) -> list[dict[str, Any]]:
declining_queries = comparison.get("declining_queries") or []
brand_terms = data.get("_brand_terms") or []
- issues.extend(decline_issue(page) for page in declining_pages[:8])
+ # Build lookup for query-page decompositions (from GSC comparison analysis)
+ page_query_lookup: dict[str, list[dict[str, Any]]] = {}
+ for decomp in (comparison.get("page_query_decompositions") or []):
+ page_url = decomp.get("page", "")
+ losing_queries = decomp.get("top_losing_queries", [])
+ if page_url and losing_queries:
+ page_query_lookup[page_url] = losing_queries
+
+ for page in declining_pages[:8]:
+ page_url = page.get("page", "")
+ top_losing = page_query_lookup.get(page_url, [])
+ page["top_losing_queries"] = top_losing
+ issues.extend([decline_issue(page)])
issues.extend(ctr_gap_issue(gap, brand_terms) for gap in ctr_gaps[:8])
issues.extend(cannibalization_issue(cannibal, brand_terms) for cannibal in cannibalization[:5])
issues.extend(query_ctr_issue(opp, brand_terms) for opp in ctr_opps[:5])
@@ -1018,7 +1120,7 @@ def add_deep_dive_diagnostics(issues: list[dict[str, Any]], site_id: str, site_p
enriched = []
for issue in issues:
updated = dict(issue)
- if issue.get("recommended_action_type") in {"snippet_content_packaging", "meta_tags", "page_improvement", "query_intent_mapping", "local_intent_ownership"}:
+ if issue.get("recommended_action_type") in {"snippet_content_packaging", "meta_tags", "page_improvement", "query_intent_mapping", "local_intent_ownership", "content_refresh"}:
updated["deep_dive"] = deep_dive_diagnostic(issue, site_id, site_profile, live=live)
enriched.append(updated)
return enriched
@@ -1364,6 +1466,8 @@ def build_payload(
"deep_dive": issue.get("deep_dive"),
"business_context_adjustment": issue.get("business_context_adjustment"),
"best_practice_alignment": issue.get("best_practice_alignment"),
+ "primary_query": issue.get("primary_query"),
+ "top_losing_queries": issue.get("top_losing_queries", []),
"consolidation_targets": issue.get("consolidation_targets"),
"source_target": issue.get("source_target"),
"needs_business_context": context_check["score"] < 0.75,
@@ -1394,6 +1498,8 @@ def build_payload(
"deep_dive": issue.get("deep_dive"),
"business_context_adjustment": issue.get("business_context_adjustment"),
"best_practice_alignment": issue.get("best_practice_alignment"),
+ "primary_query": issue.get("primary_query"),
+ "top_losing_queries": issue.get("top_losing_queries", []),
"consolidation_targets": issue.get("consolidation_targets"),
"source_target": issue.get("source_target"),
"business_context_score": context_check["score"],
@@ -1515,6 +1621,16 @@ def build_user_message(site_id: str, result: dict[str, Any], payload: dict[str,
f"snippet {'yes' if checks.get('current_snippet_inspected') else 'no'}, "
f"above-fold {'yes' if checks.get('above_the_fold_inspected') else 'no'})"
)
+ # Surface top-losing-query decomposition if available
+ top_losing = top_action.get("top_losing_queries", [])
+ if top_losing:
+ top_query = top_losing[0]
+ lines.append(
+ f"- Top losing query: `{top_query.get('query', '?')}` "
+ f"(lost {top_query.get('absolute_click_loss', 0):g} clicks, "
+ f"pos {top_query.get('position_prev', '?')}→{top_query.get('position_now', '?')}, "
+ f"CTR {top_query.get('ctr_prev', '?')}%→{top_query.get('ctr_now', '?')}%)"
+ )
if proposal_path:
lines.extend(["", f"Proposal file: `{proposal_path}`"])
if context_request:
diff --git a/openclaw/tests/test_weekly_review_scoring.py b/openclaw/tests/test_weekly_review_scoring.py
index 56071270..6c8c6e8d 100644
--- a/openclaw/tests/test_weekly_review_scoring.py
+++ b/openclaw/tests/test_weekly_review_scoring.py
@@ -523,6 +523,166 @@ class WeeklyReviewScoringTest(unittest.TestCase):
self.assertEqual(alignment["primary_area"], area)
self.assertNotEqual(alignment["primary_area"], "unmapped")
+ def test_classify_ranking_loss_pattern(self) -> None:
+ """Position drop ≥1.5 → content_refresh, content_usefulness_trust."""
+ top_query = {
+ "query": "dog sitting seattle",
+ "position_now": 8.5,
+ "position_prev": 5.2,
+ "ctr_now": 3.0,
+ "ctr_prev": 5.0,
+ "impressions_now": 500,
+ "impressions_prev": 1200,
+ }
+ action_type, area, notes = weekly_review.classify_regression_pattern(top_query)
+ self.assertEqual(action_type, "content_refresh")
+ self.assertEqual(area, "content_usefulness_trust")
+ self.assertTrue(any("dropped" in n for n in notes))
+
+ def test_classify_serp_displacement_pattern(self) -> None:
+ """CTR drop with stable position → snippet_content_packaging."""
+ top_query = {
+ "query": "dog sitting seattle",
+ "position_now": 3.5,
+ "position_prev": 3.2,
+ "ctr_now": 2.0,
+ "ctr_prev": 6.0,
+ "impressions_now": 1200,
+ "impressions_prev": 1100,
+ }
+ action_type, area, notes = weekly_review.classify_regression_pattern(top_query)
+ self.assertEqual(action_type, "snippet_content_packaging")
+ self.assertEqual(area, "on_page_relevance_serp_packaging")
+ self.assertTrue(any("displacement" in n for n in notes))
+
+ def test_classify_demand_fade_pattern(self) -> None:
+ """Impression drop without position/CTR change → query_intent_mapping."""
+ top_query = {
+ "query": "dog sitting seattle",
+ "position_now": 4.5,
+ "position_prev": 4.2,
+ "ctr_now": 5.0,
+ "ctr_prev": 5.5,
+ "impressions_now": 200,
+ "impressions_prev": 800,
+ }
+ action_type, area, notes = weekly_review.classify_regression_pattern(top_query)
+ self.assertEqual(action_type, "query_intent_mapping")
+ self.assertEqual(area, "demand_intent_targeting")
+ self.assertTrue(any("intent" in n or "demand" in n for n in notes))
+
+ def test_classify_mixed_pattern(self) -> None:
+ """Small changes across metrics → page_improvement."""
+ top_query = {
+ "query": "dog sitting seattle",
+ "position_now": 4.5,
+ "position_prev": 4.3,
+ "ctr_now": 4.8,
+ "ctr_prev": 5.0,
+ "impressions_now": 800,
+ "impressions_prev": 850,
+ }
+ action_type, area, notes = weekly_review.classify_regression_pattern(top_query)
+ self.assertEqual(action_type, "page_improvement")
+ self.assertTrue(any("Mixed" in n for n in notes))
+
+ def test_classify_no_position_data(self) -> None:
+ """Missing position data → fall through to other signals."""
+ top_query = {
+ "query": "dog sitting",
+ "ctr_now": 2.0,
+ "ctr_prev": 6.0,
+ "impressions_now": 500,
+ "impressions_prev": 520,
+ }
+ action_type, area, notes = weekly_review.classify_regression_pattern(top_query)
+ self.assertEqual(action_type, "snippet_content_packaging")
+
+ def test_decline_issue_uses_top_losing_query_ranking_loss(self) -> None:
+ """decline_issue should set primary_query and reclassify to content_refresh when a top query shows ranking loss."""
+ page = {
+ "page": "https://example.com/dog-sitting-seattle",
+ "clicks_now": 50,
+ "clicks_prev": 100,
+ "change_pct": -50.0,
+ "top_losing_queries": [
+ {
+ "query": "dog sitting seattle",
+ "clicks_now": 30,
+ "clicks_prev": 70,
+ "absolute_click_loss": 40,
+ "change_pct": -57.1,
+ "position_now": 7.5,
+ "position_prev": 4.0,
+ "ctr_now": 3.0,
+ "ctr_prev": 5.0,
+ "impressions_now": 600,
+ "impressions_prev": 1400,
+ }
+ ],
+ }
+ issue = weekly_review.decline_issue(page)
+ self.assertEqual(issue["primary_query"], "dog sitting seattle")
+ self.assertEqual(issue["recommended_action_type"], "content_refresh")
+ self.assertIn("top_losing_queries", issue)
+ self.assertEqual(len(issue["top_losing_queries"]), 1)
+
+ def test_decline_issue_uses_top_losing_query_serp_displacement(self) -> None:
+ """decline_issue should reclassify to snippet_content_packaging when CTR drops with stable position."""
+ page = {
+ "page": "https://example.com/dog-sitting-seattle",
+ "clicks_now": 60,
+ "clicks_prev": 100,
+ "change_pct": -40.0,
+ "top_losing_queries": [
+ {
+ "query": "dog sitting seattle",
+ "clicks_now": 35,
+ "clicks_prev": 65,
+ "absolute_click_loss": 30,
+ "change_pct": -46.2,
+ "position_now": 3.5,
+ "position_prev": 3.2,
+ "ctr_now": 2.0,
+ "ctr_prev": 6.0,
+ "impressions_now": 1200,
+ "impressions_prev": 1100,
+ }
+ ],
+ }
+ issue = weekly_review.decline_issue(page)
+ self.assertEqual(issue["recommended_action_type"], "snippet_content_packaging")
+ self.assertEqual(issue["primary_query"], "dog sitting seattle")
+
+ def test_decline_issue_distributed_no_primary_query(self) -> None:
+ """When top query drives <30% of lost clicks, don't set primary_query or reclassify."""
+ page = {
+ "page": "https://example.com/dog-sitting-seattle",
+ "clicks_now": 80,
+ "clicks_prev": 100,
+ "change_pct": -20.0,
+ "top_losing_queries": [
+ {
+ "query": "dog sitting seattle",
+ "clicks_now": 45,
+ "clicks_prev": 50,
+ "absolute_click_loss": 5,
+ "change_pct": -10.0,
+ "position_now": 3.5,
+ "position_prev": 3.2,
+ "ctr_now": 4.0,
+ "ctr_prev": 4.5,
+ "impressions_now": 1100,
+ "impressions_prev": 1150,
+ }
+ ],
+ }
+ issue = weekly_review.decline_issue(page)
+ # primary_query should be set (it's still the top query) but action_type stays page_improvement
+ # because share_of_loss = 5/20 = 25% < 30%
+ self.assertNotIn("primary_query", issue)
+ self.assertEqual(issue["recommended_action_type"], "page_improvement")
+
def test_unknown_action_type_warns_as_unmapped(self) -> None:
alignment = weekly_review.best_practice_alignment_for_issue({"recommended_action_type": "core_web_vitals"})
diff --git a/seo/seo-analysis/scripts/analyze_gsc.py b/seo/seo-analysis/scripts/analyze_gsc.py
index f9a00c0a..0470ba6f 100644
--- a/seo/seo-analysis/scripts/analyze_gsc.py
+++ b/seo/seo-analysis/scripts/analyze_gsc.py
@@ -254,11 +254,71 @@ def pull_period_comparison(token, site, days):
})
query_changes.sort(key=lambda x: (-x.get("absolute_click_loss", 0), x["change_pct"]))
+ # --- Decompose each declining page into its top losing queries ---
+ # For each page that lost meaningful traffic, fetch its top queries from
+ # both periods so we can see which queries drove the decline.
+ page_query_decompositions = []
+ for pg in page_changes[:10]:
+ page_url = pg["page"]
+ # Fetch top queries for this page URL in both periods
+ def _page_queries(start, end):
+ body = {
+ "startDate": start.isoformat(), "endDate": end.isoformat(),
+ "dimensions": ["query"],
+ "dimensionFilterGroups": [{
+ "filters": [{"dimension": "page", "operator": "equals", "expression": page_url}]
+ }],
+ "rowLimit": 25,
+ "orderBy": [{"fieldName": "clicks", "sortOrder": "DESCENDING"}]
+ }
+ data = gsc_query(token, site, body)
+ return {r["keys"][0]: r for r in data.get("rows", [])}
+
+ curr_page_q = _page_queries(start_curr, end_curr)
+ prev_page_q = _page_queries(start_prev, end_prev)
+
+ # Compute query-level deltas — only for queries present in BOTH periods
+ # to avoid inflating losses from truncated top-25 results.
+ page_losing_queries = []
+ shared_queries = set(curr_page_q.keys()) & set(prev_page_q.keys())
+ for q in shared_queries:
+ c = curr_page_q[q]
+ p = prev_page_q[q]
+ clicks_now = c.get("clicks", 0)
+ clicks_prev = p.get("clicks", 0)
+ click_delta = clicks_now - clicks_prev
+ if click_delta >= 0:
+ continue # only losing queries matter
+ pct = round((click_delta / max(clicks_prev, 1)) * 100, 1)
+ page_losing_queries.append({
+ "query": q,
+ "clicks_now": clicks_now,
+ "clicks_prev": clicks_prev,
+ "click_delta": click_delta,
+ "absolute_click_loss": abs(click_delta),
+ "change_pct": pct,
+ "impressions_now": c.get("impressions", 0),
+ "impressions_prev": p.get("impressions", 0),
+ "impression_delta": c.get("impressions", 0) - p.get("impressions", 0),
+ "ctr_now": round(c.get("ctr", 0) * 100, 2) if c else 0,
+ "ctr_prev": round(p.get("ctr", 0) * 100, 2) if p else 0,
+ "position_now": round(c.get("position", 0), 1) if c else None,
+ "position_prev": round(p.get("position", 0), 1) if p else None,
+ })
+
+ page_losing_queries.sort(key=lambda x: -x["absolute_click_loss"])
+ if page_losing_queries:
+ page_query_decompositions.append({
+ "page": page_url,
+ "top_losing_queries": page_losing_queries[:10],
+ })
+
return {
"period": f"{start_curr.isoformat()} to {end_curr.isoformat()}",
"prior_period": f"{start_prev.isoformat()} to {end_prev.isoformat()}",
"declining_pages": page_changes[:20],
- "declining_queries": query_changes[:20]
+ "declining_queries": query_changes[:20],
+ "page_query_decompositions": page_query_decompositions,
}