Open-source SEO, GEO, and marketing skills for AI agents.
fix: decompose page-level traffic drops into query sub-issues with regression classifier (#58)
Tong Chen516ecd439342
diff --git a/openclaw/bin/weekly_review.py b/openclaw/bin/weekly_review.pyindex 247a7148..b5ccac3d 100755--- a/openclaw/bin/weekly_review.py+++ b/openclaw/bin/weekly_review.py@@ -297,7 +297,9 @@ def best_practice_alignment_for_issue(issue: dict[str, Any]) -> dict[str, Any]:area = SEO_BEST_PRACTICE_AREAS[area_key]notes = [area["why"]]- if action_type in {"meta_tags", "snippet_content_packaging"}:+ if action_type == "content_refresh":+ notes.append("Content has lost ranking position; refresh with current data, intent alignment, and competitive comparison before republishing.")+ elif action_type in {"meta_tags", "snippet_content_packaging"}:notes.append("Validate the query/page/SERP fit before treating low CTR as a title or meta-description problem.")elif action_type in {"query_intent_mapping", "local_intent_ownership"}:notes.append("Resolve search intent ownership before changing content, redirects, canonicals, or metadata.")@@ -416,6 +418,61 @@ def goal_alignment_for_query(query: str | None, brand_terms: list[str] | None) -return 1.0, []+def classify_regression_pattern(top_query: dict[str, Any]) -> tuple[str, str, list[str]]:+ """Classify a losing query into an action type and best-practice area.++ Examines which metric moved first to determine the root cause:++ - Position drop + impression drop + CTR ~stable → ranking loss → content_refresh+ - CTR drop + position stable + impressions ~stable → SERP displacement → snippet_content_packaging+ - Impression drop + position stable + CTR stable → demand fade → query_intent_mapping+ - Mixed/ambiguous → page_improvement (defer to operator)+ """+ pos_now = top_query.get("position_now")+ pos_prev = top_query.get("position_prev")+ pos_delta = None+ if pos_now is not None and pos_prev is not None:+ pos_delta = pos_now - pos_prev # positive = rank dropped (higher number = worse)++ ctr_now = top_query.get("ctr_now") or 0+ ctr_prev = top_query.get("ctr_prev") or 0+ ctr_delta = ctr_now - ctr_prev++ impr_now = top_query.get("impressions_now", 0)+ impr_prev = top_query.get("impressions_prev", 0)+ impr_delta = impr_now - impr_prev+ impr_decline_pct = (impr_delta / max(impr_prev, 1)) * 100 if impr_prev > 0 else 0++ notes = []++ # Rank position drop is the strongest signal+ if pos_delta is not None and pos_delta >= 1.5:+ notes.append(f"Position dropped {pos_delta:+.1f} positions; likely ranking loss or competitive displacement")+ return "content_refresh", "content_usefulness_trust", notes++ if pos_delta is not None and pos_delta >= 0.5:+ notes.append(f"Position drifted {pos_delta:+.1f} positions; mild ranking regression")+ return "content_refresh", "content_usefulness_trust", notes++ # CTR drop with stable position → SERP displacement+ if ctr_delta < -2.0 and impr_decline_pct > -30:+ notes.append(f"CTR dropped {ctr_delta:+.1f}% while position held; likely SERP feature displacement")+ return "snippet_content_packaging", "on_page_relevance_serp_packaging", notes++ if ctr_delta < -1.0 and pos_delta is not None and abs(pos_delta) < 0.5:+ notes.append(f"Mild CTR decline {ctr_delta:+.1f}% with stable position; possible SERP layout change")+ return "snippet_content_packaging", "on_page_relevance_serp_packaging", notes++ # Impression drop without position or CTR change → demand fade / intent shift+ if impr_decline_pct < -40 and ctr_delta > -1.0 and (pos_delta is None or abs(pos_delta) < 0.5):+ notes.append(f"Impressions declined {impr_decline_pct:.0f}% without ranking or CTR change; possible intent shift or demand fade")+ return "query_intent_mapping", "demand_intent_targeting", notes++ # Mixed signals — keep as page_improvement+ notes.append("Mixed or ambiguous regression signals; operator investigation required")+ return "page_improvement", "content_usefulness_trust", notes++def decline_issue(page: dict[str, Any]) -> dict[str, Any]:raw_target = page.get("page")context = url_context(raw_target)@@ -429,6 +486,31 @@ def decline_issue(page: dict[str, Any]) -> dict[str, Any]:action_type = "page_improvement"title_target = raw_targetnotes = []++ # Decompose with top-losing-queries from query-page GSC data+ top_losing_queries = page.get("top_losing_queries", [])+ if top_losing_queries:+ top_losing_queries = [q for q in top_losing_queries if q.get("absolute_click_loss", 0) > 0]++ primary_query: str | None = None+ if top_losing_queries:+ top = top_losing_queries[0]+ total_lost = top.get("absolute_click_loss", 0)+ share_of_loss = total_lost / max(lost_clicks, 1)+ top_name = top.get("query", "")++ if share_of_loss >= 0.3:+ # Dominant losing query — reclassify action type based on regression pattern+ primary_query = top_name+ classified_type, classified_area, classify_notes = classify_regression_pattern(top)+ action_type = classified_type+ notes.extend(classify_notes)+ notes.append(f"Top query '{primary_query}' drives {share_of_loss:.0%} of lost clicks ({total_lost:g} of {lost_clicks:g})")+ else:+ # Distributed across queries — keep page_improvement but attach context+ notes.append(f"Top query '{top_name}' drives {share_of_loss:.0%} of lost clicks; decline is distributed")+ notes.append("Distributed decline; diagnose SERP changes and content freshness across multiple queries")+if context["has_tracking_params"]:action_type = "canonical_or_tracking_investigation"actionability_score = 0.45@@ -439,6 +521,25 @@ def decline_issue(page: dict[str, Any]) -> dict[str, Any]:url_quality = float(context["url_quality_score"])score = impact_score * confidence * actionability_score * url_quality++ extra_kwargs: dict[str, Any] = {+ "target": raw_target,+ "base_priority": impact_score,+ "expected_click_delta": round(-lost_clicks, 1),+ "priority_score": round(score, 3),+ "score_components": {+ "expected_impact": round(impact_score, 3),+ "confidence_score": confidence,+ "goal_alignment_score": 1.0,+ "actionability_score": actionability_score,+ "url_quality_score": url_quality,+ },+ "operator_judgment_notes": notes,+ "top_losing_queries": top_losing_queries,+ }+ if primary_query:+ extra_kwargs["primary_query"] = primary_query+return make_issue(f"Traffic dropped on {title_target}",severity_from_score(score),@@ -448,18 +549,7 @@ def decline_issue(page: dict[str, Any]) -> dict[str, Any]:f"Current clicks: {clicks_now:g} | previous clicks: {clicks_prev:g} | lost clicks: {lost_clicks:g}",],action_type,- target=raw_target,- base_priority=impact_score,- expected_click_delta=round(-lost_clicks, 1),- priority_score=round(score, 3),- score_components={- "expected_impact": round(impact_score, 3),- "confidence_score": confidence,- "goal_alignment_score": 1.0,- "actionability_score": actionability_score,- "url_quality_score": url_quality,- },- operator_judgment_notes=notes,+ **extra_kwargs,)@@ -650,7 +740,19 @@ def derive_candidate_issues(data: dict[str, Any]) -> list[dict[str, Any]]:declining_queries = comparison.get("declining_queries") or []brand_terms = data.get("_brand_terms") or []- issues.extend(decline_issue(page) for page in declining_pages[:8])+ # Build lookup for query-page decompositions (from GSC comparison analysis)+ page_query_lookup: dict[str, list[dict[str, Any]]] = {}+ for decomp in (comparison.get("page_query_decompositions") or []):+ page_url = decomp.get("page", "")+ losing_queries = decomp.get("top_losing_queries", [])+ if page_url and losing_queries:+ page_query_lookup[page_url] = losing_queries++ for page in declining_pages[:8]:+ page_url = page.get("page", "")+ top_losing = page_query_lookup.get(page_url, [])+ page["top_losing_queries"] = top_losing+ issues.extend([decline_issue(page)])issues.extend(ctr_gap_issue(gap, brand_terms) for gap in ctr_gaps[:8])issues.extend(cannibalization_issue(cannibal, brand_terms) for cannibal in cannibalization[:5])issues.extend(query_ctr_issue(opp, brand_terms) for opp in ctr_opps[:5])@@ -1018,7 +1120,7 @@ def add_deep_dive_diagnostics(issues: list[dict[str, Any]], site_id: str, site_penriched = []for issue in issues:updated = dict(issue)- if issue.get("recommended_action_type") in {"snippet_content_packaging", "meta_tags", "page_improvement", "query_intent_mapping", "local_intent_ownership"}:+ if issue.get("recommended_action_type") in {"snippet_content_packaging", "meta_tags", "page_improvement", "query_intent_mapping", "local_intent_ownership", "content_refresh"}:updated["deep_dive"] = deep_dive_diagnostic(issue, site_id, site_profile, live=live)enriched.append(updated)return enriched@@ -1364,6 +1466,8 @@ def build_payload("deep_dive": issue.get("deep_dive"),"business_context_adjustment": issue.get("business_context_adjustment"),"best_practice_alignment": issue.get("best_practice_alignment"),+ "primary_query": issue.get("primary_query"),+ "top_losing_queries": issue.get("top_losing_queries", []),"consolidation_targets": issue.get("consolidation_targets"),"source_target": issue.get("source_target"),"needs_business_context": context_check["score"] < 0.75,@@ -1394,6 +1498,8 @@ def build_payload("deep_dive": issue.get("deep_dive"),"business_context_adjustment": issue.get("business_context_adjustment"),"best_practice_alignment": issue.get("best_practice_alignment"),+ "primary_query": issue.get("primary_query"),+ "top_losing_queries": issue.get("top_losing_queries", []),"consolidation_targets": issue.get("consolidation_targets"),"source_target": issue.get("source_target"),"business_context_score": context_check["score"],@@ -1515,6 +1621,16 @@ def build_user_message(site_id: str, result: dict[str, Any], payload: dict[str,f"snippet {'yes' if checks.get('current_snippet_inspected') else 'no'}, "f"above-fold {'yes' if checks.get('above_the_fold_inspected') else 'no'})")+ # Surface top-losing-query decomposition if available+ top_losing = top_action.get("top_losing_queries", [])+ if top_losing:+ top_query = top_losing[0]+ lines.append(+ f"- Top losing query: `{top_query.get('query', '?')}` "+ f"(lost {top_query.get('absolute_click_loss', 0):g} clicks, "+ f"pos {top_query.get('position_prev', '?')}→{top_query.get('position_now', '?')}, "+ f"CTR {top_query.get('ctr_prev', '?')}%→{top_query.get('ctr_now', '?')}%)"+ )if proposal_path:lines.extend(["", f"Proposal file: `{proposal_path}`"])if context_request:diff --git a/openclaw/tests/test_weekly_review_scoring.py b/openclaw/tests/test_weekly_review_scoring.pyindex 56071270..6c8c6e8d 100644--- a/openclaw/tests/test_weekly_review_scoring.py+++ b/openclaw/tests/test_weekly_review_scoring.py@@ -523,6 +523,166 @@ class WeeklyReviewScoringTest(unittest.TestCase):self.assertEqual(alignment["primary_area"], area)self.assertNotEqual(alignment["primary_area"], "unmapped")+ def test_classify_ranking_loss_pattern(self) -> None:+ """Position drop ≥1.5 → content_refresh, content_usefulness_trust."""+ top_query = {+ "query": "dog sitting seattle",+ "position_now": 8.5,+ "position_prev": 5.2,+ "ctr_now": 3.0,+ "ctr_prev": 5.0,+ "impressions_now": 500,+ "impressions_prev": 1200,+ }+ action_type, area, notes = weekly_review.classify_regression_pattern(top_query)+ self.assertEqual(action_type, "content_refresh")+ self.assertEqual(area, "content_usefulness_trust")+ self.assertTrue(any("dropped" in n for n in notes))++ def test_classify_serp_displacement_pattern(self) -> None:+ """CTR drop with stable position → snippet_content_packaging."""+ top_query = {+ "query": "dog sitting seattle",+ "position_now": 3.5,+ "position_prev": 3.2,+ "ctr_now": 2.0,+ "ctr_prev": 6.0,+ "impressions_now": 1200,+ "impressions_prev": 1100,+ }+ action_type, area, notes = weekly_review.classify_regression_pattern(top_query)+ self.assertEqual(action_type, "snippet_content_packaging")+ self.assertEqual(area, "on_page_relevance_serp_packaging")+ self.assertTrue(any("displacement" in n for n in notes))++ def test_classify_demand_fade_pattern(self) -> None:+ """Impression drop without position/CTR change → query_intent_mapping."""+ top_query = {+ "query": "dog sitting seattle",+ "position_now": 4.5,+ "position_prev": 4.2,+ "ctr_now": 5.0,+ "ctr_prev": 5.5,+ "impressions_now": 200,+ "impressions_prev": 800,+ }+ action_type, area, notes = weekly_review.classify_regression_pattern(top_query)+ self.assertEqual(action_type, "query_intent_mapping")+ self.assertEqual(area, "demand_intent_targeting")+ self.assertTrue(any("intent" in n or "demand" in n for n in notes))++ def test_classify_mixed_pattern(self) -> None:+ """Small changes across metrics → page_improvement."""+ top_query = {+ "query": "dog sitting seattle",+ "position_now": 4.5,+ "position_prev": 4.3,+ "ctr_now": 4.8,+ "ctr_prev": 5.0,+ "impressions_now": 800,+ "impressions_prev": 850,+ }+ action_type, area, notes = weekly_review.classify_regression_pattern(top_query)+ self.assertEqual(action_type, "page_improvement")+ self.assertTrue(any("Mixed" in n for n in notes))++ def test_classify_no_position_data(self) -> None:+ """Missing position data → fall through to other signals."""+ top_query = {+ "query": "dog sitting",+ "ctr_now": 2.0,+ "ctr_prev": 6.0,+ "impressions_now": 500,+ "impressions_prev": 520,+ }+ action_type, area, notes = weekly_review.classify_regression_pattern(top_query)+ self.assertEqual(action_type, "snippet_content_packaging")++ def test_decline_issue_uses_top_losing_query_ranking_loss(self) -> None:+ """decline_issue should set primary_query and reclassify to content_refresh when a top query shows ranking loss."""+ page = {+ "page": "https://example.com/dog-sitting-seattle",+ "clicks_now": 50,+ "clicks_prev": 100,+ "change_pct": -50.0,+ "top_losing_queries": [+ {+ "query": "dog sitting seattle",+ "clicks_now": 30,+ "clicks_prev": 70,+ "absolute_click_loss": 40,+ "change_pct": -57.1,+ "position_now": 7.5,+ "position_prev": 4.0,+ "ctr_now": 3.0,+ "ctr_prev": 5.0,+ "impressions_now": 600,+ "impressions_prev": 1400,+ }+ ],+ }+ issue = weekly_review.decline_issue(page)+ self.assertEqual(issue["primary_query"], "dog sitting seattle")+ self.assertEqual(issue["recommended_action_type"], "content_refresh")+ self.assertIn("top_losing_queries", issue)+ self.assertEqual(len(issue["top_losing_queries"]), 1)++ def test_decline_issue_uses_top_losing_query_serp_displacement(self) -> None:+ """decline_issue should reclassify to snippet_content_packaging when CTR drops with stable position."""+ page = {+ "page": "https://example.com/dog-sitting-seattle",+ "clicks_now": 60,+ "clicks_prev": 100,+ "change_pct": -40.0,+ "top_losing_queries": [+ {+ "query": "dog sitting seattle",+ "clicks_now": 35,+ "clicks_prev": 65,+ "absolute_click_loss": 30,+ "change_pct": -46.2,+ "position_now": 3.5,+ "position_prev": 3.2,+ "ctr_now": 2.0,+ "ctr_prev": 6.0,+ "impressions_now": 1200,+ "impressions_prev": 1100,+ }+ ],+ }+ issue = weekly_review.decline_issue(page)+ self.assertEqual(issue["recommended_action_type"], "snippet_content_packaging")+ self.assertEqual(issue["primary_query"], "dog sitting seattle")++ def test_decline_issue_distributed_no_primary_query(self) -> None:+ """When top query drives <30% of lost clicks, don't set primary_query or reclassify."""+ page = {+ "page": "https://example.com/dog-sitting-seattle",+ "clicks_now": 80,+ "clicks_prev": 100,+ "change_pct": -20.0,+ "top_losing_queries": [+ {+ "query": "dog sitting seattle",+ "clicks_now": 45,+ "clicks_prev": 50,+ "absolute_click_loss": 5,+ "change_pct": -10.0,+ "position_now": 3.5,+ "position_prev": 3.2,+ "ctr_now": 4.0,+ "ctr_prev": 4.5,+ "impressions_now": 1100,+ "impressions_prev": 1150,+ }+ ],+ }+ issue = weekly_review.decline_issue(page)+ # primary_query should be set (it's still the top query) but action_type stays page_improvement+ # because share_of_loss = 5/20 = 25% < 30%+ self.assertNotIn("primary_query", issue)+ self.assertEqual(issue["recommended_action_type"], "page_improvement")+def test_unknown_action_type_warns_as_unmapped(self) -> None:alignment = weekly_review.best_practice_alignment_for_issue({"recommended_action_type": "core_web_vitals"})diff --git a/seo/seo-analysis/scripts/analyze_gsc.py b/seo/seo-analysis/scripts/analyze_gsc.pyindex f9a00c0a..0470ba6f 100644--- a/seo/seo-analysis/scripts/analyze_gsc.py+++ b/seo/seo-analysis/scripts/analyze_gsc.py@@ -254,11 +254,71 @@ def pull_period_comparison(token, site, days):})query_changes.sort(key=lambda x: (-x.get("absolute_click_loss", 0), x["change_pct"]))+ # --- Decompose each declining page into its top losing queries ---+ # For each page that lost meaningful traffic, fetch its top queries from+ # both periods so we can see which queries drove the decline.+ page_query_decompositions = []+ for pg in page_changes[:10]:+ page_url = pg["page"]+ # Fetch top queries for this page URL in both periods+ def _page_queries(start, end):+ body = {+ "startDate": start.isoformat(), "endDate": end.isoformat(),+ "dimensions": ["query"],+ "dimensionFilterGroups": [{+ "filters": [{"dimension": "page", "operator": "equals", "expression": page_url}]+ }],+ "rowLimit": 25,+ "orderBy": [{"fieldName": "clicks", "sortOrder": "DESCENDING"}]+ }+ data = gsc_query(token, site, body)+ return {r["keys"][0]: r for r in data.get("rows", [])}++ curr_page_q = _page_queries(start_curr, end_curr)+ prev_page_q = _page_queries(start_prev, end_prev)++ # Compute query-level deltas — only for queries present in BOTH periods+ # to avoid inflating losses from truncated top-25 results.+ page_losing_queries = []+ shared_queries = set(curr_page_q.keys()) & set(prev_page_q.keys())+ for q in shared_queries:+ c = curr_page_q[q]+ p = prev_page_q[q]+ clicks_now = c.get("clicks", 0)+ clicks_prev = p.get("clicks", 0)+ click_delta = clicks_now - clicks_prev+ if click_delta >= 0:+ continue # only losing queries matter+ pct = round((click_delta / max(clicks_prev, 1)) * 100, 1)+ page_losing_queries.append({+ "query": q,+ "clicks_now": clicks_now,+ "clicks_prev": clicks_prev,+ "click_delta": click_delta,+ "absolute_click_loss": abs(click_delta),+ "change_pct": pct,+ "impressions_now": c.get("impressions", 0),+ "impressions_prev": p.get("impressions", 0),+ "impression_delta": c.get("impressions", 0) - p.get("impressions", 0),+ "ctr_now": round(c.get("ctr", 0) * 100, 2) if c else 0,+ "ctr_prev": round(p.get("ctr", 0) * 100, 2) if p else 0,+ "position_now": round(c.get("position", 0), 1) if c else None,+ "position_prev": round(p.get("position", 0), 1) if p else None,+ })++ page_losing_queries.sort(key=lambda x: -x["absolute_click_loss"])+ if page_losing_queries:+ page_query_decompositions.append({+ "page": page_url,+ "top_losing_queries": page_losing_queries[:10],+ })+return {"period": f"{start_curr.isoformat()} to {end_curr.isoformat()}","prior_period": f"{start_prev.isoformat()} to {end_prev.isoformat()}","declining_pages": page_changes[:20],- "declining_queries": query_changes[:20]+ "declining_queries": query_changes[:20],+ "page_query_decompositions": page_query_decompositions,}