:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add ahrefs keyword, refdomain and rank tracker tables (#116957)
Tom Owers70d7598c26f8
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 7d618f2f3f13..70630ca942c4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -286,17 +286,17 @@ Note: Parsed collection GET operations from the official OpenAPI spec, including## Ahrefs — **thin**-Today (7): `backlinks`, `domain_rating_history`, `metrics_history`, `projects`, `site_audit_health_scores`, `site_audit_issues`, `site_audit_pages`+Today (14): `backlinks`, `brand_radar_web_pages`, `domain_rating_history`, `domain_rating_top_domains`, `link_intersect_refdomains`, `link_intersect_refpages`, `metrics_history`, `organic_keywords`, `projects`, `rank_tracker_overview`, `referring_domains`, `site_audit_health_scores`, `site_audit_issues`, `site_audit_pages`Diffed against: <https://docs.ahrefs.com/openapi.json>- [x] `Projects (GET /v3/management/projects)` — provides project metadata for joining audit project IDs to target websites (high)-- [ ] `Organic keywords (GET /v3/site-explorer/organic-keywords)` — tracks search positions, traffic, and keyword value for SEO reports (high)+- [x] `Organic keywords (GET /v3/site-explorer/organic-keywords)` — tracks search positions, traffic, and keyword value for SEO reports (high)- [x] `Backlinks (GET /v3/site-explorer/all-backlinks)` — supports link acquisition, loss, and link quality analysis (high)- [x] `Metrics history (GET /v3/site-explorer/metrics-history)` — measures organic traffic and traffic value trends over time (high)- [x] `Domain Rating history (GET /v3/site-explorer/domain-rating-history)` — tracks the vendor's core authority metric alongside audit health (high)-- [ ] `Rank Tracker overview (GET /v3/rank-tracker/overview)` — measures tracked keyword rankings and visibility by project (high)-- [ ] `Referring domains (GET /v3/site-explorer/refdomains)` — measures unique linking domains and their authority (medium)+- [x] `Rank Tracker overview (GET /v3/rank-tracker/overview)` — measures tracked keyword rankings and visibility by project (high)+- [x] `Referring domains (GET /v3/site-explorer/refdomains)` — measures unique linking domains and their authority (medium)- [ ] `Top pages (GET /v3/site-explorer/top-pages)` — identifies pages that drive organic traffic and revenue opportunities (medium)- [ ] `Brand Radar mentions history (GET /v3/brand-radar/mentions-history)` — measures brand visibility trends across AI search platforms (medium)- [x] `Link intersect referring domains (POST /v3/competitive-analysis/link-intersect-refdomains)` — finds domains that link to competitors, for link building (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/ahrefs.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/ahrefs.pyindex c1366d1e4abe..b6a5c1f8b7c7 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/ahrefs.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/ahrefs.py@@ -201,7 +201,13 @@ def _endpoint_requests(params["date_from"] = _history_start(should_use_incremental_field, db_incremental_field_last_value, datetime.now(UTC).date())- yield {"path": settings["path"], "params": params}, {}+ if settings.get("snapshot_date"):+ params["date"] = datetime.now(UTC).date().isoformat()+ # Rank Tracker reports one device per request, so each device is read and its rows are tagged with it.+ for device in settings.get("devices", []):+ yield {"path": settings["path"], "params": {**params, "device": device}}, {"device": device}+ if not settings.get("devices"):+ yield {"path": settings["path"], "params": params}, {}def _history_start(should_use_incremental_field: bool, last_value: Any, today: date) -> str:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/canonical_descriptions.pyindex 2741ea1f9459..1135f67425db 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/canonical_descriptions.py@@ -165,4 +165,67 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"rank": "Position of the domain in the ranking. 1 is the highest Domain Rating.",},},+ "organic_keywords": {+ "description": "Keywords the selected Site Audit project's target ranks for in organic search, highest traffic first.",+ "docs_url": "https://docs.ahrefs.com/en/api/reference/site-explorer/get-organic-keywords",+ "columns": {+ "project_id": "Identifier of the selected Site Audit project.",+ "keyword": "The keyword the target ranks for.",+ "keyword_country": "Country of the keyword the target ranks for.",+ "language": "Language of the search results page.",+ "best_position": "Top position the target ranks at in organic search results for the keyword.",+ "best_position_url": "URL of the target that ranks at the top position.",+ "best_position_kind": "Kind of the top position: organic, paid, or a SERP feature.",+ "volume": "Estimated number of searches for the keyword over the latest month.",+ "sum_traffic": "Estimated monthly visitors the target gets from organic search for the keyword.",+ "cpc": "Average price advertisers pay for an ad click on the keyword, in USD cents.",+ "serp_features": "SERP features that appear in search results for the keyword.",+ "is_branded": "Whether the search intent is branded.",+ "is_commercial": "Whether the search intent is commercial.",+ "is_informational": "Whether the search intent is informational.",+ "is_navigational": "Whether the search intent is navigational.",+ "is_transactional": "Whether the search intent is transactional.",+ "last_update": "Date Ahrefs last checked the search results for the keyword.",+ },+ },+ "referring_domains": {+ "description": "Domains with live links to the selected Site Audit project's target, highest Domain Rating first.",+ "docs_url": "https://docs.ahrefs.com/en/api/reference/site-explorer/get-refdomains",+ "columns": {+ "project_id": "Identifier of the selected Site Audit project.",+ "domain": "Referring domain that has at least one link to the target.",+ "domain_rating": "Strength of the referring domain's backlink profile, from 0 to 100.",+ "links_to_target": "Number of backlinks from the referring domain to the target.",+ "dofollow_links": "Number of links from the referring domain to the target without the nofollow attribute.",+ "is_root_domain": "Whether the referring domain is a root domain.",+ "is_spam": "Whether the referring domain is a known spam domain.",+ "positions_source_domain": "Number of keywords the referring domain ranks for in the top 100 positions.",+ "first_seen": "Date Ahrefs first found a backlink to the target from the referring domain.",+ "last_seen": "Date the target lost its last live backlink from the referring domain.",+ },+ },+ "rank_tracker_overview": {+ "description": "Current rankings of the keywords tracked in the project's Rank Tracker, per device.",+ "docs_url": "https://docs.ahrefs.com/en/api/reference/rank-tracker/get-overview",+ "columns": {+ "project_id": "Identifier of the Ahrefs project.",+ "device": "Device the rankings are for: desktop or mobile.",+ "keyword": "Tracked keyword.",+ "country": "Two-letter code of the country the keyword is tracked in.",+ "location": "Country, region, or city the keyword is tracked in.",+ "language": "Search results language the keyword is tracked for.",+ "position": "Top position of the target in organic search for the keyword.",+ "url": "Top-ranking URL of the target in organic search.",+ "best_position_kind": "Kind of the top position: organic, paid, or a SERP feature.",+ "traffic": "Estimated monthly visits from organic search for the keyword.",+ "volume": "Estimated average monthly number of searches for the keyword.",+ "keyword_difficulty": "Estimated difficulty of ranking in the top 10 results for the keyword, from 0 to 100.",+ "cost_per_click": "Average price advertisers pay for an ad click on the keyword, in USD cents.",+ "serp_features": "SERP features that appear in search results for the keyword.",+ "tags": "Tags assigned to the keyword.",+ "keyword_is_frozen": "Whether the keyword exceeds the plan's tracked keyword limit and is not updated.",+ "serp_updated": "Date Ahrefs last checked the search results for the keyword.",+ "created_at": "Date the keyword was added to the project.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/settings.pyindex 61ff5460a1ff..ca6ff1a20bb6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/settings.py@@ -6,6 +6,10 @@ from products.warehouse_sources.backend.types import IncrementalField, IncrementAPI_BASE_URL = "https://api.ahrefs.com/v3"MAX_PAGE_ROWS = 100MAX_BACKLINK_ROWS = 1000+MAX_KEYWORD_ROWS = 1000+MAX_REFDOMAIN_ROWS = 1000+# Matches the largest Rank Tracker plan's tracked keyword allowance; the endpoint is free.+MAX_RANK_TRACKER_ROWS = 10000# History rows cost API units and plans cap how far back history goes, so a first sync reads a bounded window.HISTORY_DAYS = 180# Link intersect requests accept at most 20 competitors.@@ -38,6 +42,56 @@ BACKLINK_FIELDS = ("last_visited",)+ORGANIC_KEYWORD_FIELDS = (+ "keyword",+ "keyword_country",+ "language",+ "best_position",+ "best_position_url",+ "best_position_kind",+ "volume",+ "sum_traffic",+ "cpc",+ "serp_features",+ "is_branded",+ "is_commercial",+ "is_informational",+ "is_navigational",+ "is_transactional",+ "last_update",+)++REFDOMAIN_FIELDS = (+ "domain",+ "domain_rating",+ "links_to_target",+ "dofollow_links",+ "is_root_domain",+ "is_spam",+ "positions_source_domain",+ "first_seen",+ "last_seen",+)++RANK_TRACKER_FIELDS = (+ "keyword",+ "country",+ "location",+ "language",+ "position",+ "url",+ "best_position_kind",+ "traffic",+ "volume",+ "keyword_difficulty",+ "cost_per_click",+ "serp_features",+ "tags",+ "keyword_is_frozen",+ "serp_updated",+ "created_at",+)+class EndpointSettings(TypedDict):path: str@@ -54,6 +108,10 @@ class EndpointSettings(TypedDict):body: NotRequired[dict[str, Any]]# Reports that need more inputs than the project's target, or more than one request.kind: NotRequired[Literal["link_intersect", "brand_radar", "top_domains"]]+ # Snapshot reports require the report date; each sync reads today's state.+ snapshot_date: NotRequired[bool]+ # Rank Tracker reports one device per request, so the table reads each and tags rows with it.+ devices: NotRequired[list[str]]ENDPOINTS: dict[str, EndpointSettings] = {@@ -185,6 +243,51 @@ ENDPOINTS: dict[str, EndpointSettings] = {"project_scoped": False,"kind": "top_domains",},+ "organic_keywords": {+ "path": "site-explorer/organic-keywords",+ "data_selector": "keywords",+ "primary_keys": ["project_id", "keyword", "keyword_country"],+ "params": {+ "select": ",".join(ORGANIC_KEYWORD_FIELDS),+ "limit": MAX_KEYWORD_ROWS,+ "order_by": "sum_traffic:desc",+ },+ "description": (+ f"Up to {MAX_KEYWORD_ROWS} keywords the selected project's target ranks for in organic search, "+ "highest traffic first."+ ),+ "target_params": ["target", "mode", "protocol"],+ "snapshot_date": True,+ },+ "referring_domains": {+ "path": "site-explorer/refdomains",+ "data_selector": "refdomains",+ "primary_keys": ["project_id", "domain"],+ "params": {+ "select": ",".join(REFDOMAIN_FIELDS),+ "limit": MAX_REFDOMAIN_ROWS,+ "order_by": "domain_rating:desc",+ "history": "live",+ },+ "description": (+ f"Up to {MAX_REFDOMAIN_ROWS} domains with live links to the selected project's target, "+ "highest Domain Rating first."+ ),+ "target_params": ["target", "mode", "protocol"],+ },+ "rank_tracker_overview": {+ "path": "rank-tracker/overview",+ "data_selector": "overviews",+ "primary_keys": ["project_id", "device", "keyword", "country", "location", "language"],+ "params": {+ "select": ",".join(RANK_TRACKER_FIELDS),+ "limit": MAX_RANK_TRACKER_ROWS,+ "order_by": "keyword:asc",+ },+ "description": "Current desktop and mobile rankings of the keywords tracked in the project's Rank Tracker.",+ "snapshot_date": True,+ "devices": ["desktop", "mobile"],+ },}INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/source.pyindex 33e41619ce89..801f87b593d0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/source.py@@ -20,7 +20,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.ahrefs.setHISTORY_DAYS,INCREMENTAL_FIELDS,MAX_BACKLINK_ROWS,+ MAX_KEYWORD_ROWS,MAX_PAGE_ROWS,+ MAX_REFDOMAIN_ROWS,NON_RETRYABLE_ERRORS,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, SimpleSource@@ -106,13 +108,15 @@ class AhrefsSource(SimpleSource[AhrefsSourceConfig]):caption=("Requires an Ahrefs Lite plan or higher. ""Create an API key in [Account settings > API keys](https://app.ahrefs.com/account/api-keys). "- "Syncs use your account's API units. Health scores and projects are free. "+ "Syncs use your account's API units. Health scores, projects, and Rank Tracker rankings are free. ""Issues and pages each cost 50 units per request. ""Metrics history costs 10 units per metric for each month returned. "f"Each sync imports the latest crawl, with a sample of up to {MAX_PAGE_ROWS} pages. ""Page data requires verified project ownership. "- "Domain Rating, metrics history, and backlinks report on the Site Audit project's target. "- f"History tables start {HISTORY_DAYS} days back, and backlinks import up to {MAX_BACKLINK_ROWS} live links. "+ "Domain Rating, metrics history, backlinks, organic keywords, and referring domains report on the "+ "Site Audit project's target. Rank Tracker rankings come from the same project. "+ f"History tables start {HISTORY_DAYS} days back. Backlinks import up to {MAX_BACKLINK_ROWS} live links, "+ f"organic keywords up to {MAX_KEYWORD_ROWS} keywords, and referring domains up to {MAX_REFDOMAIN_ROWS} domains. ""Link intersect tables compare the target with the project's competitors in Ahrefs. ""Brand Radar web pages come from the Brand Radar reports linked to the project. ""The top domains table is free and has 1 million rows."diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/tests/test_ahrefs.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/tests/test_ahrefs.pyindex 91d51b697df9..231450f7e891 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/tests/test_ahrefs.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/tests/test_ahrefs.py@@ -11,7 +11,13 @@ from requests import Response, Sessionfrom requests.exceptions import HTTPErrorfrom products.warehouse_sources.backend.temporal.data_imports.sources.ahrefs.ahrefs import validate_credentials-from products.warehouse_sources.backend.temporal.data_imports.sources.ahrefs.settings import BACKLINK_FIELDS, ENDPOINTS+from products.warehouse_sources.backend.temporal.data_imports.sources.ahrefs.settings import (+ BACKLINK_FIELDS,+ ENDPOINTS,+ ORGANIC_KEYWORD_FIELDS,+ RANK_TRACKER_FIELDS,+ REFDOMAIN_FIELDS,+)from products.warehouse_sources.backend.temporal.data_imports.sources.ahrefs.source import AhrefsSourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (RESTClientRetryableError,@@ -169,6 +175,40 @@ TARGET = {"protocol": ["https"],},),+ (+ "organic_keywords",+ "keywords",+ [+ {"keyword": "posthog", "keyword_country": "us", "sum_traffic": 50},+ {"keyword": "posthog", "keyword_country": "gb", "sum_traffic": 10},+ ],+ [+ {"keyword": "posthog", "keyword_country": "us", "sum_traffic": 50},+ {"keyword": "posthog", "keyword_country": "gb", "sum_traffic": 10},+ ],+ {+ "select": [",".join(ORGANIC_KEYWORD_FIELDS)],+ "limit": ["1000"],+ "order_by": ["sum_traffic:desc"],+ "date": ["2026-10-13"],+ "mode": ["domain"],+ "protocol": ["https"],+ },+ ),+ (+ "referring_domains",+ "refdomains",+ [{"domain": "a.example", "domain_rating": 80}, {"domain": "b.example", "domain_rating": 40}],+ [{"domain": "a.example", "domain_rating": 80}, {"domain": "b.example", "domain_rating": 40}],+ {+ "select": [",".join(REFDOMAIN_FIELDS)],+ "limit": ["1000"],+ "order_by": ["domain_rating:desc"],+ "history": ["live"],+ "mode": ["domain"],+ "protocol": ["https"],+ },+ ),],)@pytest.mark.parametrize(@@ -199,10 +239,39 @@ def test_target_report_resolves_site_audit_target(assert target_request.query["project_id"] == ("123",)assert report_request.path == f"/v3/{ENDPOINTS[table]['path']}"query = {name: list(values) for name, values in report_request.query.items()}- history_params = {"date_from": [expected_date_from]} if table != "backlinks" else {}+ history_params = {"date_from": [expected_date_from]} if ENDPOINTS[table].get("history") else {}assert query == {"output": ["json"], "target": ["https://example.com/"], **expected_params, **history_params}+@time_machine.travel("2026-10-13T12:00:00Z", tick=False)+def test_rank_tracker_reads_each_device() -> None:+ row = {"keyword": "posthog", "country": "us", "location": "United States", "language": "en", "position": 1}+ result = SourceDriver(AhrefsSource(), AhrefsSourceConfig(api_key="test-key", project_id="00123")).run(+ "rank_tracker_overview",+ [+ ScriptedResponse(json={"overviews": [row]}),+ ScriptedResponse(json={"overviews": [{**row, "position": 3}]}),+ ],+ )++ assert result.raised is None+ assert result.rows == [+ {**row, "project_id": "123", "device": "desktop"},+ {**row, "position": 3, "project_id": "123", "device": "mobile"},+ ]+ for request, device in zip(result.requests, ["desktop", "mobile"], strict=True):+ assert request.path == "/v3/rank-tracker/overview"+ assert {name: list(values) for name, values in request.query.items()} == {+ "output": ["json"],+ "select": [",".join(RANK_TRACKER_FIELDS)],+ "limit": ["10000"],+ "order_by": ["keyword:asc"],+ "project_id": ["123"],+ "date": ["2026-10-13"],+ "device": [device],+ }++@pytest.mark.parametrize("healthscores", [[], [{"project_id": "123", "target_url": None}]])def test_target_report_without_site_audit_target_fails_permanently(healthscores: list[dict[str, object]]) -> None:source = AhrefsSource()