Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add ahrefs keyword, refdomain and rank tracker tables (#116957)

Tom Owers70d7598c26f8
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 7d618f2f3f13..70630ca942c4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -286,17 +286,17 @@ Note: Parsed collection GET operations from the official OpenAPI spec, including
## Ahrefs — **thin**
-Today (7): `backlinks`, `domain_rating_history`, `metrics_history`, `projects`, `site_audit_health_scores`, `site_audit_issues`, `site_audit_pages`
+Today (14): `backlinks`, `brand_radar_web_pages`, `domain_rating_history`, `domain_rating_top_domains`, `link_intersect_refdomains`, `link_intersect_refpages`, `metrics_history`, `organic_keywords`, `projects`, `rank_tracker_overview`, `referring_domains`, `site_audit_health_scores`, `site_audit_issues`, `site_audit_pages`
Diffed against: <https://docs.ahrefs.com/openapi.json>
- [x] `Projects (GET /v3/management/projects)` — provides project metadata for joining audit project IDs to target websites (high)
-- [ ] `Organic keywords (GET /v3/site-explorer/organic-keywords)` — tracks search positions, traffic, and keyword value for SEO reports (high)
+- [x] `Organic keywords (GET /v3/site-explorer/organic-keywords)` — tracks search positions, traffic, and keyword value for SEO reports (high)
- [x] `Backlinks (GET /v3/site-explorer/all-backlinks)` — supports link acquisition, loss, and link quality analysis (high)
- [x] `Metrics history (GET /v3/site-explorer/metrics-history)` — measures organic traffic and traffic value trends over time (high)
- [x] `Domain Rating history (GET /v3/site-explorer/domain-rating-history)` — tracks the vendor's core authority metric alongside audit health (high)
-- [ ] `Rank Tracker overview (GET /v3/rank-tracker/overview)` — measures tracked keyword rankings and visibility by project (high)
-- [ ] `Referring domains (GET /v3/site-explorer/refdomains)` — measures unique linking domains and their authority (medium)
+- [x] `Rank Tracker overview (GET /v3/rank-tracker/overview)` — measures tracked keyword rankings and visibility by project (high)
+- [x] `Referring domains (GET /v3/site-explorer/refdomains)` — measures unique linking domains and their authority (medium)
- [ ] `Top pages (GET /v3/site-explorer/top-pages)` — identifies pages that drive organic traffic and revenue opportunities (medium)
- [ ] `Brand Radar mentions history (GET /v3/brand-radar/mentions-history)` — measures brand visibility trends across AI search platforms (medium)
- [x] `Link intersect referring domains (POST /v3/competitive-analysis/link-intersect-refdomains)` — finds domains that link to competitors, for link building (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/ahrefs.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/ahrefs.py
index c1366d1e4abe..b6a5c1f8b7c7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/ahrefs.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/ahrefs.py
@@ -201,7 +201,13 @@ def _endpoint_requests(
params["date_from"] = _history_start(
should_use_incremental_field, db_incremental_field_last_value, datetime.now(UTC).date()
)
- yield {"path": settings["path"], "params": params}, {}
+ if settings.get("snapshot_date"):
+ params["date"] = datetime.now(UTC).date().isoformat()
+ # Rank Tracker reports one device per request, so each device is read and its rows are tagged with it.
+ for device in settings.get("devices", []):
+ yield {"path": settings["path"], "params": {**params, "device": device}}, {"device": device}
+ if not settings.get("devices"):
+ yield {"path": settings["path"], "params": params}, {}
def _history_start(should_use_incremental_field: bool, last_value: Any, today: date) -> str:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/canonical_descriptions.py
index 2741ea1f9459..1135f67425db 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/canonical_descriptions.py
@@ -165,4 +165,67 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"rank": "Position of the domain in the ranking. 1 is the highest Domain Rating.",
},
},
+ "organic_keywords": {
+ "description": "Keywords the selected Site Audit project's target ranks for in organic search, highest traffic first.",
+ "docs_url": "https://docs.ahrefs.com/en/api/reference/site-explorer/get-organic-keywords",
+ "columns": {
+ "project_id": "Identifier of the selected Site Audit project.",
+ "keyword": "The keyword the target ranks for.",
+ "keyword_country": "Country of the keyword the target ranks for.",
+ "language": "Language of the search results page.",
+ "best_position": "Top position the target ranks at in organic search results for the keyword.",
+ "best_position_url": "URL of the target that ranks at the top position.",
+ "best_position_kind": "Kind of the top position: organic, paid, or a SERP feature.",
+ "volume": "Estimated number of searches for the keyword over the latest month.",
+ "sum_traffic": "Estimated monthly visitors the target gets from organic search for the keyword.",
+ "cpc": "Average price advertisers pay for an ad click on the keyword, in USD cents.",
+ "serp_features": "SERP features that appear in search results for the keyword.",
+ "is_branded": "Whether the search intent is branded.",
+ "is_commercial": "Whether the search intent is commercial.",
+ "is_informational": "Whether the search intent is informational.",
+ "is_navigational": "Whether the search intent is navigational.",
+ "is_transactional": "Whether the search intent is transactional.",
+ "last_update": "Date Ahrefs last checked the search results for the keyword.",
+ },
+ },
+ "referring_domains": {
+ "description": "Domains with live links to the selected Site Audit project's target, highest Domain Rating first.",
+ "docs_url": "https://docs.ahrefs.com/en/api/reference/site-explorer/get-refdomains",
+ "columns": {
+ "project_id": "Identifier of the selected Site Audit project.",
+ "domain": "Referring domain that has at least one link to the target.",
+ "domain_rating": "Strength of the referring domain's backlink profile, from 0 to 100.",
+ "links_to_target": "Number of backlinks from the referring domain to the target.",
+ "dofollow_links": "Number of links from the referring domain to the target without the nofollow attribute.",
+ "is_root_domain": "Whether the referring domain is a root domain.",
+ "is_spam": "Whether the referring domain is a known spam domain.",
+ "positions_source_domain": "Number of keywords the referring domain ranks for in the top 100 positions.",
+ "first_seen": "Date Ahrefs first found a backlink to the target from the referring domain.",
+ "last_seen": "Date the target lost its last live backlink from the referring domain.",
+ },
+ },
+ "rank_tracker_overview": {
+ "description": "Current rankings of the keywords tracked in the project's Rank Tracker, per device.",
+ "docs_url": "https://docs.ahrefs.com/en/api/reference/rank-tracker/get-overview",
+ "columns": {
+ "project_id": "Identifier of the Ahrefs project.",
+ "device": "Device the rankings are for: desktop or mobile.",
+ "keyword": "Tracked keyword.",
+ "country": "Two-letter code of the country the keyword is tracked in.",
+ "location": "Country, region, or city the keyword is tracked in.",
+ "language": "Search results language the keyword is tracked for.",
+ "position": "Top position of the target in organic search for the keyword.",
+ "url": "Top-ranking URL of the target in organic search.",
+ "best_position_kind": "Kind of the top position: organic, paid, or a SERP feature.",
+ "traffic": "Estimated monthly visits from organic search for the keyword.",
+ "volume": "Estimated average monthly number of searches for the keyword.",
+ "keyword_difficulty": "Estimated difficulty of ranking in the top 10 results for the keyword, from 0 to 100.",
+ "cost_per_click": "Average price advertisers pay for an ad click on the keyword, in USD cents.",
+ "serp_features": "SERP features that appear in search results for the keyword.",
+ "tags": "Tags assigned to the keyword.",
+ "keyword_is_frozen": "Whether the keyword exceeds the plan's tracked keyword limit and is not updated.",
+ "serp_updated": "Date Ahrefs last checked the search results for the keyword.",
+ "created_at": "Date the keyword was added to the project.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/settings.py
index 61ff5460a1ff..ca6ff1a20bb6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/settings.py
@@ -6,6 +6,10 @@ from products.warehouse_sources.backend.types import IncrementalField, Increment
API_BASE_URL = "https://api.ahrefs.com/v3"
MAX_PAGE_ROWS = 100
MAX_BACKLINK_ROWS = 1000
+MAX_KEYWORD_ROWS = 1000
+MAX_REFDOMAIN_ROWS = 1000
+# Matches the largest Rank Tracker plan's tracked keyword allowance; the endpoint is free.
+MAX_RANK_TRACKER_ROWS = 10000
# History rows cost API units and plans cap how far back history goes, so a first sync reads a bounded window.
HISTORY_DAYS = 180
# Link intersect requests accept at most 20 competitors.
@@ -38,6 +42,56 @@ BACKLINK_FIELDS = (
"last_visited",
)
+ORGANIC_KEYWORD_FIELDS = (
+ "keyword",
+ "keyword_country",
+ "language",
+ "best_position",
+ "best_position_url",
+ "best_position_kind",
+ "volume",
+ "sum_traffic",
+ "cpc",
+ "serp_features",
+ "is_branded",
+ "is_commercial",
+ "is_informational",
+ "is_navigational",
+ "is_transactional",
+ "last_update",
+)
+
+REFDOMAIN_FIELDS = (
+ "domain",
+ "domain_rating",
+ "links_to_target",
+ "dofollow_links",
+ "is_root_domain",
+ "is_spam",
+ "positions_source_domain",
+ "first_seen",
+ "last_seen",
+)
+
+RANK_TRACKER_FIELDS = (
+ "keyword",
+ "country",
+ "location",
+ "language",
+ "position",
+ "url",
+ "best_position_kind",
+ "traffic",
+ "volume",
+ "keyword_difficulty",
+ "cost_per_click",
+ "serp_features",
+ "tags",
+ "keyword_is_frozen",
+ "serp_updated",
+ "created_at",
+)
+
class EndpointSettings(TypedDict):
path: str
@@ -54,6 +108,10 @@ class EndpointSettings(TypedDict):
body: NotRequired[dict[str, Any]]
# Reports that need more inputs than the project's target, or more than one request.
kind: NotRequired[Literal["link_intersect", "brand_radar", "top_domains"]]
+ # Snapshot reports require the report date; each sync reads today's state.
+ snapshot_date: NotRequired[bool]
+ # Rank Tracker reports one device per request, so the table reads each and tags rows with it.
+ devices: NotRequired[list[str]]
ENDPOINTS: dict[str, EndpointSettings] = {
@@ -185,6 +243,51 @@ ENDPOINTS: dict[str, EndpointSettings] = {
"project_scoped": False,
"kind": "top_domains",
},
+ "organic_keywords": {
+ "path": "site-explorer/organic-keywords",
+ "data_selector": "keywords",
+ "primary_keys": ["project_id", "keyword", "keyword_country"],
+ "params": {
+ "select": ",".join(ORGANIC_KEYWORD_FIELDS),
+ "limit": MAX_KEYWORD_ROWS,
+ "order_by": "sum_traffic:desc",
+ },
+ "description": (
+ f"Up to {MAX_KEYWORD_ROWS} keywords the selected project's target ranks for in organic search, "
+ "highest traffic first."
+ ),
+ "target_params": ["target", "mode", "protocol"],
+ "snapshot_date": True,
+ },
+ "referring_domains": {
+ "path": "site-explorer/refdomains",
+ "data_selector": "refdomains",
+ "primary_keys": ["project_id", "domain"],
+ "params": {
+ "select": ",".join(REFDOMAIN_FIELDS),
+ "limit": MAX_REFDOMAIN_ROWS,
+ "order_by": "domain_rating:desc",
+ "history": "live",
+ },
+ "description": (
+ f"Up to {MAX_REFDOMAIN_ROWS} domains with live links to the selected project's target, "
+ "highest Domain Rating first."
+ ),
+ "target_params": ["target", "mode", "protocol"],
+ },
+ "rank_tracker_overview": {
+ "path": "rank-tracker/overview",
+ "data_selector": "overviews",
+ "primary_keys": ["project_id", "device", "keyword", "country", "location", "language"],
+ "params": {
+ "select": ",".join(RANK_TRACKER_FIELDS),
+ "limit": MAX_RANK_TRACKER_ROWS,
+ "order_by": "keyword:asc",
+ },
+ "description": "Current desktop and mobile rankings of the keywords tracked in the project's Rank Tracker.",
+ "snapshot_date": True,
+ "devices": ["desktop", "mobile"],
+ },
}
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/source.py
index 33e41619ce89..801f87b593d0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/source.py
@@ -20,7 +20,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.ahrefs.set
HISTORY_DAYS,
INCREMENTAL_FIELDS,
MAX_BACKLINK_ROWS,
+ MAX_KEYWORD_ROWS,
MAX_PAGE_ROWS,
+ MAX_REFDOMAIN_ROWS,
NON_RETRYABLE_ERRORS,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, SimpleSource
@@ -106,13 +108,15 @@ class AhrefsSource(SimpleSource[AhrefsSourceConfig]):
caption=(
"Requires an Ahrefs Lite plan or higher. "
"Create an API key in [Account settings > API keys](https://app.ahrefs.com/account/api-keys). "
- "Syncs use your account's API units. Health scores and projects are free. "
+ "Syncs use your account's API units. Health scores, projects, and Rank Tracker rankings are free. "
"Issues and pages each cost 50 units per request. "
"Metrics history costs 10 units per metric for each month returned. "
f"Each sync imports the latest crawl, with a sample of up to {MAX_PAGE_ROWS} pages. "
"Page data requires verified project ownership. "
- "Domain Rating, metrics history, and backlinks report on the Site Audit project's target. "
- f"History tables start {HISTORY_DAYS} days back, and backlinks import up to {MAX_BACKLINK_ROWS} live links. "
+ "Domain Rating, metrics history, backlinks, organic keywords, and referring domains report on the "
+ "Site Audit project's target. Rank Tracker rankings come from the same project. "
+ f"History tables start {HISTORY_DAYS} days back. Backlinks import up to {MAX_BACKLINK_ROWS} live links, "
+ f"organic keywords up to {MAX_KEYWORD_ROWS} keywords, and referring domains up to {MAX_REFDOMAIN_ROWS} domains. "
"Link intersect tables compare the target with the project's competitors in Ahrefs. "
"Brand Radar web pages come from the Brand Radar reports linked to the project. "
"The top domains table is free and has 1 million rows."
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/tests/test_ahrefs.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/tests/test_ahrefs.py
index 91d51b697df9..231450f7e891 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/tests/test_ahrefs.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/tests/test_ahrefs.py
@@ -11,7 +11,13 @@ from requests import Response, Session
from requests.exceptions import HTTPError
from products.warehouse_sources.backend.temporal.data_imports.sources.ahrefs.ahrefs import validate_credentials
-from products.warehouse_sources.backend.temporal.data_imports.sources.ahrefs.settings import BACKLINK_FIELDS, ENDPOINTS
+from products.warehouse_sources.backend.temporal.data_imports.sources.ahrefs.settings import (
+ BACKLINK_FIELDS,
+ ENDPOINTS,
+ ORGANIC_KEYWORD_FIELDS,
+ RANK_TRACKER_FIELDS,
+ REFDOMAIN_FIELDS,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.ahrefs.source import AhrefsSource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (
RESTClientRetryableError,
@@ -169,6 +175,40 @@ TARGET = {
"protocol": ["https"],
},
),
+ (
+ "organic_keywords",
+ "keywords",
+ [
+ {"keyword": "posthog", "keyword_country": "us", "sum_traffic": 50},
+ {"keyword": "posthog", "keyword_country": "gb", "sum_traffic": 10},
+ ],
+ [
+ {"keyword": "posthog", "keyword_country": "us", "sum_traffic": 50},
+ {"keyword": "posthog", "keyword_country": "gb", "sum_traffic": 10},
+ ],
+ {
+ "select": [",".join(ORGANIC_KEYWORD_FIELDS)],
+ "limit": ["1000"],
+ "order_by": ["sum_traffic:desc"],
+ "date": ["2026-10-13"],
+ "mode": ["domain"],
+ "protocol": ["https"],
+ },
+ ),
+ (
+ "referring_domains",
+ "refdomains",
+ [{"domain": "a.example", "domain_rating": 80}, {"domain": "b.example", "domain_rating": 40}],
+ [{"domain": "a.example", "domain_rating": 80}, {"domain": "b.example", "domain_rating": 40}],
+ {
+ "select": [",".join(REFDOMAIN_FIELDS)],
+ "limit": ["1000"],
+ "order_by": ["domain_rating:desc"],
+ "history": ["live"],
+ "mode": ["domain"],
+ "protocol": ["https"],
+ },
+ ),
],
)
@pytest.mark.parametrize(
@@ -199,10 +239,39 @@ def test_target_report_resolves_site_audit_target(
assert target_request.query["project_id"] == ("123",)
assert report_request.path == f"/v3/{ENDPOINTS[table]['path']}"
query = {name: list(values) for name, values in report_request.query.items()}
- history_params = {"date_from": [expected_date_from]} if table != "backlinks" else {}
+ history_params = {"date_from": [expected_date_from]} if ENDPOINTS[table].get("history") else {}
assert query == {"output": ["json"], "target": ["https://example.com/"], **expected_params, **history_params}
+@time_machine.travel("2026-10-13T12:00:00Z", tick=False)
+def test_rank_tracker_reads_each_device() -> None:
+ row = {"keyword": "posthog", "country": "us", "location": "United States", "language": "en", "position": 1}
+ result = SourceDriver(AhrefsSource(), AhrefsSourceConfig(api_key="test-key", project_id="00123")).run(
+ "rank_tracker_overview",
+ [
+ ScriptedResponse(json={"overviews": [row]}),
+ ScriptedResponse(json={"overviews": [{**row, "position": 3}]}),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.rows == [
+ {**row, "project_id": "123", "device": "desktop"},
+ {**row, "position": 3, "project_id": "123", "device": "mobile"},
+ ]
+ for request, device in zip(result.requests, ["desktop", "mobile"], strict=True):
+ assert request.path == "/v3/rank-tracker/overview"
+ assert {name: list(values) for name, values in request.query.items()} == {
+ "output": ["json"],
+ "select": [",".join(RANK_TRACKER_FIELDS)],
+ "limit": ["10000"],
+ "order_by": ["keyword:asc"],
+ "project_id": ["123"],
+ "date": ["2026-10-13"],
+ "device": [device],
+ }
+
+
@pytest.mark.parametrize("healthscores", [[], [{"project_id": "123", "target_url": None}]])
def test_target_report_without_site_audit_target_fails_permanently(healthscores: list[dict[str, object]]) -> None:
source = AhrefsSource()