Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add octolens search and usage tables (#123161)

Tom Owersc2a04e567e00
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index fbc6ec96ea95..f5fd51a4e1f5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -8542,8 +8542,8 @@ Today (5): `feeds`, `keywords`, `mentions`, `notifications`, `org_members`
Diffed against: <https://octolens.com/openapi.json>
-- [ ] `List past on-demand searches and search results (GET /api/v2/search; GET /api/v2/search/{searchId})` — include on-demand search activity and scored results that never appear in the synced mentions feed (high)
-- [ ] `Get usage stats (GET /api/v2/org/usage)` — track billing-cycle consumption, remaining allowances, plan limits, and usage-based billing state (medium)
+- [x] `List past on-demand searches and search results (GET /api/v2/search; GET /api/v2/search/{searchId})` — include on-demand search activity and scored results that never appear in the synced mentions feed (high)
+- [x] `Get usage stats (GET /api/v2/org/usage)` — track billing-cycle consumption, remaining allowances, plan limits, and usage-based billing state (medium)
- [ ] `List agency workspaces and members (GET /api/v2/agency/workspaces; GET /api/v2/agency/workspaces/{orgId}/members)` — compare client workspace usage and memberships for agency reporting (medium)
Note: Parsed every operation in the official OpenAPI 3.1 document, including nested agency and search routes. Search results are separate from monitored mentions and need requests for each search ID. Agency coverage requires agency permissions and workspace selection; the current source imports one authenticated organization. Excluded authentication, configuration, and analytic summaries that can be calculated from synced mentions, including sentiment and volume.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/octolens/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/octolens/canonical_descriptions.py
index fd976b613af3..b679fee60f12 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/octolens/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/octolens/canonical_descriptions.py
@@ -94,4 +94,43 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"createdAt": "When the member joined the workspace.",
},
},
+ "usage": {
+ "description": "Usage counters for the current billing cycle and the plan limits of the authenticated Octolens workspace, as a single row.",
+ "docs_url": "https://octolens.com/docs/api/v2/overview",
+ "columns": {
+ "plan": "Current subscription plan, for example `Pro_v3`. Null if there is no active plan.",
+ "mentions": "Mentions consumed in the current billing cycle (`count`), the cycle allowance (`limit`) and when the counter resets (`resetAt`).",
+ "keywords": "Live keyword count (`count`) and the keyword allowance (`limit`).",
+ "searches": "Lifetime on-demand search allowance (`used`, `limit`, `remaining`). Only present on the Agents plan.",
+ "aiCredits": "AI credits the hosted agents used this calendar month, with the allowance, the remainder and the reset time.",
+ "flex": "Usage-based pricing state (budget, use and reset time). Only present when flex pricing is on.",
+ },
+ },
+ "searches": {
+ "description": "Past on-demand searches run in the workspace. On-demand results are separate from the monitored mentions feed.",
+ "docs_url": "https://octolens.com/docs/api/v2/overview",
+ "columns": {
+ "searchId": "Search id.",
+ "status": "Search state: `running`, `completed`, `failed` or `quota_exhausted`.",
+ "query": "Search query text.",
+ "timeWindow": "Look-back window of the search: `1d`, `7d` or `30d`.",
+ "sources": "Platforms the search covered.",
+ "createdAt": "When the search started.",
+ "completedAt": "When the search finished. Null while it runs.",
+ "mentionsConsumed": "Mentions the search consumed from the workspace allowance.",
+ },
+ },
+ "search_results": {
+ "description": "Scored posts that each on-demand search returned, one row per search and post.",
+ "docs_url": "https://octolens.com/docs/api/v2/overview",
+ "columns": {
+ "searchId": "Id of the search that returned this post.",
+ "id": "Internal numeric post id.",
+ "sourceId": "Stable composite key for the post across the collection pipeline, for example `reddit_t3_1abc234`.",
+ "url": "Canonical URL of the original post.",
+ "timestamp": "When the post was published, as `YYYY-MM-DD HH:MM:SS.SSS` in UTC with no offset suffix.",
+ "relevance": "AI relevance verdict, either `relevant` or `not_relevant`.",
+ "alreadyInWorkspace": "Whether the post is already in the workspace's monitored mentions.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/octolens/octolens.py b/products/warehouse_sources/backend/temporal/data_imports/sources/octolens/octolens.py
index 75ad3a293880..e2fc9a12baa6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/octolens/octolens.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/octolens/octolens.py
@@ -1,4 +1,5 @@
import dataclasses
+from collections.abc import Iterable
from typing import Any, Optional
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
@@ -6,12 +7,16 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
RESTAPIConfig,
rest_api_resource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ build_dependent_resource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
BasePaginator,
JSONResponseCursorPaginator,
SinglePagePaginator,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
+ ClientConfig,
Endpoint,
EndpointResource,
)
@@ -19,21 +24,22 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.octolens.settings import (
OCTOLENS_ENDPOINTS,
+ PAGE_SIZE,
OctolensEndpointConfig,
)
OCTOLENS_BASE_URL = "https://app.octolens.com"
-# Documented maximum page size on the mentions feed (1-100, default 20).
-PAGE_SIZE = 100
# Cheap credential introspection endpoint: returns the workspace the key is bound to plus its
# effective scopes. Every read-capable key can call it, so it validates the key without needing
# access to any particular table.
AUTH_PROBE_PATH = "/auth"
[email protected](frozen=True)
class OctolensResumeConfig:
- cursor: str
+ cursor: str = ""
+ # Fan-out progress (completed parents, current parent, child page) for the search_results table.
+ fanout_state: Optional[dict[str, Any]] = None
def _api_path(api_version: str, path: str) -> str:
@@ -55,9 +61,11 @@ def _mentions_body() -> dict[str, Any]:
def _paginator(config: OctolensEndpointConfig) -> BasePaginator:
if not config.paginated:
return SinglePagePaginator()
- # The cursor rides in the POST body alongside the filters, not the query string.
+ # On the POST mentions feed the cursor rides in the body alongside the filters, not the query string.
return JSONResponseCursorPaginator(
- cursor_path="pagination.nextCursor", cursor_param="cursor", param_location="json"
+ cursor_path="pagination.nextCursor",
+ cursor_param="cursor",
+ param_location="json" if config.method == "post" else "query",
)
@@ -70,6 +78,8 @@ def _resource(config: OctolensEndpointConfig, api_version: str) -> EndpointResou
}
if config.method == "post":
endpoint["json"] = _mentions_body()
+ elif config.paginated:
+ endpoint["params"] = {"limit": PAGE_SIZE}
return {
"name": config.name,
@@ -81,6 +91,74 @@ def _resource(config: OctolensEndpointConfig, api_version: str) -> EndpointResou
}
+def _client_config(api_key: str) -> ClientConfig:
+ return {
+ "base_url": OCTOLENS_BASE_URL,
+ "headers": {"Content-Type": "application/json", "Accept": "application/json"},
+ "auth": {"type": "bearer", "token": api_key},
+ }
+
+
+def _dependent_resource(
+ config: OctolensEndpointConfig,
+ api_key: str,
+ api_version: str,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[OctolensResumeConfig],
+) -> Iterable[Any]:
+ assert config.fanout is not None
+ parent_config = OCTOLENS_ENDPOINTS[config.fanout.parent_name]
+ versioned_endpoints = {
+ name: dataclasses.replace(endpoint, path=_api_path(api_version, endpoint.path))
+ for name, endpoint in OCTOLENS_ENDPOINTS.items()
+ }
+
+ initial_paginator_state: Optional[dict[str, Any]] = None
+ if resumable_source_manager.can_resume():
+ resume = resumable_source_manager.load_state()
+ if resume is not None:
+ initial_paginator_state = resume.fanout_state
+
+ def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ if state:
+ resumable_source_manager.save_state(OctolensResumeConfig(fanout_state=dict(state)))
+
+ return build_dependent_resource(
+ endpoint_configs=versioned_endpoints,
+ child_endpoint=config.name,
+ fanout=config.fanout,
+ client_config=_client_config(api_key),
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ # Only the searches parent takes a page size, so it rides in its `parent_params`.
+ page_size_param=None,
+ parent_endpoint_extra={
+ "paginator": _paginator(parent_config),
+ "data_selector": parent_config.data_selector,
+ },
+ child_endpoint_extra={
+ "paginator": SinglePagePaginator(),
+ "data_selector": config.data_selector,
+ },
+ resume_hook=save_checkpoint,
+ initial_paginator_state=initial_paginator_state,
+ )
+
+
+def _source_response(endpoint: str, config: OctolensEndpointConfig, resource: Iterable[Any]) -> SourceResponse:
+ return SourceResponse(
+ name=endpoint,
+ items=lambda: resource,
+ primary_keys=config.primary_key,
+ partition_mode="datetime" if config.partition_key else None,
+ partition_format="month" if config.partition_key else None,
+ partition_keys=[config.partition_key] if config.partition_key else None,
+ )
+
+
def octolens_source(
api_key: str,
endpoint: str,
@@ -91,12 +169,12 @@ def octolens_source(
) -> SourceResponse:
config = OCTOLENS_ENDPOINTS[endpoint]
+ if config.fanout is not None:
+ resource = _dependent_resource(config, api_key, api_version, team_id, job_id, resumable_source_manager)
+ return _source_response(endpoint, config, resource)
+
rest_config: RESTAPIConfig = {
- "client": {
- "base_url": OCTOLENS_BASE_URL,
- "headers": {"Content-Type": "application/json", "Accept": "application/json"},
- "auth": {"type": "bearer", "token": api_key},
- },
+ "client": _client_config(api_key),
"resource_defaults": {},
"resources": [_resource(config, api_version)],
}
@@ -104,7 +182,7 @@ def octolens_source(
initial_paginator_state: Optional[dict[str, Any]] = None
if config.paginated and resumable_source_manager.can_resume():
resume = resumable_source_manager.load_state()
- if resume is not None:
+ if resume is not None and resume.cursor:
initial_paginator_state = {"cursor": resume.cursor}
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
@@ -122,14 +200,7 @@ def octolens_source(
initial_paginator_state=initial_paginator_state,
)
- return SourceResponse(
- name=endpoint,
- items=lambda: resource,
- primary_keys=config.primary_key,
- partition_mode="datetime" if config.partition_key else None,
- partition_format="month" if config.partition_key else None,
- partition_keys=[config.partition_key] if config.partition_key else None,
- )
+ return _source_response(endpoint, config, resource)
def check_access(api_key: str, api_version: str) -> tuple[int, Optional[str]]:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/octolens/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/octolens/settings.py
index f046c68f21c4..7b968c4c55db 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/octolens/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/octolens/settings.py
@@ -1,23 +1,35 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
from typing import Literal, Optional
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+)
from products.warehouse_sources.backend.types import IncrementalField
+# Documented maximum page size on the cursor-paginated feeds (1-100, default 20).
+PAGE_SIZE = 100
-@dataclass
+
+@dataclass(frozen=True)
class OctolensEndpointConfig:
name: str
"""Table name we expose to the user (snake_case)."""
path: str
"""Path under `/api/<version>`, e.g. `/mentions`."""
- primary_key: list[str]
+ primary_key: Optional[list[str]]
+ """`None` for single-object endpoints, which have no identifier to key on."""
method: Literal["get", "post"] = "get"
data_selector: str = "data"
paginated: bool = False
- """True only for the cursor-paginated mentions feed. The dimension endpoints document no
- pagination parameters and return the full collection in one response."""
+ """True only for the cursor-paginated feeds (mentions, searches). The other list endpoints document
+ no pagination parameters and return the full collection in one response."""
partition_key: Optional[str] = None
"""A STABLE creation-time field to partition on. `None` disables partitioning."""
+ fanout: Optional[DependentEndpointConfig] = None
+ """Set for child tables fetched once per row of a parent table."""
+ page_size: int = PAGE_SIZE
+ incremental_fields: list[IncrementalField] = field(default_factory=list)
+ default_incremental_field: Optional[str] = None
OCTOLENS_ENDPOINTS: dict[str, OctolensEndpointConfig] = {
@@ -33,6 +45,31 @@ OCTOLENS_ENDPOINTS: dict[str, OctolensEndpointConfig] = {
"feeds": OctolensEndpointConfig(name="feeds", path="/feeds", primary_key=["id"]),
"notifications": OctolensEndpointConfig(name="notifications", path="/notifications", primary_key=["id"]),
"org_members": OctolensEndpointConfig(name="org_members", path="/org/members", primary_key=["id"]),
+ "usage": OctolensEndpointConfig(name="usage", path="/org/usage", primary_key=None, data_selector="$"),
+ "searches": OctolensEndpointConfig(
+ name="searches",
+ path="/search",
+ primary_key=["searchId"],
+ paginated=True,
+ partition_key="createdAt",
+ ),
+ # The list endpoint carries no results, so each search is fetched on its own. A `sourceId` can
+ # appear in several searches, so the key includes the search.
+ "search_results": OctolensEndpointConfig(
+ name="search_results",
+ path="/search/{searchId}",
+ primary_key=["searchId", "sourceId"],
+ data_selector="mentions",
+ partition_key="timestamp",
+ fanout=DependentEndpointConfig(
+ parent_name="searches",
+ resolve_param="searchId",
+ resolve_field="searchId",
+ include_from_parent=["searchId"],
+ parent_field_renames={"searchId": "searchId"},
+ parent_params={"limit": PAGE_SIZE},
+ ),
+ ),
}
ENDPOINTS = tuple(OCTOLENS_ENDPOINTS.keys())
@@ -48,6 +85,6 @@ ENDPOINTS = tuple(OCTOLENS_ENDPOINTS.keys())
# resynced in full until Octolens exposes an update-aware cursor (an `updatedAt` field or a
# collected-at filter) we can safely watermark on.
#
-# The other four endpoints expose no date filter and no pagination, so they full-refresh in a single
-# request each regardless.
+# The searches feed pages by cursor but has no date filter, and the other endpoints expose neither, so
+# they full-refresh regardless.
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {name: [] for name in OCTOLENS_ENDPOINTS}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/octolens/tests/test_octolens.py b/products/warehouse_sources/backend/temporal/data_imports/sources/octolens/tests/test_octolens.py
index 6aade5ba2177..7eb744e12f2d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/octolens/tests/test_octolens.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/octolens/tests/test_octolens.py
@@ -117,6 +117,86 @@ class TestDimensionEndpoints:
assert result.requests[0].body is None
+class TestSearches:
+ def test_cursor_rides_in_the_query_string_until_exhausted(self) -> None:
+ result = _driver().run(
+ "searches",
+ [
+ _mentions_page([{"searchId": "s1"}], next_cursor="c1"),
+ _mentions_page([{"searchId": "s2"}]),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.rows == [{"searchId": "s1"}, {"searchId": "s2"}]
+ assert [request.method for request in result.requests] == ["GET", "GET"]
+ assert [request.path for request in result.requests] == ["/api/v2/search", "/api/v2/search"]
+ assert result.requests[0].query == {"limit": (str(PAGE_SIZE),)}
+ assert result.requests[1].query == {"limit": (str(PAGE_SIZE),), "cursor": ("c1",)}
+ assert result.requests[1].body is None
+
+
+class TestFanOut:
+ def test_search_results_fetch_each_search_and_carry_its_id(self) -> None:
+ result = _driver().run(
+ "search_results",
+ [
+ _mentions_page([{"searchId": "s1"}], next_cursor="c1"),
+ ScriptedResponse(json={"searchId": "s1", "mentions": [{"sourceId": "reddit_1"}]}),
+ _mentions_page([{"searchId": "s2"}]),
+ ScriptedResponse(json={"searchId": "s2", "mentions": [{"sourceId": "reddit_1"}]}),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.rows == [
+ {"sourceId": "reddit_1", "searchId": "s1"},
+ {"sourceId": "reddit_1", "searchId": "s2"},
+ ]
+ assert [request.path for request in result.requests] == [
+ "/api/v2/search",
+ "/api/v2/search/s1",
+ "/api/v2/search",
+ "/api/v2/search/s2",
+ ]
+ assert result.requests[2].query["cursor"] == ("c1",)
+ # The single-search endpoint takes no page size.
+ assert result.requests[1].query == {}
+
+ def test_search_results_resume_skips_completed_searches(self) -> None:
+ searches = _mentions_page([{"searchId": "s1"}, {"searchId": "s2"}])
+ first = _driver().run(
+ "search_results",
+ [
+ searches,
+ ScriptedResponse(json={"mentions": [{"sourceId": "reddit_1"}]}),
+ ScriptedResponse(json={"mentions": [{"sourceId": "reddit_2"}]}),
+ ],
+ )
+ after_first_search = first.saved_states[0]
+ assert after_first_search.fanout_state is not None
+
+ resumed = _driver().run(
+ "search_results",
+ [searches, ScriptedResponse(json={"mentions": [{"sourceId": "reddit_2"}]})],
+ resume_state=after_first_search,
+ )
+
+ assert resumed.raised is None
+ assert resumed.rows == [{"sourceId": "reddit_2", "searchId": "s2"}]
+ assert [request.path for request in resumed.requests] == ["/api/v2/search", "/api/v2/search/s2"]
+
+
+class TestUsage:
+ def test_the_response_object_is_one_row(self) -> None:
+ usage = {"plan": "Pro_v3", "mentions": {"count": 5, "limit": 100, "resetAt": None}}
+ result = _driver().run("usage", [ScriptedResponse(json=usage)])
+
+ assert result.raised is None
+ assert result.rows == [usage]
+ assert result.requests[0].path == "/api/v2/org/usage"
+
+
class TestHttpErrors:
@pytest.mark.parametrize("status_code", [401, 403])
def test_auth_failures_raise_a_matchable_error(self, status_code: int) -> None: