Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 1 missing table to the Cohere source (#120898)

Tom Owers843b05352c32
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 6d0d633b95e6..51367c42e2f2 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -403,7 +403,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/codescene/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/cody/cody.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/cody/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/cohere/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/coin_api/coin_api.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/coin_api/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/coingecko/coingecko.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 443730e000ba..e49a26475ce0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -3010,7 +3010,7 @@ Today (3): `datasets`, `embed_jobs`, `models`
Diffed against: <https://raw.githubusercontent.com/cohere-ai/cohere-developer-experience/main/cohere-openapi.yaml>
-- [ ] `GET /v2/batches (ListBatches)` — batch inference jobs with status and token counts - the only remaining job/usage table not synced (medium)
+- [x] `GET /v2/batches (ListBatches)` — batch inference jobs with status and token counts - the only remaining job/usage table not synced (medium)
- [ ] `GET /v1/finetuning/finetuned-models/{id}/events (ListEvents)` — fine-tune job state-transition history; resolves how a synced finetuned_model reached its status (medium)
- [ ] `GET /v1/finetuning/finetuned-models/{id}/training-step-metrics (ListTrainingStepMetrics)` — per-step training loss/accuracy for evaluating fine-tunes (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/canonical_descriptions.py
index 02ab98ba64d6..6bbb1df2712b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/canonical_descriptions.py
@@ -52,4 +52,26 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"meta": "Billed-unit metadata and API version info for the job.",
},
},
+ "batches": {
+ "description": "A batch job that runs an uploaded dataset of requests through a model.",
+ "docs_url": "https://docs.cohere.com/reference/list-batches",
+ "columns": {
+ "id": "Unique identifier for the batch.",
+ "name": "Human-readable name of the batch.",
+ "creator_id": "ID of the user who created the batch.",
+ "org_id": "ID of the organization that owns the batch.",
+ "status": "Current stage in the life-cycle of the batch (e.g. BATCH_STATUS_IN_PROGRESS, BATCH_STATUS_COMPLETED).",
+ "status_reason": "More details about the reason for the status of the batch.",
+ "created_at": "Time at which the batch was created.",
+ "updated_at": "Time at which the batch was last updated.",
+ "input_dataset_id": "ID of the dataset the batch reads its requests from.",
+ "output_dataset_id": "ID of the dataset the batch writes its outputs to.",
+ "model": "The name of the model the batch uses.",
+ "input_tokens": "The total number of input tokens in the batch.",
+ "output_tokens": "The total number of output tokens in the batch.",
+ "num_records": "The total number of records in the batch.",
+ "num_successful_records": "The current number of successful records in the batch.",
+ "num_failed_records": "The current number of failed records in the batch.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/cohere.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/cohere.py
index 6c1fdf7ff25c..ec526314db44 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/cohere.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/cohere.py
@@ -1,6 +1,7 @@
from typing import Any
from products.warehouse_sources.backend.temporal.data_imports.sources.cohere.settings import (
+ COHERE_API_VERSION_V1,
COHERE_ENDPOINTS,
RETIRED_ENDPOINTS,
CohereEndpointConfig,
@@ -20,14 +21,14 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-# Cohere versions its API in the URL path. Every list endpoint this source reads is served at
-# /v1/; Cohere's v2 generation covers the inference surface only (/v2/chat, /v2/embed, /v2/rerank),
+# Cohere serves a single global API host; there are no regional variants.
+COHERE_API_HOST = "https://api.cohere.com"
+
+# Every list endpoint this source reads is served at /v1/, except batches, which exist only on /v2/.
+# Cohere's v2 generation otherwise covers the inference surface (/v2/chat, /v2/embed, /v2/rerank),
# which this source does not touch. The source-level version label is bound to this constant so a
# declared version can never drift from the path the requests actually use.
-COHERE_API_VERSION_V1 = "v1"
-
-# Cohere serves a single global API host; there are no regional variants.
-COHERE_BASE_URL = f"https://api.cohere.com/{COHERE_API_VERSION_V1}"
+COHERE_BASE_URL = f"{COHERE_API_HOST}/{COHERE_API_VERSION_V1}"
def _headers() -> dict[str, str]:
@@ -67,14 +68,14 @@ def cohere_source(
config = COHERE_ENDPOINTS[endpoint]
partitioned = config.partition_key is not None
- params: dict[str, Any] = {}
+ params: dict[str, Any] = dict(config.extra_params)
if config.pagination == CoherePagination.PAGE_TOKEN:
# The cursor param is injected by the paginator; the static page size is a plain query param.
params["page_size"] = config.page_size
rest_config: RESTAPIConfig = {
"client": {
- "base_url": COHERE_BASE_URL,
+ "base_url": f"{COHERE_API_HOST}/{config.api_version}",
"headers": _headers(),
"auth": {"type": "bearer", "token": api_key},
"paginator": _build_paginator(config),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/settings.py
index 4cad86dd4924..349198ed26f6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/settings.py
@@ -1,19 +1,27 @@
-from dataclasses import dataclass, field
+from dataclasses import field
from enum import Enum
+from typing import Any
+
+from posthog.dataclasses import frozen
+
+# Cohere versions its API in the URL path. Most list endpoints live only on /v1/, while batches
+# live only on /v2/, so the path version is fixed per endpoint rather than chosen by the user.
+COHERE_API_VERSION_V1 = "v1"
+COHERE_API_VERSION_V2 = "v2"
class CoherePagination(Enum):
# ?limit=&offset= — used by /datasets. Terminate when a page returns fewer rows than the
# page size.
OFFSET = "offset"
- # ?page_size=&page_token= with a next_page_token in the body — used by /models. Terminate
- # when the response omits the token.
+ # ?page_size=&page_token= with a next_page_token in the body — used by /models and /batches.
+ # Terminate when the response omits the token.
PAGE_TOKEN = "page_token"
# Single unpaginated request returning every row — used by /embed-jobs.
NONE = "none"
-@dataclass
+@frozen
class CohereEndpointConfig:
name: str
path: str
@@ -27,6 +35,8 @@ class CohereEndpointConfig:
page_size: int = 100
# Body key carrying the cursor for PAGE_TOKEN endpoints.
next_token_key: str = "next_page_token"
+ api_version: str = COHERE_API_VERSION_V1
+ extra_params: dict[str, Any] = field(default_factory=dict)
# The list endpoints a user connecting Cohere actually wants to warehouse: syncable asset and
@@ -56,6 +66,16 @@ COHERE_ENDPOINTS: dict[str, CohereEndpointConfig] = {
pagination=CoherePagination.NONE,
primary_keys=["job_id"],
),
+ "batches": CohereEndpointConfig(
+ name="batches",
+ path="/batches",
+ data_key="batches",
+ pagination=CoherePagination.PAGE_TOKEN,
+ api_version=COHERE_API_VERSION_V2,
+ page_size=1000, # /batches caps page_size at 1000
+ # An explicit, stable order keeps page boundaries steady while new batches are created mid-sync.
+ extra_params={"order_by": "created_at"},
+ ),
}
# Tables whose Cohere endpoint no longer exists. Dropping the name from COHERE_ENDPOINTS is what
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/source.py
index 0bd6ae1e8929..6eb894b2e2c6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/source.py
@@ -39,11 +39,11 @@ class CohereSource(SimpleSource[CohereSourceConfig]):
)
lists_tables_without_credentials = True # static endpoint catalog — safe for public docs
- # Cohere's v2 generation adds no route for anything this source reads: its own v2 reference
- # documents datasets, models, and embed jobs at /v1/ paths. A "v2" pin would therefore name a
- # wire Cohere does not serve for these tables. v1 is not sunset either - the 2025-09-15
- # announcement retires individual endpoints, not the version - so v1 stays supported and
- # undeprecated.
+ # Cohere's v2 reference documents datasets, models, and embed jobs at /v1/ paths, so a "v2" pin
+ # would name a wire Cohere does not serve for these tables. Batches exist only at /v2/, with no
+ # v1 route to choose between, so that endpoint's path version is fixed in settings and is not a
+ # version pin. v1 is not sunset either - the 2025-09-15 announcement retires individual
+ # endpoints, not the version - so v1 stays supported and undeprecated.
supported_versions = (COHERE_API_VERSION_V1,)
default_version = COHERE_API_VERSION_V1
api_docs_url = "https://docs.cohere.com/reference/about"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/tests/test_cohere.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/tests/test_cohere.py
index 6bd5cd582f3b..fdff76427ef0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/tests/test_cohere.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/tests/test_cohere.py
@@ -46,22 +46,34 @@ class TestOffsetPagination:
class TestPageTokenPagination:
- def test_follows_next_page_token(self) -> None:
+ @pytest.mark.parametrize(
+ "endpoint, key, expected_path, expected_order_by",
+ [
+ ("models", "name", "/v1/models", [None, None]),
+ # Batches exist only on /v2/; a /v1/ request would 404 on every sync.
+ ("batches", "id", "/v2/batches", ["created_at", "created_at"]),
+ ],
+ )
+ def test_follows_next_page_token(
+ self, endpoint: str, key: str, expected_path: str, expected_order_by: list[str | None]
+ ) -> None:
result = _driver().run(
- "models",
+ endpoint,
[
- _response("models", [{"name": "command"}], extra={"next_page_token": "t2"}),
- _response("models", [{"name": "embed"}]),
+ _response(endpoint, [{key: "first"}], extra={"next_page_token": "t2"}),
+ _response(endpoint, [{key: "second"}]),
],
)
assert result.raised is None
- assert [row["name"] for row in result.rows] == ["command", "embed"]
+ assert [row[key] for row in result.rows] == ["first", "second"]
+ assert {request.path for request in result.requests} == {expected_path}
# First request carries the page size but no token; the second carries the token from the
- # first response. Absence of a token in the second response ends pagination. /models caps
- # page_size at 1000.
+ # first response. Absence of a token in the second response ends pagination. Both endpoints
+ # cap page_size at 1000.
assert result.params("page_size") == ["1000", "1000"]
assert result.params("page_token") == [None, "t2"]
+ assert result.params("order_by") == expected_order_by
assert len(result.requests) == 2