:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 1 missing table to the Cohere source (#120898)
Tom Owers843b05352c32
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 6d0d633b95e6..51367c42e2f2 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -403,7 +403,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/codescene/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/cody/cody.py1 products/warehouse_sources/backend/temporal/data_imports/sources/cody/settings.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/cohere/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/coin_api/coin_api.py1 products/warehouse_sources/backend/temporal/data_imports/sources/coin_api/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/coingecko/coingecko.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 443730e000ba..e49a26475ce0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -3010,7 +3010,7 @@ Today (3): `datasets`, `embed_jobs`, `models`Diffed against: <https://raw.githubusercontent.com/cohere-ai/cohere-developer-experience/main/cohere-openapi.yaml>-- [ ] `GET /v2/batches (ListBatches)` — batch inference jobs with status and token counts - the only remaining job/usage table not synced (medium)+- [x] `GET /v2/batches (ListBatches)` — batch inference jobs with status and token counts - the only remaining job/usage table not synced (medium)- [ ] `GET /v1/finetuning/finetuned-models/{id}/events (ListEvents)` — fine-tune job state-transition history; resolves how a synced finetuned_model reached its status (medium)- [ ] `GET /v1/finetuning/finetuned-models/{id}/training-step-metrics (ListTrainingStepMetrics)` — per-step training loss/accuracy for evaluating fine-tunes (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/canonical_descriptions.pyindex 02ab98ba64d6..6bbb1df2712b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/canonical_descriptions.py@@ -52,4 +52,26 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"meta": "Billed-unit metadata and API version info for the job.",},},+ "batches": {+ "description": "A batch job that runs an uploaded dataset of requests through a model.",+ "docs_url": "https://docs.cohere.com/reference/list-batches",+ "columns": {+ "id": "Unique identifier for the batch.",+ "name": "Human-readable name of the batch.",+ "creator_id": "ID of the user who created the batch.",+ "org_id": "ID of the organization that owns the batch.",+ "status": "Current stage in the life-cycle of the batch (e.g. BATCH_STATUS_IN_PROGRESS, BATCH_STATUS_COMPLETED).",+ "status_reason": "More details about the reason for the status of the batch.",+ "created_at": "Time at which the batch was created.",+ "updated_at": "Time at which the batch was last updated.",+ "input_dataset_id": "ID of the dataset the batch reads its requests from.",+ "output_dataset_id": "ID of the dataset the batch writes its outputs to.",+ "model": "The name of the model the batch uses.",+ "input_tokens": "The total number of input tokens in the batch.",+ "output_tokens": "The total number of output tokens in the batch.",+ "num_records": "The total number of records in the batch.",+ "num_successful_records": "The current number of successful records in the batch.",+ "num_failed_records": "The current number of failed records in the batch.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/cohere.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/cohere.pyindex 6c1fdf7ff25c..ec526314db44 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/cohere.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/cohere.py@@ -1,6 +1,7 @@from typing import Anyfrom products.warehouse_sources.backend.temporal.data_imports.sources.cohere.settings import (+ COHERE_API_VERSION_V1,COHERE_ENDPOINTS,RETIRED_ENDPOINTS,CohereEndpointConfig,@@ -20,14 +21,14 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse-# Cohere versions its API in the URL path. Every list endpoint this source reads is served at-# /v1/; Cohere's v2 generation covers the inference surface only (/v2/chat, /v2/embed, /v2/rerank),+# Cohere serves a single global API host; there are no regional variants.+COHERE_API_HOST = "https://api.cohere.com"++# Every list endpoint this source reads is served at /v1/, except batches, which exist only on /v2/.+# Cohere's v2 generation otherwise covers the inference surface (/v2/chat, /v2/embed, /v2/rerank),# which this source does not touch. The source-level version label is bound to this constant so a# declared version can never drift from the path the requests actually use.-COHERE_API_VERSION_V1 = "v1"--# Cohere serves a single global API host; there are no regional variants.-COHERE_BASE_URL = f"https://api.cohere.com/{COHERE_API_VERSION_V1}"+COHERE_BASE_URL = f"{COHERE_API_HOST}/{COHERE_API_VERSION_V1}"def _headers() -> dict[str, str]:@@ -67,14 +68,14 @@ def cohere_source(config = COHERE_ENDPOINTS[endpoint]partitioned = config.partition_key is not None- params: dict[str, Any] = {}+ params: dict[str, Any] = dict(config.extra_params)if config.pagination == CoherePagination.PAGE_TOKEN:# The cursor param is injected by the paginator; the static page size is a plain query param.params["page_size"] = config.page_sizerest_config: RESTAPIConfig = {"client": {- "base_url": COHERE_BASE_URL,+ "base_url": f"{COHERE_API_HOST}/{config.api_version}","headers": _headers(),"auth": {"type": "bearer", "token": api_key},"paginator": _build_paginator(config),diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/settings.pyindex 4cad86dd4924..349198ed26f6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/settings.py@@ -1,19 +1,27 @@-from dataclasses import dataclass, field+from dataclasses import fieldfrom enum import Enum+from typing import Any++from posthog.dataclasses import frozen++# Cohere versions its API in the URL path. Most list endpoints live only on /v1/, while batches+# live only on /v2/, so the path version is fixed per endpoint rather than chosen by the user.+COHERE_API_VERSION_V1 = "v1"+COHERE_API_VERSION_V2 = "v2"class CoherePagination(Enum):# ?limit=&offset= — used by /datasets. Terminate when a page returns fewer rows than the# page size.OFFSET = "offset"- # ?page_size=&page_token= with a next_page_token in the body — used by /models. Terminate- # when the response omits the token.+ # ?page_size=&page_token= with a next_page_token in the body — used by /models and /batches.+ # Terminate when the response omits the token.PAGE_TOKEN = "page_token"# Single unpaginated request returning every row — used by /embed-jobs.NONE = "none"-@dataclass+@frozenclass CohereEndpointConfig:name: strpath: str@@ -27,6 +35,8 @@ class CohereEndpointConfig:page_size: int = 100# Body key carrying the cursor for PAGE_TOKEN endpoints.next_token_key: str = "next_page_token"+ api_version: str = COHERE_API_VERSION_V1+ extra_params: dict[str, Any] = field(default_factory=dict)# The list endpoints a user connecting Cohere actually wants to warehouse: syncable asset and@@ -56,6 +66,16 @@ COHERE_ENDPOINTS: dict[str, CohereEndpointConfig] = {pagination=CoherePagination.NONE,primary_keys=["job_id"],),+ "batches": CohereEndpointConfig(+ name="batches",+ path="/batches",+ data_key="batches",+ pagination=CoherePagination.PAGE_TOKEN,+ api_version=COHERE_API_VERSION_V2,+ page_size=1000, # /batches caps page_size at 1000+ # An explicit, stable order keeps page boundaries steady while new batches are created mid-sync.+ extra_params={"order_by": "created_at"},+ ),}# Tables whose Cohere endpoint no longer exists. Dropping the name from COHERE_ENDPOINTS is whatdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/source.pyindex 0bd6ae1e8929..6eb894b2e2c6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/source.py@@ -39,11 +39,11 @@ class CohereSource(SimpleSource[CohereSourceConfig]):)lists_tables_without_credentials = True # static endpoint catalog — safe for public docs- # Cohere's v2 generation adds no route for anything this source reads: its own v2 reference- # documents datasets, models, and embed jobs at /v1/ paths. A "v2" pin would therefore name a- # wire Cohere does not serve for these tables. v1 is not sunset either - the 2025-09-15- # announcement retires individual endpoints, not the version - so v1 stays supported and- # undeprecated.+ # Cohere's v2 reference documents datasets, models, and embed jobs at /v1/ paths, so a "v2" pin+ # would name a wire Cohere does not serve for these tables. Batches exist only at /v2/, with no+ # v1 route to choose between, so that endpoint's path version is fixed in settings and is not a+ # version pin. v1 is not sunset either - the 2025-09-15 announcement retires individual+ # endpoints, not the version - so v1 stays supported and undeprecated.supported_versions = (COHERE_API_VERSION_V1,)default_version = COHERE_API_VERSION_V1api_docs_url = "https://docs.cohere.com/reference/about"diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/tests/test_cohere.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/tests/test_cohere.pyindex 6bd5cd582f3b..fdff76427ef0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/tests/test_cohere.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cohere/tests/test_cohere.py@@ -46,22 +46,34 @@ class TestOffsetPagination:class TestPageTokenPagination:- def test_follows_next_page_token(self) -> None:+ @pytest.mark.parametrize(+ "endpoint, key, expected_path, expected_order_by",+ [+ ("models", "name", "/v1/models", [None, None]),+ # Batches exist only on /v2/; a /v1/ request would 404 on every sync.+ ("batches", "id", "/v2/batches", ["created_at", "created_at"]),+ ],+ )+ def test_follows_next_page_token(+ self, endpoint: str, key: str, expected_path: str, expected_order_by: list[str | None]+ ) -> None:result = _driver().run(- "models",+ endpoint,[- _response("models", [{"name": "command"}], extra={"next_page_token": "t2"}),- _response("models", [{"name": "embed"}]),+ _response(endpoint, [{key: "first"}], extra={"next_page_token": "t2"}),+ _response(endpoint, [{key: "second"}]),],)assert result.raised is None- assert [row["name"] for row in result.rows] == ["command", "embed"]+ assert [row[key] for row in result.rows] == ["first", "second"]+ assert {request.path for request in result.requests} == {expected_path}# First request carries the page size but no token; the second carries the token from the- # first response. Absence of a token in the second response ends pagination. /models caps- # page_size at 1000.+ # first response. Absence of a token in the second response ends pagination. Both endpoints+ # cap page_size at 1000.assert result.params("page_size") == ["1000", "1000"]assert result.params("page_token") == [None, "t2"]+ assert result.params("order_by") == expected_order_byassert len(result.requests) == 2