Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add openai models, batches, completions and evals tables (#123365)

Tom Owersaa86410b6a80
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index a059307bc00b..01c64d5a9261 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -8719,14 +8719,14 @@ Today (20): `admin_api_keys`, `audit_logs`, `costs`, `invites`, `project_api_key
Diffed against: <https://raw.githubusercontent.com/openai/openai-openapi/master/openapi.yaml>
-- [ ] `/models` — lookup resolving the model IDs that appear in every usage\_\* and costs row (high)
-- [ ] `/chat/completions (list stored completions)` — per-request log with model, tokens and metadata - the finest-grained usage fact table available (high)
+- [x] `/models` — lookup resolving the model IDs that appear in every usage\_\* and costs row (high). Added as `models`; like every platform table below it needs the optional project API key, because OpenAI rejects Admin API keys outside the organization endpoints
+- [x] `/chat/completions (list stored completions)` — per-request log with model, tokens and metadata - the finest-grained usage fact table available (high). Added as `chat_completions`; only completions created with `store: true` are listed, and there is no server-side time filter, so it is full refresh
- [x] `/organization/groups and /organization/groups/{id}/users` — group membership table resolving which users belong to which group (high)
- [x] `/organization/roles and /organization/users/{id}/roles` — role assignments for the users we already sync (medium)
- [ ] `/organization/projects/{id}/groups` — project-to-group membership, complements the project_users table we sync (medium)
-- [ ] `/batches` — batch job records with request counts and completion status (medium)
+- [x] `/batches` — batch job records with request counts and completion status (medium). Added as `batches`
- [ ] `/fine_tuning/jobs (+ /events, /checkpoints)` — training job history, cost and status over time (medium)
-- [ ] `/evals, /evals/{id}/runs, /evals/{id}/runs/{id}/output_items` — eval run results - the analytical object for model quality tracking (medium)
+- [x] `/evals, /evals/{id}/runs, /evals/{id}/runs/{id}/output_items` — eval run results - the analytical object for model quality tracking (medium). Added as `evals`, `eval_runs` and `eval_run_output_items`
- [ ] `/chat/completions/{id}/messages` — message-level detail behind each stored completion (low)
- [ ] `/vector_stores` — lookup resolving the vector store IDs in usage_vector_stores (low)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/openai.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/openai.py
index d6babf62eaf9..678f74cf9181 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/openai.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/openai.py
@@ -7,3 +7,4 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common imp
@config.config
class OpenAISourceConfig(config.Config):
api_key: str
+ project_api_key: str | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openai/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openai/canonical_descriptions.py
index 612e5c102b96..f824a6d78e48 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openai/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openai/canonical_descriptions.py
@@ -302,4 +302,99 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"event_data": "JSON object with event-type-specific details.",
},
},
+ "models": {
+ "description": "Models available to the project of the project API key, including fine-tuned models.",
+ "docs_url": f"{_API_DOCS}/models/list",
+ "columns": {
+ "id": "Model identifier, as referenced by the model column of usage and request tables.",
+ "object": 'Object type, always "model".',
+ "created": "Unix timestamp (seconds) of when the model was created.",
+ "owned_by": "Organization that owns the model.",
+ },
+ },
+ "chat_completions": {
+ "description": "Chat completions created with store set to true in the project of the project API key.",
+ "docs_url": f"{_API_DOCS}/chat/list",
+ "columns": {
+ "id": "Unique identifier for the chat completion.",
+ "object": 'Object type, always "chat.completion".',
+ "created": "Unix timestamp (seconds) of when the chat completion was created.",
+ "model": "Model used for the chat completion.",
+ "choices": "JSON array of the completion choices, each with its message and finish reason.",
+ "usage": "JSON object of token usage for the request: prompt, completion, and total tokens.",
+ "metadata": "JSON object of key-value pairs attached to the request, or null.",
+ "service_tier": "Service tier used to process the request.",
+ },
+ },
+ "batches": {
+ "description": "Batch API jobs in the project of the project API key, with their status and request counts.",
+ "docs_url": f"{_API_DOCS}/batch/list",
+ "columns": {
+ "id": "Unique identifier for the batch.",
+ "object": 'Object type, always "batch".',
+ "endpoint": "OpenAI API endpoint the batch requests are sent to.",
+ "model": "Model ID used to process the batch.",
+ "input_file_id": "ID of the input file for the batch.",
+ "output_file_id": "ID of the file with the outputs of successful requests, or null.",
+ "error_file_id": "ID of the file with the outputs of failed requests, or null.",
+ "completion_window": "Time frame within which the batch must be processed.",
+ "status": "Batch status: validating, failed, in_progress, finalizing, completed, expired, cancelling, or cancelled.",
+ "created_at": "Unix timestamp (seconds) of when the batch was created.",
+ "completed_at": "Unix timestamp (seconds) of when the batch completed, or null.",
+ "failed_at": "Unix timestamp (seconds) of when the batch failed, or null.",
+ "request_counts": "JSON object with the total, completed, and failed request counts.",
+ "usage": "JSON object of token usage for the batch.",
+ "errors": "JSON object listing errors for the batch, or null.",
+ "metadata": "JSON object of key-value pairs attached to the batch, or null.",
+ },
+ },
+ "evals": {
+ "description": "Evaluations defined in the project of the project API key.",
+ "docs_url": f"{_API_DOCS}/evals/list",
+ "columns": {
+ "id": "Unique identifier for the evaluation.",
+ "object": 'Object type, always "eval".',
+ "name": "Name of the evaluation.",
+ "created_at": "Unix timestamp (seconds) of when the evaluation was created.",
+ "data_source_config": "JSON object configuring the data source used by runs of the evaluation.",
+ "testing_criteria": "JSON array of the graders applied to each sample.",
+ "metadata": "JSON object of key-value pairs attached to the evaluation, or null.",
+ },
+ },
+ "eval_runs": {
+ "description": "Runs of each evaluation, with their status, result counts, and per-criteria results.",
+ "docs_url": f"{_API_DOCS}/evals/getRuns",
+ "columns": {
+ "id": "Unique identifier for the evaluation run.",
+ "object": 'Object type, always "eval.run".',
+ "eval_id": "ID of the evaluation the run belongs to.",
+ "name": "Name of the run, or null.",
+ "status": "Run status: queued, in_progress, completed, canceled, or failed.",
+ "model": "Model evaluated in the run, or null.",
+ "created_at": "Unix timestamp (seconds) of when the run was created.",
+ "result_counts": "JSON object with the total, passed, failed, and errored sample counts.",
+ "per_model_usage": "JSON array of token usage per model during the run.",
+ "per_testing_criteria_results": "JSON array of passed and failed counts per testing criterion.",
+ "data_source": "JSON object describing the run's data source.",
+ "report_url": "URL of the run report in the OpenAI dashboard.",
+ "error": "JSON object describing why the run failed, or null.",
+ "metadata": "JSON object of key-value pairs attached to the run, or null.",
+ },
+ },
+ "eval_run_output_items": {
+ "description": "Per-sample results of each evaluation run, with the grader results for each sample.",
+ "docs_url": f"{_API_DOCS}/evals/getRunOutputItems",
+ "columns": {
+ "id": "Unique identifier for the output item.",
+ "object": 'Object type, always "eval.run.output_item".',
+ "run_id": "ID of the evaluation run the output item belongs to.",
+ "eval_id": "ID of the evaluation the output item belongs to.",
+ "created_at": "Unix timestamp (seconds) of when the output item was created.",
+ "status": "Status of the output item, such as pass or fail.",
+ "datasource_item_id": "Identifier of the data source item that was evaluated.",
+ "datasource_item": "JSON object with the input data source item.",
+ "results": "JSON array of grader results for the sample.",
+ "sample": "JSON object with the sample's model input, output, token usage, and any error.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openai/openai.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openai/openai.py
index a5285bc85103..bc1fe687c067 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openai/openai.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openai/openai.py
@@ -13,7 +13,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
rest_api_resource,
rest_api_resources,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import BasePaginator
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
+ BasePaginator,
+ SinglePagePaginator,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
ClientConfig,
@@ -210,8 +213,19 @@ _KEY_REJECTED_MESSAGE = (
# when OpenAI never answered sends them off to mint a replacement that fails the same way.
_PROBE_FAILED_MESSAGE = "PostHog couldn't check your Admin API key with OpenAI. Wait a few minutes and try again."
+_PROJECT_KEY_REJECTED_MESSAGE = (
+ "OpenAI rejected your project API key. Check it's a project API key (not an Admin API key) and hasn't been revoked."
+)
+_PROJECT_KEY_PROBE_FAILED_MESSAGE = (
+ "PostHog couldn't check your project API key with OpenAI. Wait a few minutes and try again."
+)
+
+PROJECT_KEY_MISSING_MESSAGE = (
+ "This OpenAI table needs a project API key. Add one to the source settings, or stop syncing this table."
+)
+
-def validate_credentials(api_key: str) -> tuple[bool, str | None]:
+def validate_credentials(api_key: str, project_api_key: Optional[str] = None) -> tuple[bool, str | None]:
# A single cheap probe against the smallest list endpoint confirms the admin key is genuine.
# 200 => valid. 403 => valid key without a scope we probed here; still a real key, so accept it
# at create time (sync-time 403s are caught by get_non_retryable_errors). 401 => bad key.
@@ -221,11 +235,21 @@ def validate_credentials(api_key: str) -> tuple[bool, str | None]:
headers={"Authorization": f"Bearer {api_key}", **_headers()},
ok_statuses=(200, 403),
)
+ if not ok:
+ return False, _KEY_REJECTED_MESSAGE if status == 401 else _PROBE_FAILED_MESSAGE
+ if not project_api_key:
+ return True, None
+ # Platform endpoints reject Admin API keys, so the project key gets its own probe. A 403 is a
+ # real key restricted away from models; the tables it can read still sync.
+ ok, status = validate_via_probe(
+ lambda: make_tracked_session(redact_values=(project_api_key,)),
+ f"{OPENAI_BASE_URL}/v1/models",
+ headers={"Authorization": f"Bearer {project_api_key}", **_headers()},
+ ok_statuses=(200, 403),
+ )
if ok:
return True, None
- if status == 401:
- return False, _KEY_REJECTED_MESSAGE
- return False, _PROBE_FAILED_MESSAGE
+ return False, _PROJECT_KEY_REJECTED_MESSAGE if status == 401 else _PROJECT_KEY_PROBE_FAILED_MESSAGE
def _row_id(*parts: Any) -> str:
@@ -330,6 +354,52 @@ def _make_fanout_data_map(endpoint: str, parent: str, parent_key: str) -> Callab
return _stamp
+def _fan_out_chain(endpoint: str) -> list[str]:
+ """Endpoint names from the root parent down to `endpoint`, e.g. evals -> eval_runs -> output items."""
+ chain = [endpoint]
+ while (parent := OPENAI_ENDPOINTS[chain[0]].fan_out_parent) is not None:
+ chain.insert(0, parent)
+ return chain
+
+
+def _fan_out_resources(chain: list[str]) -> list[str | EndpointResource]:
+ # The root parent sends no extra params: archived projects are left out of project fan-outs
+ # (see openai_source).
+ root = chain[0]
+ resources: list[str | EndpointResource] = [
+ {
+ "name": root,
+ "endpoint": {
+ "path": OPENAI_ENDPOINTS[root].path,
+ "params": {"limit": ENTITY_PAGE_SIZE},
+ "data_selector": "data",
+ "paginator": _EntityPaginator(),
+ },
+ }
+ ]
+ for name in chain[1:]:
+ config = OPENAI_ENDPOINTS[name]
+ parent, parent_key = config.fan_out_parent, config.fan_out_parent_key
+ assert parent is not None and parent_key is not None
+ resolved: dict[str, Any] = {parent_key: {"type": "resolve", "resource": parent, "field": "id"}}
+ for placeholder, parent_field in config.fan_out_extra_path_fields.items():
+ resolved[placeholder] = {"type": "resolve", "resource": parent, "field": parent_field}
+ resources.append(
+ {
+ "name": name,
+ "endpoint": {
+ "path": config.path,
+ "params": {"limit": ENTITY_PAGE_SIZE, **config.extra_params, **resolved},
+ "data_selector": "data",
+ "paginator": _EntityPaginator(),
+ },
+ "include_from_parent": ["id"],
+ "data_map": _make_fanout_data_map(name, parent, parent_key),
+ }
+ )
+ return resources
+
+
def _is_lookback_exceeded(error: HTTPError) -> bool:
response = error.response
if response is None or response.status_code != 400:
@@ -404,13 +474,18 @@ def openai_source(
job_id: str,
resumable_source_manager: ResumableSourceManager[OpenAIResumeConfig],
db_incremental_field_last_value: Optional[Any] = None,
+ project_api_key: Optional[str] = None,
) -> SourceResponse:
config = OPENAI_ENDPOINTS[endpoint]
+ if config.requires_project_key and not project_api_key:
+ raise ValueError(PROJECT_KEY_MISSING_MESSAGE)
+ token = project_api_key if config.requires_project_key and project_api_key else api_key
+
client_config: ClientConfig = {
"base_url": OPENAI_BASE_URL,
"headers": _headers(),
- "auth": {"type": "bearer", "token": api_key},
+ "auth": {"type": "bearer", "token": token},
}
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
@@ -418,40 +493,16 @@ def openai_source(
if config.fan_out_parent is not None and config.fan_out_parent_key is not None:
# Parent-scoped resources have no org-wide list endpoint; enumerate the parent and fetch
- # the resource per parent row. Archived projects are left out: OpenAI rejects every
- # project-scoped read under an archived project, which would fail the whole schema on the
- # first one. The `projects` table keeps them (see its `extra_params`) so historical
- # usage/cost rows still resolve their project.
+ # the resource per parent row. Nested resources chain the fan-out through each level, and
+ # the framework keeps no resume state for a chain. Archived projects are left out: OpenAI
+ # rejects every project-scoped read under an archived project, which would fail the whole
+ # schema on the first one. The `projects` table keeps them (see its `extra_params`) so
+ # historical usage/cost rows still resolve their project.
parent = config.fan_out_parent
- parent_config = OPENAI_ENDPOINTS[parent]
rest_config: RESTAPIConfig = {
"client": client_config,
"resource_defaults": None,
- "resources": [
- {
- "name": parent,
- "endpoint": {
- "path": parent_config.path,
- "params": {"limit": ENTITY_PAGE_SIZE},
- "data_selector": "data",
- "paginator": _EntityPaginator(),
- },
- },
- {
- "name": endpoint,
- "endpoint": {
- "path": config.path,
- "params": {
- "limit": ENTITY_PAGE_SIZE,
- config.fan_out_parent_key: {"type": "resolve", "resource": parent, "field": "id"},
- },
- "data_selector": "data",
- "paginator": _EntityPaginator(),
- },
- "include_from_parent": ["id"],
- "data_map": _make_fanout_data_map(endpoint, parent, config.fan_out_parent_key),
- },
- ],
+ "resources": _fan_out_resources(_fan_out_chain(endpoint)),
}
# Only a framework-shaped checkpoint can seed the fan-out; a legacy (cursor, project_id)
@@ -497,6 +548,10 @@ def openai_source(
params["group_by"] = config.group_by
paginator: BasePaginator = _BucketPaginator()
data_map = _make_bucket_data_map(config)
+ elif config.pagination == PaginationType.SINGLE:
+ params = {**config.extra_params}
+ paginator = SinglePagePaginator()
+ data_map = None
else:
params = {"limit": ENTITY_PAGE_SIZE, **config.extra_params}
paginator = _EntityPaginator()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openai/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openai/settings.py
index 74ec437bfdc3..71c02e41f0d5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openai/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openai/settings.py
@@ -15,6 +15,8 @@ class PaginationType(Enum):
# The usage/costs endpoints page with an opaque `page` token echoed back as `next_page`,
# alongside has_more. Each page holds time buckets, not rows.
PAGE = "page"
+ # The models list returns every model in one response, with no cursor or limit param.
+ SINGLE = "single"
def _datetime_incremental_field(name: str) -> IncrementalField:
@@ -60,8 +62,15 @@ class OpenAIEndpointConfig:
# placeholder the parent id fills and the column that carries it on each row.
fan_out_parent: Optional[str] = None
fan_out_parent_key: Optional[str] = None
+ # Extra path placeholders bound from fields of the parent row, as {placeholder: parent field}.
+ # A nested resource like eval run output items needs the grandparent id too, which its parent
+ # row carries.
+ fan_out_extra_path_fields: dict[str, str] = field(default_factory=dict)
# Extra static query params merged into every request (e.g. include_archived on projects).
extra_params: dict[str, str] = field(default_factory=dict)
+ # Platform resources (models, batches, stored completions, evals) reject Admin API keys, so
+ # they authenticate with the optional project API key instead.
+ requires_project_key: bool = False
should_sync_default: bool = True
@@ -244,10 +253,83 @@ OPENAI_ENDPOINTS: dict[str, OpenAIEndpointConfig] = {
# checkpoint the watermark per batch — "desc" defers the commit to sync completion.
sort_mode="desc",
),
+ # Platform resources are scoped to the project the project API key belongs to. None of them has
+ # a server-side time filter, so they are full refresh only. They are off by default because the
+ # project API key is optional.
+ "models": OpenAIEndpointConfig(
+ name="models",
+ path="/v1/models",
+ pagination=PaginationType.SINGLE,
+ primary_keys=["id"],
+ requires_project_key=True,
+ should_sync_default=False,
+ ),
+ # Lists only completions created with `store: true`.
+ "chat_completions": OpenAIEndpointConfig(
+ name="chat_completions",
+ path="/v1/chat/completions",
+ pagination=PaginationType.CURSOR,
+ primary_keys=["id"],
+ partition_key="created",
+ extra_params={"order": "asc"},
+ requires_project_key=True,
+ should_sync_default=False,
+ ),
+ "batches": OpenAIEndpointConfig(
+ name="batches",
+ path="/v1/batches",
+ pagination=PaginationType.CURSOR,
+ primary_keys=["id"],
+ partition_key="created_at",
+ requires_project_key=True,
+ should_sync_default=False,
+ ),
+ "evals": OpenAIEndpointConfig(
+ name="evals",
+ path="/v1/evals",
+ pagination=PaginationType.CURSOR,
+ primary_keys=["id"],
+ partition_key="created_at",
+ extra_params={"order": "asc", "order_by": "created_at"},
+ requires_project_key=True,
+ should_sync_default=False,
+ ),
+ "eval_runs": OpenAIEndpointConfig(
+ name="eval_runs",
+ path="/v1/evals/{eval_id}/runs",
+ pagination=PaginationType.CURSOR,
+ primary_keys=["eval_id", "id"],
+ partition_key="created_at",
+ fan_out_parent="evals",
+ fan_out_parent_key="eval_id",
+ extra_params={"order": "asc"},
+ requires_project_key=True,
+ should_sync_default=False,
+ ),
+ "eval_run_output_items": OpenAIEndpointConfig(
+ name="eval_run_output_items",
+ path="/v1/evals/{eval_id}/runs/{run_id}/output_items",
+ pagination=PaginationType.CURSOR,
+ primary_keys=["run_id", "id"],
+ partition_key="created_at",
+ fan_out_parent="eval_runs",
+ fan_out_parent_key="run_id",
+ fan_out_extra_path_fields={"eval_id": "eval_id"},
+ extra_params={"order": "asc"},
+ requires_project_key=True,
+ should_sync_default=False,
+ ),
}
ENDPOINTS = tuple(OPENAI_ENDPOINTS.keys())
+# Path prefixes of the endpoints that authenticate with the project API key.
+PROJECT_KEY_PATHS = tuple(
+ sorted(
+ {config.path.split("{")[0].rstrip("/") for config in OPENAI_ENDPOINTS.values() if config.requires_project_key}
+ )
+)
+
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
name: config.incremental_fields for name, config in OPENAI_ENDPOINTS.items()
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openai/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openai/source.py
index 990bdbf95a3e..ef8a83e814bc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openai/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openai/source.py
@@ -19,6 +19,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.typ
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.openai import OpenAISourceConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.openai.openai import (
OPENAI_BASE_URL,
+ PROJECT_KEY_MISSING_MESSAGE,
OpenAIResumeConfig,
openai_source,
validate_credentials as validate_openai_credentials,
@@ -27,6 +28,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.openai.set
ENDPOINTS,
INCREMENTAL_FIELDS,
OPENAI_ENDPOINTS,
+ PROJECT_KEY_PATHS,
)
from products.warehouse_sources.backend.types import ExternalDataSourceType
@@ -115,7 +117,9 @@ class OpenAISource(ResumableSource[OpenAISourceConfig, OpenAIResumeConfig]):
releaseStatus=ReleaseStatus.GA,
caption="""Enter your OpenAI Admin API key to pull your organization's API usage, cost, and admin data into the PostHog Data warehouse.
-Create an Admin API key (prefixed `sk-admin...`) in your [OpenAI organization settings](https://platform.openai.com/settings/organization/admin-keys). Only organization owners can create one; a regular project API key cannot read organization usage or costs.""",
+Create an Admin API key (prefixed `sk-admin...`) in your [OpenAI organization settings](https://platform.openai.com/settings/organization/admin-keys). Only organization owners can create one; a regular project API key cannot read organization usage or costs.
+
+To also sync models, batches, stored chat completions, and evals, add a [project API key](https://platform.openai.com/api-keys). OpenAI doesn't accept Admin API keys for these tables, and they only include data from the key's project. A restricted key needs read access to each of them.""",
iconPath="/static/services/openai.svg",
docsUrl="https://posthog.com/docs/cdp/sources/openai",
keywords=["llm", "gpt", "chatgpt", "ai usage", "cost"],
@@ -130,6 +134,14 @@ Create an Admin API key (prefixed `sk-admin...`) in your [OpenAI organization se
placeholder="sk-admin...",
secret=True,
),
+ SourceFieldInputConfig(
+ name="project_api_key",
+ label="Project API key (optional)",
+ type=SourceFieldInputConfigType.PASSWORD,
+ required=False,
+ placeholder="sk-proj-...",
+ secret=True,
+ ),
],
),
)
@@ -142,7 +154,19 @@ Create an Admin API key (prefixed `sk-admin...`) in your [OpenAI organization se
return CANONICAL_DESCRIPTIONS
def get_non_retryable_errors(self) -> dict[str, str | None]:
+ project_key_errors: dict[str, str | None] = {}
+ for path in PROJECT_KEY_PATHS:
+ project_key_errors[f"401 Client Error: Unauthorized for url: {OPENAI_BASE_URL}{path}"] = (
+ "Your OpenAI project API key is invalid or has been revoked. Create a new project API key, then update it in the source settings."
+ )
+ project_key_errors[f"403 Client Error: Forbidden for url: {OPENAI_BASE_URL}{path}"] = (
+ "Your OpenAI project API key can't read this table. Give the key read access to it in your OpenAI project, or stop syncing this table."
+ )
return {
+ # Platform tables authenticate with the project API key, so their auth failures point
+ # at that key. Listed first because the organization messages below also match them.
+ **project_key_errors,
+ PROJECT_KEY_MISSING_MESSAGE: None,
"401 Client Error: Unauthorized for url: https://api.openai.com": "Your OpenAI Admin API key is invalid or has been revoked. Create a new Admin API key in your OpenAI organization settings, then reconnect.",
"403 Client Error: Forbidden for url: https://api.openai.com": "Your OpenAI API key does not have organization admin access. Use an Admin API key (prefixed sk-admin) created by an organization owner, then reconnect.",
# Reached only after every later start time in the source's fallback was rejected too.
@@ -195,7 +219,22 @@ Create an Admin API key (prefixed `sk-admin...`) in your [OpenAI organization se
schema_name: Optional[str] = None,
api_version: str | None = None,
) -> tuple[bool, str | None]:
- return validate_openai_credentials(config.api_key)
+ return validate_openai_credentials(config.api_key, config.project_api_key)
+
+ def get_endpoint_permissions(
+ self,
+ config: OpenAISourceConfig,
+ team_id: int,
+ endpoints: list[str],
+ api_version: str | None = None,
+ ) -> dict[str, str | None]:
+ missing_key_reason = None if config.project_api_key else "Add a project API key to sync this table."
+ return {
+ name: missing_key_reason
+ if name in OPENAI_ENDPOINTS and OPENAI_ENDPOINTS[name].requires_project_key
+ else None
+ for name in endpoints
+ }
def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[OpenAIResumeConfig]:
return ResumableSourceManager[OpenAIResumeConfig](inputs, OpenAIResumeConfig)
@@ -212,6 +251,7 @@ Create an Admin API key (prefixed `sk-admin...`) in your [OpenAI organization se
team_id=inputs.team_id,
job_id=inputs.job_id,
resumable_source_manager=resumable_source_manager,
+ project_api_key=config.project_api_key,
db_incremental_field_last_value=inputs.db_incremental_field_last_value
if inputs.should_use_incremental_field
else None,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openai/tests/test_openai.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openai/tests/test_openai.py
index 1d2a2b53f4a2..c60a76840ba8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openai/tests/test_openai.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openai/tests/test_openai.py
@@ -13,6 +13,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.openai import OpenAISourceConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.openai.openai import (
+ PROJECT_KEY_MISSING_MESSAGE,
OpenAIResumeConfig,
_flatten_bucket_result,
_normalize_audit_log,
@@ -24,8 +25,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.openai.sou
PROJECT_USERS_PATH = "/v1/organization/projects/{project_id}/users"
-def _driver() -> SourceDriver:
- return SourceDriver(OpenAISource(), OpenAISourceConfig(api_key="sk-admin-test"))
+def _driver(project_api_key: str | None = None) -> SourceDriver:
+ return SourceDriver(OpenAISource(), OpenAISourceConfig(api_key="sk-admin-test", project_api_key=project_api_key))
def _entity_page(items: list[dict[str, Any]], *, has_more: bool, last_id: str | None = None) -> ScriptedResponse:
@@ -309,6 +310,65 @@ class TestProjectFanOut:
}
+class TestPlatformTables:
+ @parameterized.expand(
+ [
+ ("models", "/v1/models", None),
+ ("chat_completions", "/v1/chat/completions", "100"),
+ ]
+ )
+ def test_platform_tables_authenticate_with_the_project_key(
+ self, endpoint: str, path: str, expected_limit: str | None
+ ) -> None:
+ # OpenAI rejects Admin API keys outside the organization endpoints.
+ result = _driver(project_api_key="sk-proj-test").run(
+ endpoint, [ScriptedResponse(json={"object": "list", "data": [{"id": "row_1"}]})]
+ )
+
+ assert result.raised is None
+ assert [row["id"] for row in result.rows] == ["row_1"]
+ assert result.paths == [path]
+ assert {request.headers["authorization"] for request in result.requests} == {"Bearer sk-proj-test"}
+ assert result.params("limit") == [expected_limit]
+
+ def test_missing_project_key_fails_with_a_non_retryable_error_before_any_request(self) -> None:
+ result = _driver().run("batches", always(_entity_page([], has_more=False)))
+
+ assert isinstance(result.raised, ValueError)
+ assert str(result.raised) == PROJECT_KEY_MISSING_MESSAGE
+ assert result.requests == []
+ assert any(key in str(result.raised) for key in OpenAISource().get_non_retryable_errors())
+
+ def test_eval_run_output_items_fan_out_through_evals_and_runs(self) -> None:
+ # Output items sit two levels deep, so each request needs the eval id and the run id.
+ runs = {
+ "eval_1": [{"id": "run_1", "eval_id": "eval_1"}],
+ "eval_2": [{"id": "run_2", "eval_id": "eval_2"}, {"id": "run_3", "eval_id": "eval_2"}],
+ }
+
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == "/v1/evals":
+ return _entity_page([{"id": "eval_1"}, {"id": "eval_2"}], has_more=False)
+ for eval_id, eval_runs in runs.items():
+ if request.path == f"/v1/evals/{eval_id}/runs":
+ return _entity_page(eval_runs, has_more=False)
+ for run in eval_runs:
+ if request.path == f"/v1/evals/{eval_id}/runs/{run['id']}/output_items":
+ return _entity_page([{"id": "item_1", "eval_id": eval_id}], has_more=False)
+ raise AssertionError(f"Unexpected request: {request.path}")
+
+ result = _driver(project_api_key="sk-proj-test").run("eval_run_output_items", respond)
+
+ assert result.raised is None
+ assert sorted((row["eval_id"], row["run_id"], row["id"]) for row in result.rows) == [
+ ("eval_1", "run_1", "item_1"),
+ ("eval_2", "run_2", "item_1"),
+ ("eval_2", "run_3", "item_1"),
+ ]
+ assert not any(key.startswith("_") for row in result.rows for key in row)
+ assert {request.headers["authorization"] for request in result.requests} == {"Bearer sk-proj-test"}
+
+
class TestRetries:
@parameterized.expand([("rate_limited", 429), ("server_error", 503)])
def test_retryable_status_is_retried_then_succeeds(self, _name: str, status: int) -> None:
@@ -353,6 +413,31 @@ class TestValidateCredentials:
else:
assert message is not None and fragment in message
+ @parameterized.expand(
+ [
+ ("valid", 200, True, None),
+ # A restricted key without models access still reads the tables it has access to.
+ ("restricted", 403, True, None),
+ ("rejected", 401, False, "rejected your project API key"),
+ ]
+ )
+ def test_project_key_status_mapping(self, _name: str, status: int, expected: bool, fragment: str | None) -> None:
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == "/v1/organization/projects":
+ return ScriptedResponse(json={"data": [], "has_more": False})
+ return ScriptedResponse(status=status, json={})
+
+ with scripted_network(respond) as network:
+ is_valid, message = validate_credentials("sk-admin-test", "sk-proj-test")
+
+ models_requests = [request for request in network.requests_log if request.path == "/v1/models"]
+ assert {request.headers["authorization"] for request in models_requests} == {"Bearer sk-proj-test"}
+ assert is_valid is expected
+ if fragment is None:
+ assert message is None
+ else:
+ assert message is not None and fragment in message
+
class TestRetryableErrors:
@parameterized.expand(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openai/tests/test_openai_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openai/tests/test_openai_source.py
index 4445b795d964..cc96fac2d4b7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openai/tests/test_openai_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openai/tests/test_openai_source.py
@@ -53,6 +53,23 @@ class TestOpenAISchemas:
assert schema.supports_append is False
+class TestOpenAIEndpointPermissions:
+ @parameterized.expand(
+ [
+ ("without_project_key", None, "Add a project API key to sync this table."),
+ ("with_project_key", "sk-proj-test", None),
+ ]
+ )
+ def test_platform_tables_need_the_project_key(
+ self, _name: str, project_api_key: str | None, expected_reason: str | None
+ ) -> None:
+ config = OpenAISourceConfig(api_key="sk-admin-test", project_api_key=project_api_key)
+
+ permissions = OpenAISource().get_endpoint_permissions(config, team_id=1, endpoints=["users", "models", "evals"])
+
+ assert permissions == {"users": None, "models": expected_reason, "evals": expected_reason}
+
+
class TestOpenAISourceForPipeline:
@parameterized.expand(
[