:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-quality): filter and page the subject catalog for mcp (#115017)
posthog[bot]bd0e638b1eef
diff --git a/products/data_quality/backend/facade/enums.py b/products/data_quality/backend/facade/enums.pyindex c7e81d2f04f7..9f00c865ade9 100644--- a/products/data_quality/backend/facade/enums.py+++ b/products/data_quality/backend/facade/enums.py@@ -74,7 +74,7 @@ class SuiteRunTrigger(StrEnum):SCHEDULED = "scheduled"-class SubjectType(StrEnum):+class SubjectType(LabeledStrEnum):"""Kind of catalog object a check targets.On the check itself the subject is a foreign key (``saved_query``, ``table``, or ``metric``), or@@ -82,10 +82,10 @@ class SubjectType(StrEnum):``(subject_type, subject_uuid, name)`` tuples so it outlives hard deletes."""- TABLE = "table"- VIEW = "view"- METRIC = "metric"- POSTHOG_TABLE = "posthog_table"+ TABLE = "table", "table"+ VIEW = "view", "view"+ METRIC = "metric", "metric"+ POSTHOG_TABLE = "posthog_table", "posthog_table"# The labels repeat the values because the published OpenAPI enum lists these exact pairs.diff --git a/products/data_quality/backend/logic/subjects.py b/products/data_quality/backend/logic/subjects.pyindex 03e5bc7e420a..e5abb965ffa9 100644--- a/products/data_quality/backend/logic/subjects.py+++ b/products/data_quality/backend/logic/subjects.py@@ -66,7 +66,7 @@ def unqueryable_table_ids(team_id: int) -> set[UUID]:return excluded-def selectable_subjects(team_id: int, kinds: Collection[SubjectType]) -> list[SelectableSubject]:+def selectable_subjects(team_id: int, kinds: Collection[SubjectType], search: str = "") -> list[SelectableSubject]:"""Everything in this team a check can be authored on, of the kinds asked for.The catalog, not the gate: narrowing it to what the caller may read is the caller's job.@@ -117,7 +117,11 @@ def selectable_subjects(team_id: int, kinds: Collection[SubjectType]) -> list[Se)for metric in testable_metric_subjects(team_id))- return sorted(subjects, key=lambda subject: (subject.subject_type, subject.name))+ if needle := search.strip().lower():+ subjects = [+ subject for subject in subjects if needle in subject.name.lower() or needle in subject.display_name.lower()+ ]+ return sorted(subjects, key=lambda subject: (subject.subject_type, subject.name, subject.id))def testable_metric_subjects(team_id: int) -> list[MetricSubject]:diff --git a/products/data_quality/backend/presentation/serializers.py b/products/data_quality/backend/presentation/serializers.pyindex c38c11346a42..d295a3507069 100644--- a/products/data_quality/backend/presentation/serializers.py+++ b/products/data_quality/backend/presentation/serializers.py@@ -70,6 +70,40 @@ class DataQualitySubjectSerializer(serializers.Serializer):)+class DataQualitySubjectsQuerySerializer(serializers.Serializer):+ """Filters and paging for the subject catalog. Without any of them it returns every subject."""++ subject_type = serializers.ChoiceField(+ choices=SubjectType.choices,+ required=False,+ help_text="Return only subjects of this kind: 'table', 'view', 'metric', or 'posthog_table'.",+ )+ search = serializers.CharField(+ required=False,+ allow_blank=True,+ help_text="Return only subjects whose name or display name contains this text, ignoring case.",+ )+ include_columns = serializers.BooleanField(+ required=False,+ default=True,+ help_text="Set to false to return an empty columns map for every subject. "+ "Use it to find a subject id without reading every column schema.",+ )+ limit = serializers.IntegerField(+ required=False,+ min_value=1,+ max_value=1000,+ help_text="Maximum number of subjects to return. Omit to return every match.",+ )+ offset = serializers.IntegerField(+ required=False,+ min_value=0,+ default=0,+ help_text="Number of matching subjects to skip. Subjects are sorted by subject_type, then name. "+ "When a response has limit rows, call again with offset increased by limit to read the next page.",+ )++class DataQualityOutputColumnSerializer(serializers.Serializer):name = serializers.CharField(help_text="Output column name available through the {metric} relation.")type = serializers.CharField(allow_null=True, help_text="ClickHouse type, or null when it could not be inferred.")diff --git a/products/data_quality/backend/presentation/views.py b/products/data_quality/backend/presentation/views.pyindex be9fbef9c1a4..3a135d9b0af4 100644--- a/products/data_quality/backend/presentation/views.py+++ b/products/data_quality/backend/presentation/views.py@@ -52,6 +52,7 @@ from .serializers import (DataQualitySubjectRefSerializer,DataQualitySubjectScheduleSerializer,DataQualitySubjectSerializer,+ DataQualitySubjectsQuerySerializer,DataQualitySuiteRunSerializer,SubjectHealthSerializer,)@@ -680,13 +681,21 @@ class DataQualityCheckViewSet(_ProjectQualityViewSet, viewsets.ModelViewSet):return Response(CheckTypeSerializer(api.list_check_types(subject_type), many=True).data)@extend_schema(- description="Everything in this project you can author a check on, with each subject's columns.",+ description="Everything in this project you can author a check on, with each subject's columns. "+ "Filter by subject_type or search, and page with limit and offset.",+ parameters=[DataQualitySubjectsQuerySerializer],request=None,responses={200: DataQualitySubjectSerializer(many=True)},)@action(methods=["GET"], detail=False, pagination_class=None)def subjects(self, request: Request, **kwargs) -> Response:- selectable = api.selectable_subjects(self.team_id, self._authorized_subject_types())+ query = DataQualitySubjectsQuerySerializer(data=request.query_params)+ query.is_valid(raise_exception=True)+ params = query.validated_data+ kinds = self._authorized_subject_types()+ if subject_type := params.get("subject_type"):+ kinds = kinds & {SubjectType(subject_type)}+ selectable = api.selectable_subjects(self.team_id, kinds, params.get("search", ""))editable_kinds = self._authorized_subject_types(write=True)if self._can_be_object_denied():readable = self._denial_context().readable@@ -700,6 +709,11 @@ class DataQualityCheckViewSet(_ProjectQualityViewSet, viewsets.ModelViewSet):selectable = [replace(subject, editable=SubjectType(subject.subject_type) in editable_kinds) for subject in selectable]+ offset = params["offset"]+ limit = params.get("limit")+ selectable = selectable[offset : None if limit is None else offset + limit]+ if not params["include_columns"]:+ selectable = [replace(subject, columns={}) for subject in selectable]return Response(DataQualitySubjectSerializer(selectable, many=True).data)@extend_schema(request=None, responses={200: DataQualityMetricSubjectSerializer(many=True)})diff --git a/products/data_quality/backend/tests/test_api.py b/products/data_quality/backend/tests/test_api.pyindex 76e3d9435619..acab28085ad6 100644--- a/products/data_quality/backend/tests/test_api.py+++ b/products/data_quality/backend/tests/test_api.py@@ -1481,6 +1481,31 @@ class TestDataQualityCheckAPI(APIBaseTest):assert "orders" in editable_by_nameassert not any(editable_by_name.values())+ def test_the_catalog_narrows_by_search_and_subject_type(self) -> None:+ DataWarehouseSavedQuery.objects.create(team=self.team, name="Orders_daily", query={"kind": "HogQLQuery"})+ DataWarehouseSavedQuery.objects.create(team=self.team, name="refunds", query={"kind": "HogQLQuery"})++ by_search = self.client.get(f"{self.url}/subjects/", {"search": "ORDERS", "subject_type": "view"})+ assert by_search.status_code == status.HTTP_200_OK, by_search.content+ assert [row["name"] for row in by_search.json()] == ["Orders_daily", "orders"]++ events = self.client.get(f"{self.url}/subjects/", {"search": "events", "subject_type": "posthog_table"})+ assert [(row["name"], bool(row["columns"])) for row in events.json()] == [("events", True)]++ bare = self.client.get(f"{self.url}/subjects/", {"search": "events", "include_columns": "false"})+ assert [(row["name"], row["columns"]) for row in bare.json()] == [("events", {})]++ def test_the_catalog_pages_with_limit_and_offset(self) -> None:+ DataWarehouseSavedQuery.objects.create(team=self.team, name="refunds", query={"kind": "HogQLQuery"})+ everything = [row["id"] for row in self.client.get(f"{self.url}/subjects/").json()]++ first = self.client.get(f"{self.url}/subjects/", {"limit": 2}).json()+ second = self.client.get(f"{self.url}/subjects/", {"limit": 2, "offset": 2}).json()++ assert [row["id"] for row in first + second] == everything[:4]+ assert len(everything) > 4+ assert self.client.get(f"{self.url}/subjects/", {"limit": 0}).status_code == status.HTTP_400_BAD_REQUEST+@parameterized.expand([("list", lambda self, check, suite: self.client.get(self._checks_of(self.view.id))),diff --git a/products/data_quality/frontend/generated/api.schemas.ts b/products/data_quality/frontend/generated/api.schemas.tsindex 06e8c579d809..7abede9535db 100644--- a/products/data_quality/frontend/generated/api.schemas.ts+++ b/products/data_quality/frontend/generated/api.schemas.ts@@ -1037,6 +1037,48 @@ export const DataQualityChecksScheduleRetrieveSubjectType = {View: 'view',} as const+export type DataQualityChecksSubjectsListParams = {+ /**+ * Set to false to return an empty columns map for every subject. Use it to find a subject id without reading every column schema.+ */+ include_columns?: boolean+ /**+ * Maximum number of subjects to return. Omit to return every match.+ * @minimum 1+ * @maximum 1000+ */+ limit?: number+ /**+ * Number of matching subjects to skip. Subjects are sorted by subject_type, then name. When a response has limit rows, call again with offset increased by limit to read the next page.+ * @minimum 0+ */+ offset?: number+ /**+ * Return only subjects whose name or display name contains this text, ignoring case.+ */+ search?: string+ /**+ * Return only subjects of this kind: 'table', 'view', 'metric', or 'posthog_table'.+ *+ * * `table` - table+ * * `view` - view+ * * `metric` - metric+ * * `posthog_table` - posthog_table+ * @minLength 1+ */+ subject_type?: DataQualityChecksSubjectsListSubjectType+}++export type DataQualityChecksSubjectsListSubjectType =+ (typeof DataQualityChecksSubjectsListSubjectType)[keyof typeof DataQualityChecksSubjectsListSubjectType]++export const DataQualityChecksSubjectsListSubjectType = {+ Table: 'table',+ View: 'view',+ Metric: 'metric',+ PosthogTable: 'posthog_table',+} as const+export type DataQualityRunsListParams = {/*** Number of results to return per page.diff --git a/products/data_quality/frontend/generated/api.ts b/products/data_quality/frontend/generated/api.tsindex 33f64516dcdb..b1be4713ae11 100644--- a/products/data_quality/frontend/generated/api.ts+++ b/products/data_quality/frontend/generated/api.ts@@ -19,6 +19,7 @@ import type {DataQualityChecksListParams,DataQualityChecksOutputSchemaRetrieveParams,DataQualityChecksScheduleRetrieveParams,+ DataQualityChecksSubjectsListParams,DataQualityMetricSubjectApi,DataQualityOutputSchemaApi,DataQualityRunRequestApi,@@ -393,18 +394,34 @@ export const dataQualityChecksSchedulesList = async (})}-export const getDataQualityChecksSubjectsListUrl = (projectId: string) => {- return `/api/projects/${projectId}/data_quality_checks/subjects/`+export const getDataQualityChecksSubjectsListUrl = (+ projectId: string,+ params?: DataQualityChecksSubjectsListParams+) => {+ const normalizedParams = new URLSearchParams()++ Object.entries(params || {}).forEach(([key, value]) => {+ if (value !== undefined) {+ normalizedParams.append(key, value === null ? 'null' : String(value))+ }+ })++ const stringifiedParams = normalizedParams.toString()++ return stringifiedParams.length > 0+ ? `/api/projects/${projectId}/data_quality_checks/subjects/?${stringifiedParams}`+ : `/api/projects/${projectId}/data_quality_checks/subjects/`}/**- * Everything in this project you can author a check on, with each subject's columns.+ * Everything in this project you can author a check on, with each subject's columns. Filter by subject_type or search, and page with limit and offset.*/export const dataQualityChecksSubjectsList = async (projectId: string,+ params?: DataQualityChecksSubjectsListParams,options?: RequestInit): Promise<DataQualitySubjectApi[]> => {- return apiMutator<DataQualitySubjectApi[]>(getDataQualityChecksSubjectsListUrl(projectId), {+ return apiMutator<DataQualitySubjectApi[]>(getDataQualityChecksSubjectsListUrl(projectId, params), {...options,method: 'GET',})diff --git a/products/data_quality/mcp/tools.yaml b/products/data_quality/mcp/tools.yamlindex 0be6423e9fb0..774240582767 100644--- a/products/data_quality/mcp/tools.yaml+++ b/products/data_quality/mcp/tools.yaml@@ -151,7 +151,13 @@ tools:idempotent: truetitle: List what a data quality check can be authored ondescription: >- Every table, view, metric and PostHog table in this project you can read, with each one's columns and- whether you can put a check on it. Start here: a check names its subject by the subject_type and id this- returns, and a relationships check can point at a subject you can only read. A subject whose time_column is- set can take a lookback_hours window.+ Tables, views, metrics and PostHog tables in this project you can read, with each one's columns and whether+ you can put a check on it. Start here: a check names its subject by the subject_type and id this returns,+ and a relationships check can point at a subject you can only read. A subject whose time_column is set can+ take a lookback_hours window. A project can have thousands of subjects, so pass search (a name fragment,+ such as 'events') and subject_type to find one. Results are sorted by subject_type, then name, and come back+ limit at a time (20 by default). When a page has limit rows, call again with offset increased by limit. Set+ include_columns to false to browse ids and names without column schemas.+ param_overrides:+ limit:+ default: 20diff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.jsonindex 806b3e6266a4..757751ff955d 100644--- a/services/mcp/schema/generated-tool-definitions.json+++ b/services/mcp/schema/generated-tool-definitions.json@@ -4568,7 +4568,7 @@}},"data-quality-subjects": {- "description": "Every table, view, metric and PostHog table in this project you can read, with each one's columns and whether you can put a check on it. Start here: a check names its subject by the subject_type and id this returns, and a relationships check can point at a subject you can only read. A subject whose time_column is set can take a lookback_hours window.",+ "description": "Tables, views, metrics and PostHog tables in this project you can read, with each one's columns and whether you can put a check on it. Start here: a check names its subject by the subject_type and id this returns, and a relationships check can point at a subject you can only read. A subject whose time_column is set can take a lookback_hours window. A project can have thousands of subjects, so pass search (a name fragment, such as 'events') and subject_type to find one. Results are sorted by subject_type, then name, and come back limit at a time (20 by default). When a page has limit rows, call again with offset increased by limit. Set include_columns to false to browse ids and names without column schemas.","category": "Data quality","feature": "data_quality","summary": "List what a data quality check can be authored on",diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.jsonindex f8b53cd0872d..48c065728450 100644--- a/services/mcp/schema/tool-definitions-all.json+++ b/services/mcp/schema/tool-definitions-all.json@@ -4583,7 +4583,7 @@}},"data-quality-subjects": {- "description": "Every table, view, metric and PostHog table in this project you can read, with each one's columns and whether you can put a check on it. Start here: a check names its subject by the subject_type and id this returns, and a relationships check can point at a subject you can only read. A subject whose time_column is set can take a lookback_hours window.",+ "description": "Tables, views, metrics and PostHog tables in this project you can read, with each one's columns and whether you can put a check on it. Start here: a check names its subject by the subject_type and id this returns, and a relationships check can point at a subject you can only read. A subject whose time_column is set can take a lookback_hours window. A project can have thousands of subjects, so pass search (a name fragment, such as 'events') and subject_type to find one. Results are sorted by subject_type, then name, and come back limit at a time (20 by default). When a page has limit rows, call again with offset increased by limit. Set include_columns to false to browse ids and names without column schemas.","category": "Data quality","feature": "data_quality","summary": "List what a data quality check can be authored on",diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex e63400918596..a876c9b6d7f6 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -130218,6 +130218,48 @@ export namespace Schemas {View: 'view',} as const;+ export type DataQualityChecksSubjectsListParams = {+ /**+ * Set to false to return an empty columns map for every subject. Use it to find a subject id without reading every column schema.+ */+ include_columns?: boolean;+ /**+ * Maximum number of subjects to return. Omit to return every match.+ * @minimum 1+ * @maximum 1000+ */+ limit?: number;+ /**+ * Number of matching subjects to skip. Subjects are sorted by subject_type, then name. When a response has limit rows, call again with offset increased by limit to read the next page.+ * @minimum 0+ */+ offset?: number;+ /**+ * Return only subjects whose name or display name contains this text, ignoring case.+ */+ search?: string;+ /**+ * Return only subjects of this kind: 'table', 'view', 'metric', or 'posthog_table'.+ *+ * * `table` - table+ * * `view` - view+ * * `metric` - metric+ * * `posthog_table` - posthog_table+ * @minLength 1+ */+ subject_type?: DataQualityChecksSubjectsListSubjectType;+ };++ export type DataQualityChecksSubjectsListSubjectType = typeof DataQualityChecksSubjectsListSubjectType[keyof typeof DataQualityChecksSubjectsListSubjectType];+++ export const DataQualityChecksSubjectsListSubjectType = {+ Table: 'table',+ View: 'view',+ Metric: 'metric',+ PosthogTable: 'posthog_table',+ } as const;+export type DataQualityRunsListParams = {/*** Number of results to return per page.diff --git a/services/mcp/src/generated/data_quality/api.ts b/services/mcp/src/generated/data_quality/api.tsindex 8968aef19443..08868550f5a5 100644--- a/services/mcp/src/generated/data_quality/api.ts+++ b/services/mcp/src/generated/data_quality/api.ts@@ -273,7 +273,7 @@ export const DataQualityChecksSchedulePartialUpdateBody = () => zod.describe("Which subject's schedule to change, and what to change about it.")/**- * Everything in this project you can author a check on, with each subject's columns.+ * Everything in this project you can author a check on, with each subject's columns. Filter by subject_type or search, and page with limit and offset.*/export const DataQualityChecksSubjectsListParams = () => zod.object({project_id: zod@@ -282,3 +282,41 @@ export const DataQualityChecksSubjectsListParams = () => zod.object({"Project ID of the project you're trying to access. To find the ID of the project, make a call to \/api\/projects\/."),})++export const dataQualityChecksSubjectsListQueryIncludeColumnsDefault = true+export const dataQualityChecksSubjectsListQueryLimitMax = 1000++export const dataQualityChecksSubjectsListQueryOffsetDefault = 0+export const dataQualityChecksSubjectsListQueryOffsetMin = 0++export const DataQualityChecksSubjectsListQueryParams = () => zod.object({+ include_columns: zod+ .boolean()+ .default(dataQualityChecksSubjectsListQueryIncludeColumnsDefault)+ .describe(+ 'Set to false to return an empty columns map for every subject. Use it to find a subject id without reading every column schema.'+ ),+ limit: zod+ .number()+ .min(1)+ .max(dataQualityChecksSubjectsListQueryLimitMax)+ .optional()+ .describe('Maximum number of subjects to return. Omit to return every match.'),+ offset: zod+ .number()+ .min(dataQualityChecksSubjectsListQueryOffsetMin)+ .default(dataQualityChecksSubjectsListQueryOffsetDefault)+ .describe(+ 'Number of matching subjects to skip. Subjects are sorted by subject_type, then name. When a response has limit rows, call again with offset increased by limit to read the next page.'+ ),+ search: zod+ .string()+ .optional()+ .describe('Return only subjects whose name or display name contains this text, ignoring case.'),+ subject_type: zod+ .enum(['table', 'view', 'metric', 'posthog_table'])+ .optional()+ .describe(+ "Return only subjects of this kind: 'table', 'view', 'metric', or 'posthog_table'.\n\n\* `table` - table\n\* `view` - view\n\* `metric` - metric\n\* `posthog_table` - posthog_table"+ ),+})diff --git a/services/mcp/src/tools/generated/data_quality.ts b/services/mcp/src/tools/generated/data_quality.tsindex 33e7410e52bf..04bb0139979b 100644--- a/services/mcp/src/tools/generated/data_quality.ts+++ b/services/mcp/src/tools/generated/data_quality.ts@@ -255,7 +255,12 @@ const dataQualityCheckUpdate = (): ToolBase<},})-const DataQualitySubjectsSchema = () => z.object({})+const DataQualitySubjectsSchema = () => {+ const DataQualityChecksSubjectsListQueryParams = orvalSchemas.DataQualityChecksSubjectsListQueryParams()+ return DataQualityChecksSubjectsListQueryParams.extend({+ limit: DataQualityChecksSubjectsListQueryParams.shape['limit'].default(20).optional(),+ })+}const dataQualitySubjects = (): ToolBase<ReturnType<typeof DataQualitySubjectsSchema>,@@ -263,11 +268,18 @@ const dataQualitySubjects = (): ToolBase<> => ({name: 'data-quality-subjects',schema: DataQualitySubjectsSchema(),- handler: async (context: Context, _params: z.infer<ReturnType<typeof DataQualitySubjectsSchema>>) => {+ handler: async (context: Context, params: z.infer<ReturnType<typeof DataQualitySubjectsSchema>>) => {const projectId = await context.stateManager.getProjectId()const result = await context.api.request<Schemas.DataQualitySubject[]>({method: 'GET',path: `/api/projects/${encodeURIComponent(String(projectId))}/data_quality_checks/subjects/`,+ query: {+ include_columns: params.include_columns,+ limit: params.limit,+ offset: params.offset,+ search: params.search,+ subject_type: params.subject_type,+ },})return result},diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/data-quality-subjects.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/data-quality-subjects.jsonindex 7b7ea9186078..2daad066ef14 100644--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/data-quality-subjects.json+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/data-quality-subjects.json@@ -1,5 +1,33 @@{"$schema": "https://json-schema.org/draft/2020-12/schema",- "properties": {},+ "properties": {+ "include_columns": {+ "default": true,+ "description": "Set to false to return an empty columns map for every subject. Use it to find a subject id without reading every column schema.",+ "type": "boolean"+ },+ "limit": {+ "default": 20,+ "description": "Maximum number of subjects to return. Omit to return every match.",+ "maximum": 1000,+ "minimum": 1,+ "type": "number"+ },+ "offset": {+ "default": 0,+ "description": "Number of matching subjects to skip. Subjects are sorted by subject_type, then name. When a response has limit rows, call again with offset increased by limit to read the next page.",+ "minimum": 0,+ "type": "number"+ },+ "search": {+ "description": "Return only subjects whose name or display name contains this text, ignoring case.",+ "type": "string"+ },+ "subject_type": {+ "description": "Return only subjects of this kind: 'table', 'view', 'metric', or 'posthog_table'.\n\n* `table` - table\n* `view` - view\n* `metric` - metric\n* `posthog_table` - posthog_table",+ "enum": ["table", "view", "metric", "posthog_table"],+ "type": "string"+ }+ },"type": "object"}