Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(messages): stop dropping doc_metadata and friends from GET /messages citations (#41861)
tim-gregory-pt90b8dacd12e0
diff --git a/api/fields/message_fields.py b/api/fields/message_fields.pyindex 79bfde8f77b..8bb91dee3d7 100644--- a/api/fields/message_fields.py+++ b/api/fields/message_fields.py@@ -41,6 +41,11 @@ class RetrieverResource(ResponseModel):content: str | None = Nonesummary: str | None = Nonecreated_at: Int64 | None = None+ retriever_from: str | None = None+ page: int | None = None+ doc_metadata: dict[str, JSONValueType] | None = None+ title: str | None = None+ files: list[dict[str, JSONValueType]] | None = None@field_validator("created_at", mode="before")@classmethoddiff --git a/api/openapi/markdown/console-openapi.md b/api/openapi/markdown/console-openapi.mdindex 2f4d95bc57d..19ac05f600f 100644--- a/api/openapi/markdown/console-openapi.md+++ b/api/openapi/markdown/console-openapi.md@@ -22868,17 +22868,22 @@ Resource types understood by access policies.| data_source_type | string | | No || dataset_id | string | | No || dataset_name | string | | No |+| doc_metadata | object | | No || document_id | string | | No || document_name | string | | No |+| files | [ object ] | | No || hit_count | integer | | No || id | string (uuid) | | No || index_node_hash | string | | No || message_id | string (uuid) | | No |+| page | integer | | No || position | integer | | Yes |+| retriever_from | string | | No || score | number | | No || segment_id | string | | No || segment_position | integer | | No || summary | string | | No |+| title | string | | No || word_count | integer | | No |#### RoleBindingsResponsediff --git a/api/openapi/markdown/service-openapi.md b/api/openapi/markdown/service-openapi.mdindex a30ee05de6f..c2c93229f62 100644--- a/api/openapi/markdown/service-openapi.md+++ b/api/openapi/markdown/service-openapi.md@@ -3946,17 +3946,22 @@ Model class for provider with models response.| data_source_type | string | | No || dataset_id | string | | No || dataset_name | string | | No |+| doc_metadata | object | | No || document_id | string | | No || document_name | string | | No |+| files | [ object ] | | No || hit_count | integer | | No || id | string (uuid) | | No || index_node_hash | string | | No || message_id | string (uuid) | | No |+| page | integer | | No || position | integer | | Yes |+| retriever_from | string | | No || score | number | | No || segment_id | string | | No || segment_position | integer | | No || summary | string | | No |+| title | string | | No || word_count | integer | | No |#### Rulediff --git a/api/openapi/markdown/web-openapi.md b/api/openapi/markdown/web-openapi.mdindex 1f6d7a8e723..08bc2afc817 100644--- a/api/openapi/markdown/web-openapi.md+++ b/api/openapi/markdown/web-openapi.md@@ -1442,17 +1442,22 @@ Form input definition.| data_source_type | string | | No || dataset_id | string | | No || dataset_name | string | | No |+| doc_metadata | object | | No || document_id | string | | No || document_name | string | | No |+| files | [ object ] | | No || hit_count | integer | | No || id | string (uuid) | | No || index_node_hash | string | | No || message_id | string (uuid) | | No |+| page | integer | | No || position | integer | | Yes |+| retriever_from | string | | No || score | number | | No || segment_id | string | | No || segment_position | integer | | No || summary | string | | No |+| title | string | | No || word_count | integer | | No |#### SSOProtocoldiff --git a/api/tests/unit_tests/fields/test_message_fields.py b/api/tests/unit_tests/fields/test_message_fields.pyindex 2180e0d896c..6ae1c1ce90a 100644--- a/api/tests/unit_tests/fields/test_message_fields.py+++ b/api/tests/unit_tests/fields/test_message_fields.py@@ -1,6 +1,24 @@from decimal import Decimal-from fields.message_fields import ExploreMessageListItem, MessageListItem, WebMessageListItem+from fields.message_fields import ExploreMessageListItem, MessageListItem, RetrieverResource, WebMessageListItem+++def _retrieved_video_resource():+ # Shaped like a real stored retriever_resources entry (core/rag/entities/citation_metadata.py's+ # RetrievalSourceMetadata, the write-side model) for a video citation with a start/end offset.+ return {+ "position": 1,+ "dataset_id": "af57b949-2259-4fe4-b58a-0117fd3a6b92",+ "document_id": "9196b0c1-898e-4e8e-9d78-18cb07896b89",+ "document_name": "training.mp4",+ "retriever_from": "workflow",+ "score": 0.86,+ "content": "transcript excerpt",+ "page": None,+ "title": "training.mp4",+ "files": None,+ "doc_metadata": {"start_seconds": 1036.0, "end_seconds": 1054.6, "document_name": "training.mp4"},+ }def _base_kwargs():@@ -66,3 +84,31 @@ class TestExploreMessageListItem:assert payload["message_tokens"] == 7assert payload["answer_tokens"] == 11assert payload["total_tokens"] == 18+++class TestRetrieverResourceDocMetadata:+ def test_doc_metadata_survives_validation_from_a_stored_dict(self):+ # GET /messages re-validates the raw dicts stored in Message.retriever_resources+ # (JSON-backed, written from RetrievalSourceMetadata) against RetrieverResource.+ # Before this field existed, ResponseModel's extra="ignore" silently dropped+ # doc_metadata here -- the video player on a reloaded conversation had nothing+ # to read start_seconds/end_seconds from, even though the data was never lost+ # in storage.+ resource = RetrieverResource.model_validate(_retrieved_video_resource())++ payload = resource.model_dump(mode="json")++ assert payload["doc_metadata"] == {+ "start_seconds": 1036.0,+ "end_seconds": 1054.6,+ "document_name": "training.mp4",+ }+ assert payload["title"] == "training.mp4"+ assert payload["retriever_from"] == "workflow"++ def test_message_list_item_propagates_doc_metadata_through_retriever_resources(self):+ payload = MessageListItem(+ **{**_base_kwargs(), "retriever_resources": [_retrieved_video_resource()]},+ ).model_dump(mode="json")++ assert payload["retriever_resources"][0]["doc_metadata"]["start_seconds"] == 1036.0diff --git a/packages/contracts/generated/api/console/installed-apps/types.gen.ts b/packages/contracts/generated/api/console/installed-apps/types.gen.tsindex 37d345e93bc..7377574e072 100644--- a/packages/contracts/generated/api/console/installed-apps/types.gen.ts+++ b/packages/contracts/generated/api/console/installed-apps/types.gen.ts@@ -633,17 +633,26 @@ export type RetrieverResource = {data_source_type?: string | nulldataset_id?: string | nulldataset_name?: string | null+ doc_metadata?: {+ [key: string]: JsonValueType+ } | nulldocument_id?: string | nulldocument_name?: string | null+ files?: Array<{+ [key: string]: JsonValueType+ }> | nullhit_count?: number | nullid?: stringindex_node_hash?: string | nullmessage_id?: string+ page?: number | nullposition: number+ retriever_from?: string | nullscore?: number | nullsegment_id?: string | nullsegment_position?: number | nullsummary?: string | null+ title?: string | nullword_count?: number | null}diff --git a/packages/contracts/generated/api/console/installed-apps/zod.gen.ts b/packages/contracts/generated/api/console/installed-apps/zod.gen.tsindex 93bb6ab4e52..c61a5575182 100644--- a/packages/contracts/generated/api/console/installed-apps/zod.gen.ts+++ b/packages/contracts/generated/api/console/installed-apps/zod.gen.ts@@ -689,17 +689,22 @@ export const zRetrieverResource = z.object({data_source_type: z.string().nullish(),dataset_id: z.uuid().nullish(),dataset_name: z.string().nullish(),+ doc_metadata: z.record(z.string(), zJsonValueType).nullish(),document_id: z.uuid().nullish(),document_name: z.string().nullish(),+ files: z.array(z.record(z.string(), zJsonValueType)).nullish(),hit_count: z.int().nullish(),id: z.uuid().optional(),index_node_hash: z.string().nullish(),message_id: z.uuid().optional(),+ page: z.int().nullish(),position: z.int(),+ retriever_from: z.string().nullish(),score: z.number().nullish(),segment_id: z.uuid().nullish(),segment_position: z.int().nullish(),summary: z.string().nullish(),+ title: z.string().nullish(),word_count: z.int().nullish(),})diff --git a/packages/contracts/generated/api/service/types.gen.ts b/packages/contracts/generated/api/service/types.gen.tsindex 31940741fcd..b6224965fd0 100644--- a/packages/contracts/generated/api/service/types.gen.ts+++ b/packages/contracts/generated/api/service/types.gen.ts@@ -1857,17 +1857,26 @@ export type RetrieverResource = {data_source_type?: string | nulldataset_id?: string | nulldataset_name?: string | null+ doc_metadata?: {+ [key: string]: JsonValueType+ } | nulldocument_id?: string | nulldocument_name?: string | null+ files?: Array<{+ [key: string]: JsonValueType+ }> | nullhit_count?: number | nullid?: stringindex_node_hash?: string | nullmessage_id?: string+ page?: number | nullposition: number+ retriever_from?: string | nullscore?: number | nullsegment_id?: string | nullsegment_position?: number | nullsummary?: string | null+ title?: string | nullword_count?: number | null}diff --git a/packages/contracts/generated/api/service/zod.gen.ts b/packages/contracts/generated/api/service/zod.gen.tsindex 24cfe744072..fcf453513ca 100644--- a/packages/contracts/generated/api/service/zod.gen.ts+++ b/packages/contracts/generated/api/service/zod.gen.ts@@ -1768,17 +1768,22 @@ export const zRetrieverResource = z.object({data_source_type: z.string().nullish(),dataset_id: z.uuid().nullish(),dataset_name: z.string().nullish(),+ doc_metadata: z.record(z.string(), zJsonValueType).nullish(),document_id: z.uuid().nullish(),document_name: z.string().nullish(),+ files: z.array(z.record(z.string(), zJsonValueType)).nullish(),hit_count: z.int().nullish(),id: z.uuid().optional(),index_node_hash: z.string().nullish(),message_id: z.uuid().optional(),+ page: z.int().nullish(),position: z.int(),+ retriever_from: z.string().nullish(),score: z.number().nullish(),segment_id: z.uuid().nullish(),segment_position: z.int().nullish(),summary: z.string().nullish(),+ title: z.string().nullish(),word_count: z.int().nullish(),})diff --git a/packages/contracts/generated/api/web/types.gen.ts b/packages/contracts/generated/api/web/types.gen.tsindex 09672af5c97..e46a9bc0593 100644--- a/packages/contracts/generated/api/web/types.gen.ts+++ b/packages/contracts/generated/api/web/types.gen.ts@@ -724,17 +724,26 @@ export type RetrieverResource = {data_source_type?: string | nulldataset_id?: string | nulldataset_name?: string | null+ doc_metadata?: {+ [key: string]: JsonValueType+ } | nulldocument_id?: string | nulldocument_name?: string | null+ files?: Array<{+ [key: string]: JsonValueType+ }> | nullhit_count?: number | nullid?: stringindex_node_hash?: string | nullmessage_id?: string+ page?: number | nullposition: number+ retriever_from?: string | nullscore?: number | nullsegment_id?: string | nullsegment_position?: number | nullsummary?: string | null+ title?: string | nullword_count?: number | null}diff --git a/packages/contracts/generated/api/web/zod.gen.ts b/packages/contracts/generated/api/web/zod.gen.tsindex d6c3a3a31d9..4cb59ab39d1 100644--- a/packages/contracts/generated/api/web/zod.gen.ts+++ b/packages/contracts/generated/api/web/zod.gen.ts@@ -497,17 +497,22 @@ export const zRetrieverResource = z.object({data_source_type: z.string().nullish(),dataset_id: z.uuid().nullish(),dataset_name: z.string().nullish(),+ doc_metadata: z.record(z.string(), zJsonValueType).nullish(),document_id: z.uuid().nullish(),document_name: z.string().nullish(),+ files: z.array(z.record(z.string(), zJsonValueType)).nullish(),hit_count: z.int().nullish(),id: z.uuid().optional(),index_node_hash: z.string().nullish(),message_id: z.uuid().optional(),+ page: z.int().nullish(),position: z.int(),+ retriever_from: z.string().nullish(),score: z.number().nullish(),segment_id: z.uuid().nullish(),segment_position: z.int().nullish(),summary: z.string().nullish(),+ title: z.string().nullish(),word_count: z.int().nullish(),})