Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(messages): stop dropping doc_metadata and friends from GET /messages citations (#41861)

tim-gregory-pt90b8dacd12e0
diff --git a/api/fields/message_fields.py b/api/fields/message_fields.py
index 79bfde8f77b..8bb91dee3d7 100644
--- a/api/fields/message_fields.py
+++ b/api/fields/message_fields.py
@@ -41,6 +41,11 @@ class RetrieverResource(ResponseModel):
content: str | None = None
summary: str | None = None
created_at: Int64 | None = None
+ retriever_from: str | None = None
+ page: int | None = None
+ doc_metadata: dict[str, JSONValueType] | None = None
+ title: str | None = None
+ files: list[dict[str, JSONValueType]] | None = None
@field_validator("created_at", mode="before")
@classmethod
diff --git a/api/openapi/markdown/console-openapi.md b/api/openapi/markdown/console-openapi.md
index 2f4d95bc57d..19ac05f600f 100644
--- a/api/openapi/markdown/console-openapi.md
+++ b/api/openapi/markdown/console-openapi.md
@@ -22868,17 +22868,22 @@ Resource types understood by access policies.
| data_source_type | string | | No |
| dataset_id | string | | No |
| dataset_name | string | | No |
+| doc_metadata | object | | No |
| document_id | string | | No |
| document_name | string | | No |
+| files | [ object ] | | No |
| hit_count | integer | | No |
| id | string (uuid) | | No |
| index_node_hash | string | | No |
| message_id | string (uuid) | | No |
+| page | integer | | No |
| position | integer | | Yes |
+| retriever_from | string | | No |
| score | number | | No |
| segment_id | string | | No |
| segment_position | integer | | No |
| summary | string | | No |
+| title | string | | No |
| word_count | integer | | No |
#### RoleBindingsResponse
diff --git a/api/openapi/markdown/service-openapi.md b/api/openapi/markdown/service-openapi.md
index a30ee05de6f..c2c93229f62 100644
--- a/api/openapi/markdown/service-openapi.md
+++ b/api/openapi/markdown/service-openapi.md
@@ -3946,17 +3946,22 @@ Model class for provider with models response.
| data_source_type | string | | No |
| dataset_id | string | | No |
| dataset_name | string | | No |
+| doc_metadata | object | | No |
| document_id | string | | No |
| document_name | string | | No |
+| files | [ object ] | | No |
| hit_count | integer | | No |
| id | string (uuid) | | No |
| index_node_hash | string | | No |
| message_id | string (uuid) | | No |
+| page | integer | | No |
| position | integer | | Yes |
+| retriever_from | string | | No |
| score | number | | No |
| segment_id | string | | No |
| segment_position | integer | | No |
| summary | string | | No |
+| title | string | | No |
| word_count | integer | | No |
#### Rule
diff --git a/api/openapi/markdown/web-openapi.md b/api/openapi/markdown/web-openapi.md
index 1f6d7a8e723..08bc2afc817 100644
--- a/api/openapi/markdown/web-openapi.md
+++ b/api/openapi/markdown/web-openapi.md
@@ -1442,17 +1442,22 @@ Form input definition.
| data_source_type | string | | No |
| dataset_id | string | | No |
| dataset_name | string | | No |
+| doc_metadata | object | | No |
| document_id | string | | No |
| document_name | string | | No |
+| files | [ object ] | | No |
| hit_count | integer | | No |
| id | string (uuid) | | No |
| index_node_hash | string | | No |
| message_id | string (uuid) | | No |
+| page | integer | | No |
| position | integer | | Yes |
+| retriever_from | string | | No |
| score | number | | No |
| segment_id | string | | No |
| segment_position | integer | | No |
| summary | string | | No |
+| title | string | | No |
| word_count | integer | | No |
#### SSOProtocol
diff --git a/api/tests/unit_tests/fields/test_message_fields.py b/api/tests/unit_tests/fields/test_message_fields.py
index 2180e0d896c..6ae1c1ce90a 100644
--- a/api/tests/unit_tests/fields/test_message_fields.py
+++ b/api/tests/unit_tests/fields/test_message_fields.py
@@ -1,6 +1,24 @@
from decimal import Decimal
-from fields.message_fields import ExploreMessageListItem, MessageListItem, WebMessageListItem
+from fields.message_fields import ExploreMessageListItem, MessageListItem, RetrieverResource, WebMessageListItem
+
+
+def _retrieved_video_resource():
+ # Shaped like a real stored retriever_resources entry (core/rag/entities/citation_metadata.py's
+ # RetrievalSourceMetadata, the write-side model) for a video citation with a start/end offset.
+ return {
+ "position": 1,
+ "dataset_id": "af57b949-2259-4fe4-b58a-0117fd3a6b92",
+ "document_id": "9196b0c1-898e-4e8e-9d78-18cb07896b89",
+ "document_name": "training.mp4",
+ "retriever_from": "workflow",
+ "score": 0.86,
+ "content": "transcript excerpt",
+ "page": None,
+ "title": "training.mp4",
+ "files": None,
+ "doc_metadata": {"start_seconds": 1036.0, "end_seconds": 1054.6, "document_name": "training.mp4"},
+ }
def _base_kwargs():
@@ -66,3 +84,31 @@ class TestExploreMessageListItem:
assert payload["message_tokens"] == 7
assert payload["answer_tokens"] == 11
assert payload["total_tokens"] == 18
+
+
+class TestRetrieverResourceDocMetadata:
+ def test_doc_metadata_survives_validation_from_a_stored_dict(self):
+ # GET /messages re-validates the raw dicts stored in Message.retriever_resources
+ # (JSON-backed, written from RetrievalSourceMetadata) against RetrieverResource.
+ # Before this field existed, ResponseModel's extra="ignore" silently dropped
+ # doc_metadata here -- the video player on a reloaded conversation had nothing
+ # to read start_seconds/end_seconds from, even though the data was never lost
+ # in storage.
+ resource = RetrieverResource.model_validate(_retrieved_video_resource())
+
+ payload = resource.model_dump(mode="json")
+
+ assert payload["doc_metadata"] == {
+ "start_seconds": 1036.0,
+ "end_seconds": 1054.6,
+ "document_name": "training.mp4",
+ }
+ assert payload["title"] == "training.mp4"
+ assert payload["retriever_from"] == "workflow"
+
+ def test_message_list_item_propagates_doc_metadata_through_retriever_resources(self):
+ payload = MessageListItem(
+ **{**_base_kwargs(), "retriever_resources": [_retrieved_video_resource()]},
+ ).model_dump(mode="json")
+
+ assert payload["retriever_resources"][0]["doc_metadata"]["start_seconds"] == 1036.0
diff --git a/packages/contracts/generated/api/console/installed-apps/types.gen.ts b/packages/contracts/generated/api/console/installed-apps/types.gen.ts
index 37d345e93bc..7377574e072 100644
--- a/packages/contracts/generated/api/console/installed-apps/types.gen.ts
+++ b/packages/contracts/generated/api/console/installed-apps/types.gen.ts
@@ -633,17 +633,26 @@ export type RetrieverResource = {
data_source_type?: string | null
dataset_id?: string | null
dataset_name?: string | null
+ doc_metadata?: {
+ [key: string]: JsonValueType
+ } | null
document_id?: string | null
document_name?: string | null
+ files?: Array<{
+ [key: string]: JsonValueType
+ }> | null
hit_count?: number | null
id?: string
index_node_hash?: string | null
message_id?: string
+ page?: number | null
position: number
+ retriever_from?: string | null
score?: number | null
segment_id?: string | null
segment_position?: number | null
summary?: string | null
+ title?: string | null
word_count?: number | null
}
diff --git a/packages/contracts/generated/api/console/installed-apps/zod.gen.ts b/packages/contracts/generated/api/console/installed-apps/zod.gen.ts
index 93bb6ab4e52..c61a5575182 100644
--- a/packages/contracts/generated/api/console/installed-apps/zod.gen.ts
+++ b/packages/contracts/generated/api/console/installed-apps/zod.gen.ts
@@ -689,17 +689,22 @@ export const zRetrieverResource = z.object({
data_source_type: z.string().nullish(),
dataset_id: z.uuid().nullish(),
dataset_name: z.string().nullish(),
+ doc_metadata: z.record(z.string(), zJsonValueType).nullish(),
document_id: z.uuid().nullish(),
document_name: z.string().nullish(),
+ files: z.array(z.record(z.string(), zJsonValueType)).nullish(),
hit_count: z.int().nullish(),
id: z.uuid().optional(),
index_node_hash: z.string().nullish(),
message_id: z.uuid().optional(),
+ page: z.int().nullish(),
position: z.int(),
+ retriever_from: z.string().nullish(),
score: z.number().nullish(),
segment_id: z.uuid().nullish(),
segment_position: z.int().nullish(),
summary: z.string().nullish(),
+ title: z.string().nullish(),
word_count: z.int().nullish(),
})
diff --git a/packages/contracts/generated/api/service/types.gen.ts b/packages/contracts/generated/api/service/types.gen.ts
index 31940741fcd..b6224965fd0 100644
--- a/packages/contracts/generated/api/service/types.gen.ts
+++ b/packages/contracts/generated/api/service/types.gen.ts
@@ -1857,17 +1857,26 @@ export type RetrieverResource = {
data_source_type?: string | null
dataset_id?: string | null
dataset_name?: string | null
+ doc_metadata?: {
+ [key: string]: JsonValueType
+ } | null
document_id?: string | null
document_name?: string | null
+ files?: Array<{
+ [key: string]: JsonValueType
+ }> | null
hit_count?: number | null
id?: string
index_node_hash?: string | null
message_id?: string
+ page?: number | null
position: number
+ retriever_from?: string | null
score?: number | null
segment_id?: string | null
segment_position?: number | null
summary?: string | null
+ title?: string | null
word_count?: number | null
}
diff --git a/packages/contracts/generated/api/service/zod.gen.ts b/packages/contracts/generated/api/service/zod.gen.ts
index 24cfe744072..fcf453513ca 100644
--- a/packages/contracts/generated/api/service/zod.gen.ts
+++ b/packages/contracts/generated/api/service/zod.gen.ts
@@ -1768,17 +1768,22 @@ export const zRetrieverResource = z.object({
data_source_type: z.string().nullish(),
dataset_id: z.uuid().nullish(),
dataset_name: z.string().nullish(),
+ doc_metadata: z.record(z.string(), zJsonValueType).nullish(),
document_id: z.uuid().nullish(),
document_name: z.string().nullish(),
+ files: z.array(z.record(z.string(), zJsonValueType)).nullish(),
hit_count: z.int().nullish(),
id: z.uuid().optional(),
index_node_hash: z.string().nullish(),
message_id: z.uuid().optional(),
+ page: z.int().nullish(),
position: z.int(),
+ retriever_from: z.string().nullish(),
score: z.number().nullish(),
segment_id: z.uuid().nullish(),
segment_position: z.int().nullish(),
summary: z.string().nullish(),
+ title: z.string().nullish(),
word_count: z.int().nullish(),
})
diff --git a/packages/contracts/generated/api/web/types.gen.ts b/packages/contracts/generated/api/web/types.gen.ts
index 09672af5c97..e46a9bc0593 100644
--- a/packages/contracts/generated/api/web/types.gen.ts
+++ b/packages/contracts/generated/api/web/types.gen.ts
@@ -724,17 +724,26 @@ export type RetrieverResource = {
data_source_type?: string | null
dataset_id?: string | null
dataset_name?: string | null
+ doc_metadata?: {
+ [key: string]: JsonValueType
+ } | null
document_id?: string | null
document_name?: string | null
+ files?: Array<{
+ [key: string]: JsonValueType
+ }> | null
hit_count?: number | null
id?: string
index_node_hash?: string | null
message_id?: string
+ page?: number | null
position: number
+ retriever_from?: string | null
score?: number | null
segment_id?: string | null
segment_position?: number | null
summary?: string | null
+ title?: string | null
word_count?: number | null
}
diff --git a/packages/contracts/generated/api/web/zod.gen.ts b/packages/contracts/generated/api/web/zod.gen.ts
index d6c3a3a31d9..4cb59ab39d1 100644
--- a/packages/contracts/generated/api/web/zod.gen.ts
+++ b/packages/contracts/generated/api/web/zod.gen.ts
@@ -497,17 +497,22 @@ export const zRetrieverResource = z.object({
data_source_type: z.string().nullish(),
dataset_id: z.uuid().nullish(),
dataset_name: z.string().nullish(),
+ doc_metadata: z.record(z.string(), zJsonValueType).nullish(),
document_id: z.uuid().nullish(),
document_name: z.string().nullish(),
+ files: z.array(z.record(z.string(), zJsonValueType)).nullish(),
hit_count: z.int().nullish(),
id: z.uuid().optional(),
index_node_hash: z.string().nullish(),
message_id: z.uuid().optional(),
+ page: z.int().nullish(),
position: z.int(),
+ retriever_from: z.string().nullish(),
score: z.number().nullish(),
segment_id: z.uuid().nullish(),
segment_position: z.int().nullish(),
summary: z.string().nullish(),
+ title: z.string().nullish(),
word_count: z.int().nullish(),
})