Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
feat(datasource): persist source created time; Feishu uses content edit time for source_updated_at (#3083)
小妍儿 ✨061df8efdc25
diff --git a/internal/application/service/datasource_service.go b/internal/application/service/datasource_service.goindex 9af1e47e4..61ec2d279 100644--- a/internal/application/service/datasource_service.go+++ b/internal/application/service/datasource_service.go@@ -1257,6 +1257,9 @@ func (s *DataSourceService) ingestItem(ctx context.Context, ds *types.DataSourceif !item.UpdatedAt.IsZero() {metadata["source_updated_at"] = item.UpdatedAt.UTC().Format(time.RFC3339)}+ if !item.CreatedAt.IsZero() {+ metadata["source_created_at"] = item.CreatedAt.UTC().Format(time.RFC3339)+ }for k, v := range item.Metadata {metadata[k] = v}diff --git a/internal/application/service/datasource_service_test.go b/internal/application/service/datasource_service_test.goindex 445607c8c..4a64723a6 100644--- a/internal/application/service/datasource_service_test.go+++ b/internal/application/service/datasource_service_test.go@@ -511,14 +511,17 @@ func TestIngestItem_PersistsSourceUpdatedAt(t *testing.T) {repo := &deletionLookupKnowledgeRepo{}ks := &sweepFakeKS{repo: repo, createURLKnowledge: &types.Knowledge{ID: "url-knowledge-1"}}svc := &DataSourceService{knowledgeService: ks}+ created := time.Date(2021, 1, 2, 3, 4, 5, 0, time.UTC)_, err := svc.ingestItem(context.Background(), ds, &types.FetchedItem{ExternalID: "url:1",URL: "https://example.com/doc",UpdatedAt: edited,+ CreatedAt: created,}, nil)require.NoError(t, err)require.Len(t, repo.metadataUpdates, 1)assert.Equal(t, "2023-04-04T22:07:08Z", repo.metadataUpdates[0]["source_updated_at"])+ assert.Equal(t, "2021-01-02T03:04:05Z", repo.metadataUpdates[0]["source_created_at"])repo = &deletionLookupKnowledgeRepo{}ks = &sweepFakeKS{repo: repo, createURLKnowledge: &types.Knowledge{ID: "url-knowledge-2"}}@@ -531,6 +534,8 @@ func TestIngestItem_PersistsSourceUpdatedAt(t *testing.T) {require.Len(t, repo.metadataUpdates, 1)_, present := repo.metadataUpdates[0]["source_updated_at"]assert.False(t, present)+ _, present = repo.metadataUpdates[0]["source_created_at"]+ assert.False(t, present)}func TestProcessSync_SyncDeletionsDeletesMatchingKnowledge(t *testing.T) {diff --git a/internal/datasource/connector/feishu/core/shared.go b/internal/datasource/connector/feishu/core/shared.goindex 5900ecf09..fb6cbb00f 100644--- a/internal/datasource/connector/feishu/core/shared.go+++ b/internal/datasource/connector/feishu/core/shared.go@@ -290,11 +290,13 @@ type DocxFetchInput struct {// WeKnora external_id: wiki=node.NodeToken, drive=file.TokenDocToken string// Feishu docx document token- ObjToken string- Title string- URL string- ResourceID string- EditTime time.Time+ ObjToken string+ Title string+ URL string+ ResourceID string+ EditTime time.Time+ // CreateTime is the document creation time in Feishu; zero when unknown.+ CreateTime time.TimeBaseMeta map[string]stringMultimodalEnabled bool}@@ -364,6 +366,7 @@ func FetchDocxWithBlocks(ctx context.Context, client *Client, in DocxFetchInput)FileName: SanitizeFileName(in.Title) + ".md",URL: in.URL,UpdatedAt: in.EditTime,+ CreatedAt: in.CreateTime,SourceResourceID: in.ResourceID,Metadata: in.BaseMeta,ReplacesSubtree: true, // sweep stale attachment sub-items on re-sync@@ -414,6 +417,7 @@ func FetchDocxWithBlocks(ctx context.Context, client *Client, in DocxFetchInput)FileName: SanitizeFileName(a.Name),URL: in.URL,UpdatedAt: in.EditTime,+ CreatedAt: in.CreateTime,SourceResourceID: in.ResourceID,Metadata: childMeta(),})@@ -463,6 +467,7 @@ func FetchDocxWithBlocks(ctx context.Context, client *Client, in DocxFetchInput)FileName: "image-" + b.Image.Token + ext,URL: in.URL,UpdatedAt: in.EditTime,+ CreatedAt: in.CreateTime,SourceResourceID: in.ResourceID,Metadata: imgMeta(),})@@ -496,6 +501,7 @@ func exportDocxFallback(ctx context.Context, client *Client, in DocxFetchInput)FileName: fileName,URL: in.URL,UpdatedAt: in.EditTime,+ CreatedAt: in.CreateTime,SourceResourceID: in.ResourceID,Metadata: in.BaseMeta,}, nildiff --git a/internal/datasource/connector/feishu/drive/connector.go b/internal/datasource/connector/feishu/drive/connector.goindex 1e2d24327..41acdb5a1 100644--- a/internal/datasource/connector/feishu/drive/connector.go+++ b/internal/datasource/connector/feishu/drive/connector.go@@ -363,6 +363,7 @@ func fetchDriveFileContent(}editTime := core.ParseFeishuTimestamp(file.ModifiedTime)+ createTime := core.ParseFeishuTimestamp(file.CreatedTime)// Channel marks the knowledge "source" label. Drive uses its own channel// (feishu_drive / lark_drive) so Drive docs show "飞书云盘" / "Lark 云盘"// distinct from the wiki connector's "飞书".@@ -387,6 +388,7 @@ func fetchDriveFileContent(URL: file.URL,ResourceID: resourceID,EditTime: editTime,+ CreateTime: createTime,BaseMeta: baseMeta,MultimodalEnabled: multimodalEnabled,})@@ -412,6 +414,7 @@ func fetchDriveFileContent(FileName: fileName,URL: file.URL,UpdatedAt: editTime,+ CreatedAt: createTime,SourceResourceID: resourceID,Metadata: baseMeta,}}, nil@@ -435,6 +438,7 @@ func fetchDriveFileContent(FileName: fileName,URL: file.URL,UpdatedAt: editTime,+ CreatedAt: createTime,SourceResourceID: resourceID,Metadata: baseMeta,}}, nildiff --git a/internal/datasource/connector/feishu/wiki/connector.go b/internal/datasource/connector/feishu/wiki/connector.goindex 1fc359444..761c2705b 100644--- a/internal/datasource/connector/feishu/wiki/connector.go+++ b/internal/datasource/connector/feishu/wiki/connector.go@@ -243,8 +243,9 @@ func (o wikiOps) Title(n core.WikiNode) string { return n.Title }func (o wikiOps) ObjType(n core.WikiNode) string { return n.ObjType }// EditTime is the change-detection timestamp: ObjEditTime (document content)-// with a NodeEditTime fallback for nodes that lack obj_edit_time. This drives-// the cursor comparison, NOT FetchedItem.UpdatedAt (which uses NodeEditTime).+// with a NodeEditTime fallback for nodes that lack obj_edit_time. It drives the+// cursor comparison and, parsed, FetchedItem.UpdatedAt (see contentEditTime),+// so the persisted source_updated_at tracks content edits, not node moves.func (o wikiOps) EditTime(n core.WikiNode) string {if n.ObjEditTime != "" {return n.ObjEditTime@@ -323,7 +324,8 @@ func fetchNodeContent(ctx context.Context, client *core.Client, node core.WikiNoreturn nil, nil}- editTime := core.ParseFeishuTimestamp(node.NodeEditTime)+ editTime := contentEditTime(node)+ createTime := contentCreateTime(node)baseMeta := map[string]string{"obj_token": node.ObjToken,"obj_type": node.ObjType,@@ -343,6 +345,7 @@ func fetchNodeContent(ctx context.Context, client *core.Client, node core.WikiNoURL: region.WikiURL(node.NodeToken),ResourceID: resourceID,EditTime: editTime,+ CreateTime: createTime,BaseMeta: baseMeta,MultimodalEnabled: multimodalEnabled,})@@ -351,12 +354,14 @@ func fetchNodeContent(ctx context.Context, client *core.Client, node core.WikiNoif err != nil {return nil, err}+ item.CreatedAt = createTimereturn []*types.FetchedItem{item}, nilcase "file":item, err := fetchDriveFile(ctx, client, node, resourceID, editTime, baseMeta, region)if err != nil {return nil, err}+ item.CreatedAt = createTimereturn []*types.FetchedItem{item}, nildefault:return nil, nil@@ -422,6 +427,26 @@ func fetchDriveFile(ctx context.Context, client *core.Client, node core.WikiNode}, nil}+// contentEditTime is the document's last content edit time (obj_edit_time),+// falling back to the node attribute edit time when Feishu omits it. It is+// what ingestion persists as source_updated_at; NodeEditTime alone would move+// on a rename or a move in the tree without any content change.+func contentEditTime(node core.WikiNode) time.Time {+ if t := core.ParseFeishuTimestamp(node.ObjEditTime); !t.IsZero() {+ return t+ }+ return core.ParseFeishuTimestamp(node.NodeEditTime)+}++// contentCreateTime is the document creation time (obj_create_time), falling+// back to the node creation time. Persisted as source_created_at.+func contentCreateTime(node core.WikiNode) time.Time {+ if t := core.ParseFeishuTimestamp(node.ObjCreateTime); !t.IsZero() {+ return t+ }+ return core.ParseFeishuTimestamp(node.NodeCreateTime)+}+// --- Helper functions ---func makeWikiNodeResourceID(spaceID, nodeToken string) string {diff --git a/internal/datasource/connector/feishu/wiki/connector_test.go b/internal/datasource/connector/feishu/wiki/connector_test.goindex 7d52a118c..305624a9e 100644--- a/internal/datasource/connector/feishu/wiki/connector_test.go+++ b/internal/datasource/connector/feishu/wiki/connector_test.go@@ -579,11 +579,14 @@ func TestConnectorResolveResourceAncestors(t *testing.T) {func TestFetchAll_DocxNode(t *testing.T) {nodes := []core.WikiNode{{- NodeToken: "nt1",- ObjToken: "obj-docx-1",- ObjType: "docx",- Title: "My Document",- NodeEditTime: "1711468800",+ NodeToken: "nt1",+ ObjToken: "obj-docx-1",+ ObjType: "docx",+ Title: "My Document",+ ObjCreateTime: "1700000000",+ ObjEditTime: "1711000000",+ NodeCreateTime: "1700000001",+ NodeEditTime: "1711468800",}}ts, cfg := fakeFeishu(nodes)defer ts.Close()@@ -617,6 +620,27 @@ func TestFetchAll_DocxNode(t *testing.T) {if item.Metadata["channel"] != types.ChannelFeishu {t.Errorf("Metadata[channel] = %q", item.Metadata["channel"])}+ // Source timestamps come from the document (obj_*), not the wiki node+ // attributes, so a rename or move does not look like a content edit.+ if got := item.UpdatedAt.Unix(); got != 1711000000 {+ t.Errorf("UpdatedAt = %d, want obj_edit_time 1711000000", got)+ }+ if got := item.CreatedAt.Unix(); got != 1700000000 {+ t.Errorf("CreatedAt = %d, want obj_create_time 1700000000", got)+ }+}++func TestContentTimes_FallBackToNodeTimes(t *testing.T) {+ n := core.WikiNode{NodeCreateTime: "1700000001", NodeEditTime: "1711468800"}+ if got := contentEditTime(n).Unix(); got != 1711468800 {+ t.Errorf("contentEditTime fallback = %d, want node_edit_time", got)+ }+ if got := contentCreateTime(n).Unix(); got != 1700000001 {+ t.Errorf("contentCreateTime fallback = %d, want node_create_time", got)+ }+ if !contentEditTime(core.WikiNode{}).IsZero() || !contentCreateTime(core.WikiNode{}).IsZero() {+ t.Errorf("missing times must stay zero, not epoch")+ }}func TestFetchAll_SheetNode(t *testing.T) {diff --git a/internal/datasource/connector/gitlab/connector.go b/internal/datasource/connector/gitlab/connector.goindex ec512e719..b82419541 100644--- a/internal/datasource/connector/gitlab/connector.go+++ b/internal/datasource/connector/gitlab/connector.go@@ -404,7 +404,24 @@ func (c *Connector) item(ctx context.Context, p *project, ref, file string) (typreturn types.FetchedItem{}, err}id := fmt.Sprintf("gitlab:%s:%d:%s:%s", c.canonicalBase, p.ID, ref, file)- return types.FetchedItem{ExternalID: id, Title: p.PathWithNamespace + "/" + file, FileName: knowledgeRelativePath(p.Name, ref, file), Content: body, ContentType: "text/plain", UpdatedAt: time.Now().UTC(), SourceResourceID: fmt.Sprint(p.ID), Metadata: map[string]string{"channel": types.ConnectorTypeGitLab, "source_type": "gitlab", "gitlab_project_id": fmt.Sprint(p.ID), "gitlab_ref": ref, "gitlab_path": file, "gitlab_url": p.WebURL + "/-/blob/" + ref + "/" + file}}, nil+ // UpdatedAt is intentionally left unset: the file's last commit time is not+ // fetched here, and a fetch timestamp would be a fabricated source time.+ return types.FetchedItem{+ ExternalID: id,+ Title: p.PathWithNamespace + "/" + file,+ FileName: knowledgeRelativePath(p.Name, ref, file),+ Content: body,+ ContentType: "text/plain",+ SourceResourceID: fmt.Sprint(p.ID),+ Metadata: map[string]string{+ "channel": types.ConnectorTypeGitLab,+ "source_type": "gitlab",+ "gitlab_project_id": fmt.Sprint(p.ID),+ "gitlab_ref": ref,+ "gitlab_path": file,+ "gitlab_url": p.WebURL + "/-/blob/" + ref + "/" + file,+ },+ }, nil}// knowledgeRelativePath maps a repository file to the KB folder convention:diff --git a/internal/types/datasource.go b/internal/types/datasource.goindex a1ba855cd..5130b931e 100644--- a/internal/types/datasource.go+++ b/internal/types/datasource.go@@ -332,6 +332,9 @@ type FetchedItem struct {// When last modified in external systemUpdatedAt time.Time `json:"updated_at"`+ // When created in external system. Zero when the source does not expose it.+ CreatedAt time.Time `json:"created_at"`+// Additional metadata to preserveMetadata map[string]string `json:"metadata"`