Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

feat(datasource): persist source created time; Feishu uses content edit time for source_updated_at (#3083)

小妍儿 ✨061df8efdc25
diff --git a/internal/application/service/datasource_service.go b/internal/application/service/datasource_service.go
index 9af1e47e4..61ec2d279 100644
--- a/internal/application/service/datasource_service.go
+++ b/internal/application/service/datasource_service.go
@@ -1257,6 +1257,9 @@ func (s *DataSourceService) ingestItem(ctx context.Context, ds *types.DataSource
if !item.UpdatedAt.IsZero() {
metadata["source_updated_at"] = item.UpdatedAt.UTC().Format(time.RFC3339)
}
+ if !item.CreatedAt.IsZero() {
+ metadata["source_created_at"] = item.CreatedAt.UTC().Format(time.RFC3339)
+ }
for k, v := range item.Metadata {
metadata[k] = v
}
diff --git a/internal/application/service/datasource_service_test.go b/internal/application/service/datasource_service_test.go
index 445607c8c..4a64723a6 100644
--- a/internal/application/service/datasource_service_test.go
+++ b/internal/application/service/datasource_service_test.go
@@ -511,14 +511,17 @@ func TestIngestItem_PersistsSourceUpdatedAt(t *testing.T) {
repo := &deletionLookupKnowledgeRepo{}
ks := &sweepFakeKS{repo: repo, createURLKnowledge: &types.Knowledge{ID: "url-knowledge-1"}}
svc := &DataSourceService{knowledgeService: ks}
+ created := time.Date(2021, 1, 2, 3, 4, 5, 0, time.UTC)
_, err := svc.ingestItem(context.Background(), ds, &types.FetchedItem{
ExternalID: "url:1",
URL: "https://example.com/doc",
UpdatedAt: edited,
+ CreatedAt: created,
}, nil)
require.NoError(t, err)
require.Len(t, repo.metadataUpdates, 1)
assert.Equal(t, "2023-04-04T22:07:08Z", repo.metadataUpdates[0]["source_updated_at"])
+ assert.Equal(t, "2021-01-02T03:04:05Z", repo.metadataUpdates[0]["source_created_at"])
repo = &deletionLookupKnowledgeRepo{}
ks = &sweepFakeKS{repo: repo, createURLKnowledge: &types.Knowledge{ID: "url-knowledge-2"}}
@@ -531,6 +534,8 @@ func TestIngestItem_PersistsSourceUpdatedAt(t *testing.T) {
require.Len(t, repo.metadataUpdates, 1)
_, present := repo.metadataUpdates[0]["source_updated_at"]
assert.False(t, present)
+ _, present = repo.metadataUpdates[0]["source_created_at"]
+ assert.False(t, present)
}
func TestProcessSync_SyncDeletionsDeletesMatchingKnowledge(t *testing.T) {
diff --git a/internal/datasource/connector/feishu/core/shared.go b/internal/datasource/connector/feishu/core/shared.go
index 5900ecf09..fb6cbb00f 100644
--- a/internal/datasource/connector/feishu/core/shared.go
+++ b/internal/datasource/connector/feishu/core/shared.go
@@ -290,11 +290,13 @@ type DocxFetchInput struct {
// WeKnora external_id: wiki=node.NodeToken, drive=file.Token
DocToken string
// Feishu docx document token
- ObjToken string
- Title string
- URL string
- ResourceID string
- EditTime time.Time
+ ObjToken string
+ Title string
+ URL string
+ ResourceID string
+ EditTime time.Time
+ // CreateTime is the document creation time in Feishu; zero when unknown.
+ CreateTime time.Time
BaseMeta map[string]string
MultimodalEnabled bool
}
@@ -364,6 +366,7 @@ func FetchDocxWithBlocks(ctx context.Context, client *Client, in DocxFetchInput)
FileName: SanitizeFileName(in.Title) + ".md",
URL: in.URL,
UpdatedAt: in.EditTime,
+ CreatedAt: in.CreateTime,
SourceResourceID: in.ResourceID,
Metadata: in.BaseMeta,
ReplacesSubtree: true, // sweep stale attachment sub-items on re-sync
@@ -414,6 +417,7 @@ func FetchDocxWithBlocks(ctx context.Context, client *Client, in DocxFetchInput)
FileName: SanitizeFileName(a.Name),
URL: in.URL,
UpdatedAt: in.EditTime,
+ CreatedAt: in.CreateTime,
SourceResourceID: in.ResourceID,
Metadata: childMeta(),
})
@@ -463,6 +467,7 @@ func FetchDocxWithBlocks(ctx context.Context, client *Client, in DocxFetchInput)
FileName: "image-" + b.Image.Token + ext,
URL: in.URL,
UpdatedAt: in.EditTime,
+ CreatedAt: in.CreateTime,
SourceResourceID: in.ResourceID,
Metadata: imgMeta(),
})
@@ -496,6 +501,7 @@ func exportDocxFallback(ctx context.Context, client *Client, in DocxFetchInput)
FileName: fileName,
URL: in.URL,
UpdatedAt: in.EditTime,
+ CreatedAt: in.CreateTime,
SourceResourceID: in.ResourceID,
Metadata: in.BaseMeta,
}, nil
diff --git a/internal/datasource/connector/feishu/drive/connector.go b/internal/datasource/connector/feishu/drive/connector.go
index 1e2d24327..41acdb5a1 100644
--- a/internal/datasource/connector/feishu/drive/connector.go
+++ b/internal/datasource/connector/feishu/drive/connector.go
@@ -363,6 +363,7 @@ func fetchDriveFileContent(
}
editTime := core.ParseFeishuTimestamp(file.ModifiedTime)
+ createTime := core.ParseFeishuTimestamp(file.CreatedTime)
// Channel marks the knowledge "source" label. Drive uses its own channel
// (feishu_drive / lark_drive) so Drive docs show "飞书云盘" / "Lark 云盘"
// distinct from the wiki connector's "飞书".
@@ -387,6 +388,7 @@ func fetchDriveFileContent(
URL: file.URL,
ResourceID: resourceID,
EditTime: editTime,
+ CreateTime: createTime,
BaseMeta: baseMeta,
MultimodalEnabled: multimodalEnabled,
})
@@ -412,6 +414,7 @@ func fetchDriveFileContent(
FileName: fileName,
URL: file.URL,
UpdatedAt: editTime,
+ CreatedAt: createTime,
SourceResourceID: resourceID,
Metadata: baseMeta,
}}, nil
@@ -435,6 +438,7 @@ func fetchDriveFileContent(
FileName: fileName,
URL: file.URL,
UpdatedAt: editTime,
+ CreatedAt: createTime,
SourceResourceID: resourceID,
Metadata: baseMeta,
}}, nil
diff --git a/internal/datasource/connector/feishu/wiki/connector.go b/internal/datasource/connector/feishu/wiki/connector.go
index 1fc359444..761c2705b 100644
--- a/internal/datasource/connector/feishu/wiki/connector.go
+++ b/internal/datasource/connector/feishu/wiki/connector.go
@@ -243,8 +243,9 @@ func (o wikiOps) Title(n core.WikiNode) string { return n.Title }
func (o wikiOps) ObjType(n core.WikiNode) string { return n.ObjType }
// EditTime is the change-detection timestamp: ObjEditTime (document content)
-// with a NodeEditTime fallback for nodes that lack obj_edit_time. This drives
-// the cursor comparison, NOT FetchedItem.UpdatedAt (which uses NodeEditTime).
+// with a NodeEditTime fallback for nodes that lack obj_edit_time. It drives the
+// cursor comparison and, parsed, FetchedItem.UpdatedAt (see contentEditTime),
+// so the persisted source_updated_at tracks content edits, not node moves.
func (o wikiOps) EditTime(n core.WikiNode) string {
if n.ObjEditTime != "" {
return n.ObjEditTime
@@ -323,7 +324,8 @@ func fetchNodeContent(ctx context.Context, client *core.Client, node core.WikiNo
return nil, nil
}
- editTime := core.ParseFeishuTimestamp(node.NodeEditTime)
+ editTime := contentEditTime(node)
+ createTime := contentCreateTime(node)
baseMeta := map[string]string{
"obj_token": node.ObjToken,
"obj_type": node.ObjType,
@@ -343,6 +345,7 @@ func fetchNodeContent(ctx context.Context, client *core.Client, node core.WikiNo
URL: region.WikiURL(node.NodeToken),
ResourceID: resourceID,
EditTime: editTime,
+ CreateTime: createTime,
BaseMeta: baseMeta,
MultimodalEnabled: multimodalEnabled,
})
@@ -351,12 +354,14 @@ func fetchNodeContent(ctx context.Context, client *core.Client, node core.WikiNo
if err != nil {
return nil, err
}
+ item.CreatedAt = createTime
return []*types.FetchedItem{item}, nil
case "file":
item, err := fetchDriveFile(ctx, client, node, resourceID, editTime, baseMeta, region)
if err != nil {
return nil, err
}
+ item.CreatedAt = createTime
return []*types.FetchedItem{item}, nil
default:
return nil, nil
@@ -422,6 +427,26 @@ func fetchDriveFile(ctx context.Context, client *core.Client, node core.WikiNode
}, nil
}
+// contentEditTime is the document's last content edit time (obj_edit_time),
+// falling back to the node attribute edit time when Feishu omits it. It is
+// what ingestion persists as source_updated_at; NodeEditTime alone would move
+// on a rename or a move in the tree without any content change.
+func contentEditTime(node core.WikiNode) time.Time {
+ if t := core.ParseFeishuTimestamp(node.ObjEditTime); !t.IsZero() {
+ return t
+ }
+ return core.ParseFeishuTimestamp(node.NodeEditTime)
+}
+
+// contentCreateTime is the document creation time (obj_create_time), falling
+// back to the node creation time. Persisted as source_created_at.
+func contentCreateTime(node core.WikiNode) time.Time {
+ if t := core.ParseFeishuTimestamp(node.ObjCreateTime); !t.IsZero() {
+ return t
+ }
+ return core.ParseFeishuTimestamp(node.NodeCreateTime)
+}
+
// --- Helper functions ---
func makeWikiNodeResourceID(spaceID, nodeToken string) string {
diff --git a/internal/datasource/connector/feishu/wiki/connector_test.go b/internal/datasource/connector/feishu/wiki/connector_test.go
index 7d52a118c..305624a9e 100644
--- a/internal/datasource/connector/feishu/wiki/connector_test.go
+++ b/internal/datasource/connector/feishu/wiki/connector_test.go
@@ -579,11 +579,14 @@ func TestConnectorResolveResourceAncestors(t *testing.T) {
func TestFetchAll_DocxNode(t *testing.T) {
nodes := []core.WikiNode{{
- NodeToken: "nt1",
- ObjToken: "obj-docx-1",
- ObjType: "docx",
- Title: "My Document",
- NodeEditTime: "1711468800",
+ NodeToken: "nt1",
+ ObjToken: "obj-docx-1",
+ ObjType: "docx",
+ Title: "My Document",
+ ObjCreateTime: "1700000000",
+ ObjEditTime: "1711000000",
+ NodeCreateTime: "1700000001",
+ NodeEditTime: "1711468800",
}}
ts, cfg := fakeFeishu(nodes)
defer ts.Close()
@@ -617,6 +620,27 @@ func TestFetchAll_DocxNode(t *testing.T) {
if item.Metadata["channel"] != types.ChannelFeishu {
t.Errorf("Metadata[channel] = %q", item.Metadata["channel"])
}
+ // Source timestamps come from the document (obj_*), not the wiki node
+ // attributes, so a rename or move does not look like a content edit.
+ if got := item.UpdatedAt.Unix(); got != 1711000000 {
+ t.Errorf("UpdatedAt = %d, want obj_edit_time 1711000000", got)
+ }
+ if got := item.CreatedAt.Unix(); got != 1700000000 {
+ t.Errorf("CreatedAt = %d, want obj_create_time 1700000000", got)
+ }
+}
+
+func TestContentTimes_FallBackToNodeTimes(t *testing.T) {
+ n := core.WikiNode{NodeCreateTime: "1700000001", NodeEditTime: "1711468800"}
+ if got := contentEditTime(n).Unix(); got != 1711468800 {
+ t.Errorf("contentEditTime fallback = %d, want node_edit_time", got)
+ }
+ if got := contentCreateTime(n).Unix(); got != 1700000001 {
+ t.Errorf("contentCreateTime fallback = %d, want node_create_time", got)
+ }
+ if !contentEditTime(core.WikiNode{}).IsZero() || !contentCreateTime(core.WikiNode{}).IsZero() {
+ t.Errorf("missing times must stay zero, not epoch")
+ }
}
func TestFetchAll_SheetNode(t *testing.T) {
diff --git a/internal/datasource/connector/gitlab/connector.go b/internal/datasource/connector/gitlab/connector.go
index ec512e719..b82419541 100644
--- a/internal/datasource/connector/gitlab/connector.go
+++ b/internal/datasource/connector/gitlab/connector.go
@@ -404,7 +404,24 @@ func (c *Connector) item(ctx context.Context, p *project, ref, file string) (typ
return types.FetchedItem{}, err
}
id := fmt.Sprintf("gitlab:%s:%d:%s:%s", c.canonicalBase, p.ID, ref, file)
- return types.FetchedItem{ExternalID: id, Title: p.PathWithNamespace + "/" + file, FileName: knowledgeRelativePath(p.Name, ref, file), Content: body, ContentType: "text/plain", UpdatedAt: time.Now().UTC(), SourceResourceID: fmt.Sprint(p.ID), Metadata: map[string]string{"channel": types.ConnectorTypeGitLab, "source_type": "gitlab", "gitlab_project_id": fmt.Sprint(p.ID), "gitlab_ref": ref, "gitlab_path": file, "gitlab_url": p.WebURL + "/-/blob/" + ref + "/" + file}}, nil
+ // UpdatedAt is intentionally left unset: the file's last commit time is not
+ // fetched here, and a fetch timestamp would be a fabricated source time.
+ return types.FetchedItem{
+ ExternalID: id,
+ Title: p.PathWithNamespace + "/" + file,
+ FileName: knowledgeRelativePath(p.Name, ref, file),
+ Content: body,
+ ContentType: "text/plain",
+ SourceResourceID: fmt.Sprint(p.ID),
+ Metadata: map[string]string{
+ "channel": types.ConnectorTypeGitLab,
+ "source_type": "gitlab",
+ "gitlab_project_id": fmt.Sprint(p.ID),
+ "gitlab_ref": ref,
+ "gitlab_path": file,
+ "gitlab_url": p.WebURL + "/-/blob/" + ref + "/" + file,
+ },
+ }, nil
}
// knowledgeRelativePath maps a repository file to the KB folder convention:
diff --git a/internal/types/datasource.go b/internal/types/datasource.go
index a1ba855cd..5130b931e 100644
--- a/internal/types/datasource.go
+++ b/internal/types/datasource.go
@@ -332,6 +332,9 @@ type FetchedItem struct {
// When last modified in external system
UpdatedAt time.Time `json:"updated_at"`
+ // When created in external system. Zero when the source does not expose it.
+ CreatedAt time.Time `json:"created_at"`
+
// Additional metadata to preserve
Metadata map[string]string `json:"metadata"`