Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(feishu): checkpoint sync rounds that skip every node (#4736)

u7-u7662abaf9f944
diff --git a/internal/datasource/connector/feishu/core/engine.go b/internal/datasource/connector/feishu/core/engine.go
index 59e0ee48a..c320d6b31 100644
--- a/internal/datasource/connector/feishu/core/engine.go
+++ b/internal/datasource/connector/feishu/core/engine.go
@@ -127,6 +127,24 @@ func runSync[N any](
processed := 0
lastCheckpoint := time.Now()
+ // maybeCheckpoint persists progress on a node-count or wall-clock cadence.
+ // Skipped nodes count too: a round where every node was already up to date
+ // would otherwise write no checkpoint at all, so the next run fell back to
+ // the previous cursor snapshot and re-fetched the whole tree (#4712).
+ maybeCheckpoint := func() {
+ if processed%FeishuStreamCheckpointInterval != 0 &&
+ time.Since(lastCheckpoint) < FeishuStreamCheckpointMaxInterval {
+ return
+ }
+ checkpoint := ops.EncodeCursor(newTimes, lastSync)
+ if fullRun {
+ checkpoint.ConnectorCursor[fullSyncPending] = true
+ }
+ if cerr := h.Checkpoint(ctx, checkpoint); cerr != nil {
+ logger.Warnf(ctx, "%s stream Checkpoint failed: %v", ops.LogTag(), cerr)
+ }
+ lastCheckpoint = time.Now()
+ }
for _, resourceID := range resourceIDs {
nodes, partial, err := ops.List(ctx, client, resourceID)
if err != nil {
@@ -160,6 +178,8 @@ func runSync[N any](
// and Skip re-fetching.
if !forceFull && hadPrev && prevEdit == editTimeStr {
newTimes[resourceID][tok] = editTimeStr
+ processed++
+ maybeCheckpoint()
continue
}
@@ -198,16 +218,7 @@ func runSync[N any](
}
processed++
- if processed%FeishuStreamCheckpointInterval == 0 || time.Since(lastCheckpoint) >= FeishuStreamCheckpointMaxInterval {
- checkpoint := ops.EncodeCursor(newTimes, lastSync)
- if fullRun {
- checkpoint.ConnectorCursor[fullSyncPending] = true
- }
- if cerr := h.Checkpoint(ctx, checkpoint); cerr != nil {
- logger.Warnf(ctx, "%s stream Checkpoint failed: %v", ops.LogTag(), cerr)
- }
- lastCheckpoint = time.Now()
- }
+ maybeCheckpoint()
if n := i + 1; n%100 == 0 {
logger.Infof(ctx, "%s stream progress resource=%s %d/%d (%s)",
ops.LogTag(), resourceID, n, len(nodes), tally.summary())
diff --git a/internal/datasource/connector/feishu/core/engine_full_test.go b/internal/datasource/connector/feishu/core/engine_full_test.go
index f68ff9203..62f9253a9 100644
--- a/internal/datasource/connector/feishu/core/engine_full_test.go
+++ b/internal/datasource/connector/feishu/core/engine_full_test.go
@@ -144,3 +144,30 @@ func TestFullSyncCheckpointRetainsUnseenDeletionBaseline(t *testing.T) {
require.Len(t, ops.fetched, len(ops.nodes["first"]),
"a later independent full sync must refetch everything")
}
+
+// A round where every node is already at its recorded edit time is the normal
+// incremental case after a quiet period. It must still checkpoint the cursor:
+// otherwise the next run starts from the previous snapshot and re-fetches the
+// whole tree (#4712).
+func TestStreamCheckpointsARoundThatSkipsEveryNode(t *testing.T) {
+ nodeCount := FeishuStreamCheckpointInterval + 1
+ nodes := make([]string, 0, nodeCount)
+ previous := make(map[string]string, nodeCount)
+ for i := 0; i < nodeCount; i++ {
+ name := fmt.Sprintf("node%d", i)
+ nodes = append(nodes, name)
+ previous[name] = "100"
+ }
+
+ ops := &fullSyncOps{nodes: map[string][]string{"r": nodes}}
+ cfg := &types.DataSourceConfig{ResourceIDs: []string{"r"}}
+ cursor := ops.EncodeCursor(map[string]map[string]string{"r": previous}, time.Now())
+ h := &fullSyncHandler{}
+
+ next, err := FetchStreamEngine(context.Background(), nil, cfg, cursor, h, ops)
+ require.NoError(t, err)
+ require.Empty(t, ops.fetched, "an unchanged tree must not be re-fetched")
+ require.NotNil(t, h.checkpoint, "a fully skipped round must still checkpoint the cursor")
+ require.Equal(t, previous, ops.DecodeCursorTimes(h.checkpoint.ConnectorCursor)["r"])
+ require.Equal(t, previous, ops.DecodeCursorTimes(next.ConnectorCursor)["r"])
+}