Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(feishu): checkpoint sync rounds that skip every node (#4736)
u7-u7662abaf9f944
diff --git a/internal/datasource/connector/feishu/core/engine.go b/internal/datasource/connector/feishu/core/engine.goindex 59e0ee48a..c320d6b31 100644--- a/internal/datasource/connector/feishu/core/engine.go+++ b/internal/datasource/connector/feishu/core/engine.go@@ -127,6 +127,24 @@ func runSync[N any](processed := 0lastCheckpoint := time.Now()+ // maybeCheckpoint persists progress on a node-count or wall-clock cadence.+ // Skipped nodes count too: a round where every node was already up to date+ // would otherwise write no checkpoint at all, so the next run fell back to+ // the previous cursor snapshot and re-fetched the whole tree (#4712).+ maybeCheckpoint := func() {+ if processed%FeishuStreamCheckpointInterval != 0 &&+ time.Since(lastCheckpoint) < FeishuStreamCheckpointMaxInterval {+ return+ }+ checkpoint := ops.EncodeCursor(newTimes, lastSync)+ if fullRun {+ checkpoint.ConnectorCursor[fullSyncPending] = true+ }+ if cerr := h.Checkpoint(ctx, checkpoint); cerr != nil {+ logger.Warnf(ctx, "%s stream Checkpoint failed: %v", ops.LogTag(), cerr)+ }+ lastCheckpoint = time.Now()+ }for _, resourceID := range resourceIDs {nodes, partial, err := ops.List(ctx, client, resourceID)if err != nil {@@ -160,6 +178,8 @@ func runSync[N any](// and Skip re-fetching.if !forceFull && hadPrev && prevEdit == editTimeStr {newTimes[resourceID][tok] = editTimeStr+ processed+++ maybeCheckpoint()continue}@@ -198,16 +218,7 @@ func runSync[N any](}processed++- if processed%FeishuStreamCheckpointInterval == 0 || time.Since(lastCheckpoint) >= FeishuStreamCheckpointMaxInterval {- checkpoint := ops.EncodeCursor(newTimes, lastSync)- if fullRun {- checkpoint.ConnectorCursor[fullSyncPending] = true- }- if cerr := h.Checkpoint(ctx, checkpoint); cerr != nil {- logger.Warnf(ctx, "%s stream Checkpoint failed: %v", ops.LogTag(), cerr)- }- lastCheckpoint = time.Now()- }+ maybeCheckpoint()if n := i + 1; n%100 == 0 {logger.Infof(ctx, "%s stream progress resource=%s %d/%d (%s)",ops.LogTag(), resourceID, n, len(nodes), tally.summary())diff --git a/internal/datasource/connector/feishu/core/engine_full_test.go b/internal/datasource/connector/feishu/core/engine_full_test.goindex f68ff9203..62f9253a9 100644--- a/internal/datasource/connector/feishu/core/engine_full_test.go+++ b/internal/datasource/connector/feishu/core/engine_full_test.go@@ -144,3 +144,30 @@ func TestFullSyncCheckpointRetainsUnseenDeletionBaseline(t *testing.T) {require.Len(t, ops.fetched, len(ops.nodes["first"]),"a later independent full sync must refetch everything")}++// A round where every node is already at its recorded edit time is the normal+// incremental case after a quiet period. It must still checkpoint the cursor:+// otherwise the next run starts from the previous snapshot and re-fetches the+// whole tree (#4712).+func TestStreamCheckpointsARoundThatSkipsEveryNode(t *testing.T) {+ nodeCount := FeishuStreamCheckpointInterval + 1+ nodes := make([]string, 0, nodeCount)+ previous := make(map[string]string, nodeCount)+ for i := 0; i < nodeCount; i++ {+ name := fmt.Sprintf("node%d", i)+ nodes = append(nodes, name)+ previous[name] = "100"+ }++ ops := &fullSyncOps{nodes: map[string][]string{"r": nodes}}+ cfg := &types.DataSourceConfig{ResourceIDs: []string{"r"}}+ cursor := ops.EncodeCursor(map[string]map[string]string{"r": previous}, time.Now())+ h := &fullSyncHandler{}++ next, err := FetchStreamEngine(context.Background(), nil, cfg, cursor, h, ops)+ require.NoError(t, err)+ require.Empty(t, ops.fetched, "an unchanged tree must not be re-fetched")+ require.NotNil(t, h.checkpoint, "a fully skipped round must still checkpoint the cursor")+ require.Equal(t, previous, ops.DecodeCursorTimes(h.checkpoint.ConnectorCursor)["r"])+ require.Equal(t, previous, ops.DecodeCursorTimes(next.ConnectorCursor)["r"])+}