Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(yuque): don't acknowledge failed detail fetches in the incremental cursor (#3702)
LuckVd0d3a053eb62c
diff --git a/internal/datasource/connector/yuque/connector.go b/internal/datasource/connector/yuque/connector.goindex b33130d87..3ef687e3e 100644--- a/internal/datasource/connector/yuque/connector.go+++ b/internal/datasource/connector/yuque/connector.go@@ -270,6 +270,19 @@ func (c *Connector) walk(detail, err := cli.GetDocDetail(ctx, d.ID)if err != nil {+ // Do not acknowledge the doc's new version when its detail fetch+ // failed: rewind the cursor entry to the previously synced version+ // (existing docs) or drop it (new docs), so the next incremental+ // sync re-fetches the doc once the endpoint recovers. Acking before+ // the fetch succeeded made the doc silently skipped by every later+ // incremental sync (Tencent/WeKnora#3690). The doc stays in+ // currentDocs, so deletion detection is unaffected.+ if prev != nil && prev.BookDocTimes != nil {+ newCursor.BookDocTimes[bookIDStr][docIDStr] = prev.BookDocTimes[bookIDStr][docIDStr]+ }+ if newCursor.BookDocTimes[bookIDStr][docIDStr] == "" {+ delete(newCursor.BookDocTimes[bookIDStr], docIDStr)+ }// Record failure but continue (placeholder item with error metadata).// Keep doc_id/book_id/slug for observability pipelines that join on these.out = append(out, types.FetchedItem{diff --git a/internal/datasource/connector/yuque/connector_test.go b/internal/datasource/connector/yuque/connector_test.goindex 8fc609002..f2a5e478a 100644--- a/internal/datasource/connector/yuque/connector_test.go+++ b/internal/datasource/connector/yuque/connector_test.go@@ -2,6 +2,7 @@ package yuqueimport ("context"+ "encoding/json""net/http""net/http/httptest""os"@@ -513,3 +514,172 @@ func TestConnector_FetchAll_FileNamePolicy(t *testing.T) {})}}++// Regression for Tencent/WeKnora#3690: when an existing doc's detail fetch+// fails after its content changed, the incremental cursor must keep the doc's+// previously synced version instead of acknowledging the new one — otherwise+// every later sync sees the doc as unchanged and never retries the fetch.+func TestConnector_FetchIncremental_DetailFailureDoesNotAckVersion(t *testing.T) {+ // First sync: docs 1 and 2, both at t0.+ f1 := newFakeYuque()+ f1.handleJSON("/api/v2/repos/12/docs", 200, v2DocListResponse{Data: []v2Doc{+ {ID: 1, Type: "Doc", Status: "1", Title: "A", Slug: "a", ContentUpdatedAt: "2026-04-20T10:00:00Z"},+ {ID: 2, Type: "Doc", Status: "1", Title: "B", Slug: "b", ContentUpdatedAt: "2026-04-20T10:00:00Z"},+ }})+ f1.handleJSON("/api/v2/repos/docs/1", 200, v2DocDetailResponse{Data: v2DocDetail{+ ID: 1, Title: "A", Body: "a", Status: "1", ContentUpdatedAt: "2026-04-20T10:00:00Z",+ }})+ f1.handleJSON("/api/v2/repos/docs/2", 200, v2DocDetailResponse{Data: v2DocDetail{+ ID: 2, Title: "B", Body: "b", Status: "1", ContentUpdatedAt: "2026-04-20T10:00:00Z",+ }})++ _, cursor1, err := NewConnector().FetchIncremental(context.Background(), makeDSConfig(f1, []string{"12"}), nil)+ if err != nil {+ t.Fatalf("first sync: %v", err)+ }+ f1.Close()++ // Second sync: doc 2 changed to t1 but its detail endpoint fails (mixed+ // success/failure — doc 1 is unchanged, so the batch is not all-failed).+ f2 := newFakeYuque()+ f2.handleJSON("/api/v2/repos/12/docs", 200, v2DocListResponse{Data: []v2Doc{+ {ID: 1, Type: "Doc", Status: "1", Title: "A", Slug: "a", ContentUpdatedAt: "2026-04-20T10:00:00Z"},+ {ID: 2, Type: "Doc", Status: "1", Title: "B2", Slug: "b", ContentUpdatedAt: "2026-04-20T12:00:00Z"},+ }})+ f2.mux.HandleFunc("/api/v2/repos/docs/2", func(w http.ResponseWriter, _ *http.Request) {+ w.WriteHeader(http.StatusBadRequest) // 4xx is non-retriable, bubbles up as an error+ _, _ = w.Write([]byte(`{"message":"broken"}`))+ })++ items2, cursor2, err := NewConnector().FetchIncremental(+ context.Background(), makeDSConfig(f2, []string{"12"}), cursor1)+ if err != nil {+ t.Fatalf("second sync must not fail on a single detail error, got %v", err)+ }+ f2.Close()++ var placeholder *types.FetchedItem+ for i := range items2 {+ if items2[i].Metadata["error"] != "" && items2[i].ExternalID == "2" {+ placeholder = &items2[i]+ }+ }+ if placeholder == nil {+ t.Fatalf("expected an error placeholder for doc 2, got %+v", items2)+ }++ // The cursor must still hold doc 2's previously synced version (t0), not+ // the failed t1: acknowledging t1 hid the doc from all future syncs (#3690).+ var cur2 yuqueCursor+ if b, merr := json.Marshal(cursor2.ConnectorCursor); merr != nil {+ t.Fatalf("encode cursor: %v", merr)+ } else if uerr := json.Unmarshal(b, &cur2); uerr != nil {+ t.Fatalf("decode cursor: %v", uerr)+ }+ if got := cur2.BookDocTimes["12"]["2"]; got != "2026-04-20T10:00:00Z" {+ t.Fatalf("cursor must keep doc 2's previous version after a failed detail fetch, got %q", got)+ }++ // Third sync, endpoint recovered: doc 2 must be re-fetched because its+ // cursor entry (t0) still differs from the listing (t1).+ f3 := newFakeYuque()+ defer f3.Close()+ f3.handleJSON("/api/v2/repos/12/docs", 200, v2DocListResponse{Data: []v2Doc{+ {ID: 1, Type: "Doc", Status: "1", Title: "A", Slug: "a", ContentUpdatedAt: "2026-04-20T10:00:00Z"},+ {ID: 2, Type: "Doc", Status: "1", Title: "B2", Slug: "b", ContentUpdatedAt: "2026-04-20T12:00:00Z"},+ }})+ f3.handleJSON("/api/v2/repos/docs/2", 200, v2DocDetailResponse{Data: v2DocDetail{+ ID: 2, Title: "B2", Body: "b2 recovered", Status: "1", ContentUpdatedAt: "2026-04-20T12:00:00Z",+ }})++ items3, _, err := NewConnector().FetchIncremental(context.Background(), makeDSConfig(f3, []string{"12"}), cursor2)+ if err != nil {+ t.Fatalf("third sync: %v", err)+ }+ if len(items3) != 1 || items3[0].ExternalID != "2" {+ t.Fatalf("expected recovered doc 2 to be re-fetched, got %+v", items3)+ }+ if string(items3[0].Content) != "b2 recovered" {+ t.Errorf("expected recovered body, got %q", string(items3[0].Content))+ }+}++// Regression for Tencent/WeKnora#3690 (new-doc variant): a newly listed doc+// whose detail fetch fails must not enter the cursor at all, so the next sync+// still treats it as new and ingests it.+func TestConnector_FetchIncremental_NewDocDetailFailureNotAcknowledged(t *testing.T) {+ // First sync: doc 1 only.+ f1 := newFakeYuque()+ f1.handleJSON("/api/v2/repos/13/docs", 200, v2DocListResponse{Data: []v2Doc{+ {ID: 1, Type: "Doc", Status: "1", Title: "A", Slug: "a", ContentUpdatedAt: "2026-04-20T10:00:00Z"},+ }})+ f1.handleJSON("/api/v2/repos/docs/1", 200, v2DocDetailResponse{Data: v2DocDetail{+ ID: 1, Title: "A", Body: "a", Status: "1", ContentUpdatedAt: "2026-04-20T10:00:00Z",+ }})++ _, cursor1, err := NewConnector().FetchIncremental(context.Background(), makeDSConfig(f1, []string{"13"}), nil)+ if err != nil {+ t.Fatalf("first sync: %v", err)+ }+ f1.Close()++ // Second sync: new doc 2 appears (t1) but its detail fetch fails.+ f2 := newFakeYuque()+ f2.handleJSON("/api/v2/repos/13/docs", 200, v2DocListResponse{Data: []v2Doc{+ {ID: 1, Type: "Doc", Status: "1", Title: "A", Slug: "a", ContentUpdatedAt: "2026-04-20T10:00:00Z"},+ {ID: 2, Type: "Doc", Status: "1", Title: "New", Slug: "new", ContentUpdatedAt: "2026-04-20T12:00:00Z"},+ }})+ f2.mux.HandleFunc("/api/v2/repos/docs/2", func(w http.ResponseWriter, _ *http.Request) {+ w.WriteHeader(http.StatusBadRequest) // 4xx is non-retriable, bubbles up as an error+ _, _ = w.Write([]byte(`{"message":"broken"}`))+ })++ items2, cursor2, err := NewConnector().FetchIncremental(+ context.Background(), makeDSConfig(f2, []string{"13"}), cursor1)+ if err != nil {+ t.Fatalf("second sync must not fail on a single detail error, got %v", err)+ }+ f2.Close()++ found := false+ for _, it := range items2 {+ if it.Metadata["error"] != "" && it.ExternalID == "2" {+ found = true+ }+ }+ if !found {+ t.Fatalf("expected an error placeholder for doc 2, got %+v", items2)+ }++ var cur2 yuqueCursor+ if b, merr := json.Marshal(cursor2.ConnectorCursor); merr != nil {+ t.Fatalf("encode cursor: %v", merr)+ } else if uerr := json.Unmarshal(b, &cur2); uerr != nil {+ t.Fatalf("decode cursor: %v", uerr)+ }+ if _, exists := cur2.BookDocTimes["13"]["2"]; exists {+ t.Fatal("failed new doc must not be acknowledged into the cursor")+ }+ if got := cur2.BookDocTimes["13"]["1"]; got != "2026-04-20T10:00:00Z" {+ t.Errorf("unchanged doc 1 cursor entry must be untouched, got %q", got)+ }++ // Third sync, endpoint recovered: doc 2 is ingested as a new item.+ f3 := newFakeYuque()+ defer f3.Close()+ f3.handleJSON("/api/v2/repos/13/docs", 200, v2DocListResponse{Data: []v2Doc{+ {ID: 1, Type: "Doc", Status: "1", Title: "A", Slug: "a", ContentUpdatedAt: "2026-04-20T10:00:00Z"},+ {ID: 2, Type: "Doc", Status: "1", Title: "New", Slug: "new", ContentUpdatedAt: "2026-04-20T12:00:00Z"},+ }})+ f3.handleJSON("/api/v2/repos/docs/2", 200, v2DocDetailResponse{Data: v2DocDetail{+ ID: 2, Title: "New", Body: "new content", Status: "1", ContentUpdatedAt: "2026-04-20T12:00:00Z",+ }})++ items3, _, err := NewConnector().FetchIncremental(context.Background(), makeDSConfig(f3, []string{"13"}), cursor2)+ if err != nil {+ t.Fatalf("third sync: %v", err)+ }+ if len(items3) != 1 || items3[0].ExternalID != "2" {+ t.Fatalf("expected new doc 2 to be ingested after recovery, got %+v", items3)+ }+}