Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(confluence): stop Server listings that link past the last page
wizardcheneb888d0d3dfe
diff --git a/internal/datasource/connector/confluence/client.go b/internal/datasource/connector/confluence/client.goindex dec73577d..05c9d977f 100644--- a/internal/datasource/connector/confluence/client.go+++ b/internal/datasource/connector/confluence/client.go@@ -21,6 +21,11 @@ const (maxErrorResponseRunes = 1000maxRetryDelay = 60 * time.SecondmaxPaginationHops = 10000+ // maxEmptyServerPages bounds how far a Server/Data Center page listing+ // follows _links.next through empty pages. Some builds keep returning+ // next past the end of a space (#3596); permission filtering can also+ // empty one page mid-listing, so a single empty page is not the end.+ maxEmptyServerPages = 3)type apiError struct {@@ -299,7 +304,11 @@ func withCloudPageQuery(next string) string {return parsed.String()}-func (c *client) pages(ctx context.Context, s space) ([]page, error) {+// pages lists every current page in a space. complete is false when a Server+// listing was cut short after maxEmptyServerPages empty pages that still+// linked onward; the caller must not treat pages missing from such a listing+// as deleted.+func (c *client) pages(ctx context.Context, s space) (pages []page, complete bool, err error) {if c.cfg.cloud() {var all []pagestart := "/api/v2/spaces/" + url.PathEscape(s.ID) + "/pages?status=current&depth=all&limit=250"@@ -323,27 +332,39 @@ func (c *client) pages(ctx context.Context, s space) ([]page, error) {}return next, nil})- return all, err+ return all, true, err}var all []page- err := c.paginate(ctx, serverSpacePagesEndpoint(s.Key), func(pageURL string) (string, error) {+ complete = true+ emptyPages := 0+ err = c.paginate(ctx, serverSpacePagesEndpoint(s.Key), func(pageURL string) (string, error) {var result serverSpacePageListif err := c.get(ctx, pageURL, &result); err != nil {return "", err}- for _, listed := range result.pages() {+ listedPages := result.pages()+ for _, listed := range listedPages {if listed.Space.Key == "" {listed.Space.Key, listed.Space.Name = s.Key, s.Name}all = append(all, listed)}next := result.nextLink()- if next != "" {- next = withServerPageExpand(next)+ if next == "" {+ return "", nil+ }+ if len(listedPages) > 0 {+ emptyPages = 0+ } else {+ emptyPages+++ if emptyPages >= maxEmptyServerPages {+ complete = false+ return "", nil+ }}- return next, nil+ return withServerPageExpand(next), nil})- return all, err+ return all, complete, err}func (c *client) body(ctx context.Context, id string) (pageBody, error) {diff --git a/internal/datasource/connector/confluence/client_pages_test.go b/internal/datasource/connector/confluence/client_pages_test.goindex 9f04b729f..a0407e871 100644--- a/internal/datasource/connector/confluence/client_pages_test.go+++ b/internal/datasource/connector/confluence/client_pages_test.go@@ -3,6 +3,7 @@ package confluenceimport ("context""encoding/json"+ "fmt""net/http""net/http/httptest""testing"@@ -31,7 +32,7 @@ func TestPagesUsesOfficialFlatListAndKeepsExpandOnNext(t *testing.T) {t.Cleanup(server.Close)c := &client{cfg: config{baseURL: server.URL}, http: server.Client()}- pages, err := c.pages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})+ pages, _, err := c.pages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})if err != nil {t.Fatal(err)}@@ -49,6 +50,63 @@ func TestPagesUsesOfficialFlatListAndKeepsExpandOnNext(t *testing.T) {}}+func TestServerPagesStopAfterRepeatedEmptyPages(t *testing.T) {+ calls := 0+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ calls+++ w.Header().Set("Content-Type", "application/json")+ results := `[]`+ if r.URL.Query().Get("start") == "" {+ results = `[{"id": "1", "title": "Root", "version": {"number": 1}}]`+ }+ next := fmt.Sprintf("/rest/api/space/ENG/content/page?limit=100&start=%d", calls*100)+ _, _ = fmt.Fprintf(w, `{"results": %s, "_links": {"next": %q}}`, results, next)+ }))+ t.Cleanup(server.Close)++ c := &client{cfg: config{baseURL: server.URL}, http: server.Client()}+ pages, complete, err := c.pages(context.Background(), space{Key: "ENG", Name: "Engineering"})+ if err != nil {+ t.Fatal(err)+ }+ if complete {+ t.Fatal("a listing cut short by empty pages must not report complete")+ }+ if len(pages) != 1 || pages[0].ID != "1" {+ t.Fatalf("pages = %#v", pages)+ }+ if calls != 1+maxEmptyServerPages {+ t.Fatalf("calls = %d, want %d", calls, 1+maxEmptyServerPages)+ }+}++func TestServerPagesContinuePastOneEmptyPage(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch r.URL.Query().Get("start") {+ case "":+ _, _ = w.Write([]byte(`{"results": [{"id": "1", "title": "A"}],+ "_links": {"next": "/rest/api/space/ENG/content/page?limit=100&start=100"}}`))+ case "100":+ // Every page in this window was filtered out by permissions.+ _, _ = w.Write([]byte(`{"results": [],+ "_links": {"next": "/rest/api/space/ENG/content/page?limit=100&start=200"}}`))+ default:+ _, _ = w.Write([]byte(`{"results": [{"id": "2", "title": "B"}]}`))+ }+ }))+ t.Cleanup(server.Close)++ c := &client{cfg: config{baseURL: server.URL}, http: server.Client()}+ pages, complete, err := c.pages(context.Background(), space{Key: "ENG", Name: "Engineering"})+ if err != nil {+ t.Fatal(err)+ }+ if !complete || len(pages) != 2 {+ t.Fatalf("complete = %v, pages = %#v", complete, pages)+ }+}+func TestPagesAcceptsNestedContentEnvelope(t *testing.T) {server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {w.Header().Set("Content-Type", "application/json")@@ -61,7 +119,7 @@ func TestPagesAcceptsNestedContentEnvelope(t *testing.T) {t.Cleanup(server.Close)c := &client{cfg: config{baseURL: server.URL}, http: server.Client()}- pages, err := c.pages(context.Background(), space{Key: "ENG", Name: "Engineering"})+ pages, _, err := c.pages(context.Background(), space{Key: "ENG", Name: "Engineering"})if err != nil {t.Fatal(err)}@@ -108,7 +166,7 @@ func TestPagesCloudFollowsNextAndKeepsDepth(t *testing.T) {t.Cleanup(server.Close)c := &client{cfg: config{baseURL: server.URL + "/wiki", edition: editionCloud}, http: server.Client()}- pages, err := c.pages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})+ pages, _, err := c.pages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})if err != nil {t.Fatal(err)}diff --git a/internal/datasource/connector/confluence/connector.go b/internal/datasource/connector/confluence/connector.goindex 7b10b9df9..b753484a8 100644--- a/internal/datasource/connector/confluence/connector.go+++ b/internal/datasource/connector/confluence/connector.go@@ -10,6 +10,7 @@ import (htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2""github.com/Tencent/WeKnora/internal/datasource"+ "github.com/Tencent/WeKnora/internal/logger""github.com/Tencent/WeKnora/internal/types")@@ -174,10 +175,16 @@ func (c *Connector) fetchStream(resourceID,)}- pages, err := client.pages(ctx, s)+ pages, complete, err := client.pages(ctx, s)if err != nil {return nil, fmt.Errorf("list pages in Confluence space %s: %w", s.Key, err)}+ if !complete {+ logger.Warnf(ctx,+ "[Confluence] space %s: listing stopped after %d empty pages that still linked onward; "++ "imported %d pages and skipped deletion reconciliation",+ s.Key, maxEmptyServerPages, len(pages))+ }priorPages, hadBaseline := baseline.SpacePages[resourceID]if next.SpacePages[resourceID] == nil {next.SpacePages[resourceID] = map[string]string{}@@ -221,7 +228,16 @@ func (c *Connector) fetchStream(return nil, err}}- if hadBaseline {+ if !complete {+ // A cut-short listing proves nothing about the pages it did not+ // reach: keep tracking them so a later complete listing can still+ // reconcile them, and delete nothing now.+ for id, version := range priorPages {+ if _, exists := seen[id]; !exists {+ next.SpacePages[resourceID][id] = version+ }+ }+ } else if hadBaseline {if len(pages) == 0 && len(priorPages) > 0 {return nil, fmt.Errorf("refusing Confluence mirror deletion in %s: listing returned 0 pages against a %d-page baseline",diff --git a/internal/datasource/connector/confluence/connector_stream_test.go b/internal/datasource/connector/confluence/connector_stream_test.goindex b86ef9792..07a024b0e 100644--- a/internal/datasource/connector/confluence/connector_stream_test.go+++ b/internal/datasource/connector/confluence/connector_stream_test.go@@ -5,6 +5,7 @@ import ("context""encoding/json""errors"+ "fmt""io""net/http""strings"@@ -30,6 +31,9 @@ type streamAPI struct {bodyCalls intbodyStatus map[string]intlistCalls int+ // endlessNext mimics Server/Data Center builds that keep linking to a+ // next page after the last one, returning empty results forever (#3596).+ endlessNext bool}func (a *streamAPI) jsonResponse(value any) (*http.Response, error) {@@ -66,8 +70,19 @@ func (a *streamAPI) response(req *http.Request) (*http.Response, error) {})case path == "/wiki/rest/api/space/ENG/content/page":a.listCalls+++ if !a.endlessNext {+ return a.jsonResponse(map[string]any{+ "results": a.pageMaps(),+ })+ }+ results := a.pageMaps()+ if a.listCalls > 1 {+ results = []any{}+ }+ next := fmt.Sprintf("/rest/api/space/ENG/content/page?limit=100&start=%d", a.listCalls*100)return a.jsonResponse(map[string]any{- "results": a.pageMaps(),+ "results": results,+ "_links": map[string]any{"next": next},})case strings.HasPrefix(path, "/wiki/rest/api/content/"):if req.URL.Query().Get("expand") != "body.view,version,space" {@@ -387,3 +402,37 @@ func TestListResourcesKeepsInvalidWebUI(t *testing.T) {t.Fatalf("ListResources() with hostile webui = %#v, %v", resources, err)}}++func TestFetchStreamKeepsUnreachedPagesWhenServerListingNeverEnds(t *testing.T) {+ for _, full := range []bool{false, true} {+ t.Run(fmt.Sprintf("full=%v", full), func(t *testing.T) {+ api := &streamAPI{endlessNext: true, pages: []streamPage{{id: "p1", title: "Page", version: 2}}}+ h := &captureHandler{}+ old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"})+ connector := newStreamConnector(api)+ fetch := connector.FetchStream+ if full {+ fetch = connector.FetchFullStream+ }+ got, err := fetch(context.Background(), streamConfig(), old, h)+ if err != nil {+ t.Fatalf("fetch: %v", err)+ }+ if api.listCalls != 1+maxEmptyServerPages {+ t.Fatalf("listCalls = %d, want %d", api.listCalls, 1+maxEmptyServerPages)+ }+ for _, item := range h.items {+ if item.IsDeleted {+ t.Fatalf("cut-short listing emitted a deletion for %s", item.ExternalID)+ }+ }+ if len(h.items) != 1 || h.items[0].ExternalID != "p1" {+ t.Fatalf("items = %#v, want the changed page p1 only", h.items)+ }+ pages := decodeCursor(got).SpacePages["1"]+ if pages["p1"] != "v:2" || pages["p2"] != "v:1" {+ t.Fatalf("cursor pages = %#v, want p1 advanced and unreached p2 kept", pages)+ }+ })+ }+}