Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(confluence): stop Server listings that link past the last page

wizardcheneb888d0d3dfe
diff --git a/internal/datasource/connector/confluence/client.go b/internal/datasource/connector/confluence/client.go
index dec73577d..05c9d977f 100644
--- a/internal/datasource/connector/confluence/client.go
+++ b/internal/datasource/connector/confluence/client.go
@@ -21,6 +21,11 @@ const (
maxErrorResponseRunes = 1000
maxRetryDelay = 60 * time.Second
maxPaginationHops = 10000
+ // maxEmptyServerPages bounds how far a Server/Data Center page listing
+ // follows _links.next through empty pages. Some builds keep returning
+ // next past the end of a space (#3596); permission filtering can also
+ // empty one page mid-listing, so a single empty page is not the end.
+ maxEmptyServerPages = 3
)
type apiError struct {
@@ -299,7 +304,11 @@ func withCloudPageQuery(next string) string {
return parsed.String()
}
-func (c *client) pages(ctx context.Context, s space) ([]page, error) {
+// pages lists every current page in a space. complete is false when a Server
+// listing was cut short after maxEmptyServerPages empty pages that still
+// linked onward; the caller must not treat pages missing from such a listing
+// as deleted.
+func (c *client) pages(ctx context.Context, s space) (pages []page, complete bool, err error) {
if c.cfg.cloud() {
var all []page
start := "/api/v2/spaces/" + url.PathEscape(s.ID) + "/pages?status=current&depth=all&limit=250"
@@ -323,27 +332,39 @@ func (c *client) pages(ctx context.Context, s space) ([]page, error) {
}
return next, nil
})
- return all, err
+ return all, true, err
}
var all []page
- err := c.paginate(ctx, serverSpacePagesEndpoint(s.Key), func(pageURL string) (string, error) {
+ complete = true
+ emptyPages := 0
+ err = c.paginate(ctx, serverSpacePagesEndpoint(s.Key), func(pageURL string) (string, error) {
var result serverSpacePageList
if err := c.get(ctx, pageURL, &result); err != nil {
return "", err
}
- for _, listed := range result.pages() {
+ listedPages := result.pages()
+ for _, listed := range listedPages {
if listed.Space.Key == "" {
listed.Space.Key, listed.Space.Name = s.Key, s.Name
}
all = append(all, listed)
}
next := result.nextLink()
- if next != "" {
- next = withServerPageExpand(next)
+ if next == "" {
+ return "", nil
+ }
+ if len(listedPages) > 0 {
+ emptyPages = 0
+ } else {
+ emptyPages++
+ if emptyPages >= maxEmptyServerPages {
+ complete = false
+ return "", nil
+ }
}
- return next, nil
+ return withServerPageExpand(next), nil
})
- return all, err
+ return all, complete, err
}
func (c *client) body(ctx context.Context, id string) (pageBody, error) {
diff --git a/internal/datasource/connector/confluence/client_pages_test.go b/internal/datasource/connector/confluence/client_pages_test.go
index 9f04b729f..a0407e871 100644
--- a/internal/datasource/connector/confluence/client_pages_test.go
+++ b/internal/datasource/connector/confluence/client_pages_test.go
@@ -3,6 +3,7 @@ package confluence
import (
"context"
"encoding/json"
+ "fmt"
"net/http"
"net/http/httptest"
"testing"
@@ -31,7 +32,7 @@ func TestPagesUsesOfficialFlatListAndKeepsExpandOnNext(t *testing.T) {
t.Cleanup(server.Close)
c := &client{cfg: config{baseURL: server.URL}, http: server.Client()}
- pages, err := c.pages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})
+ pages, _, err := c.pages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})
if err != nil {
t.Fatal(err)
}
@@ -49,6 +50,63 @@ func TestPagesUsesOfficialFlatListAndKeepsExpandOnNext(t *testing.T) {
}
}
+func TestServerPagesStopAfterRepeatedEmptyPages(t *testing.T) {
+ calls := 0
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ calls++
+ w.Header().Set("Content-Type", "application/json")
+ results := `[]`
+ if r.URL.Query().Get("start") == "" {
+ results = `[{"id": "1", "title": "Root", "version": {"number": 1}}]`
+ }
+ next := fmt.Sprintf("/rest/api/space/ENG/content/page?limit=100&start=%d", calls*100)
+ _, _ = fmt.Fprintf(w, `{"results": %s, "_links": {"next": %q}}`, results, next)
+ }))
+ t.Cleanup(server.Close)
+
+ c := &client{cfg: config{baseURL: server.URL}, http: server.Client()}
+ pages, complete, err := c.pages(context.Background(), space{Key: "ENG", Name: "Engineering"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if complete {
+ t.Fatal("a listing cut short by empty pages must not report complete")
+ }
+ if len(pages) != 1 || pages[0].ID != "1" {
+ t.Fatalf("pages = %#v", pages)
+ }
+ if calls != 1+maxEmptyServerPages {
+ t.Fatalf("calls = %d, want %d", calls, 1+maxEmptyServerPages)
+ }
+}
+
+func TestServerPagesContinuePastOneEmptyPage(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch r.URL.Query().Get("start") {
+ case "":
+ _, _ = w.Write([]byte(`{"results": [{"id": "1", "title": "A"}],
+ "_links": {"next": "/rest/api/space/ENG/content/page?limit=100&start=100"}}`))
+ case "100":
+ // Every page in this window was filtered out by permissions.
+ _, _ = w.Write([]byte(`{"results": [],
+ "_links": {"next": "/rest/api/space/ENG/content/page?limit=100&start=200"}}`))
+ default:
+ _, _ = w.Write([]byte(`{"results": [{"id": "2", "title": "B"}]}`))
+ }
+ }))
+ t.Cleanup(server.Close)
+
+ c := &client{cfg: config{baseURL: server.URL}, http: server.Client()}
+ pages, complete, err := c.pages(context.Background(), space{Key: "ENG", Name: "Engineering"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !complete || len(pages) != 2 {
+ t.Fatalf("complete = %v, pages = %#v", complete, pages)
+ }
+}
+
func TestPagesAcceptsNestedContentEnvelope(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
@@ -61,7 +119,7 @@ func TestPagesAcceptsNestedContentEnvelope(t *testing.T) {
t.Cleanup(server.Close)
c := &client{cfg: config{baseURL: server.URL}, http: server.Client()}
- pages, err := c.pages(context.Background(), space{Key: "ENG", Name: "Engineering"})
+ pages, _, err := c.pages(context.Background(), space{Key: "ENG", Name: "Engineering"})
if err != nil {
t.Fatal(err)
}
@@ -108,7 +166,7 @@ func TestPagesCloudFollowsNextAndKeepsDepth(t *testing.T) {
t.Cleanup(server.Close)
c := &client{cfg: config{baseURL: server.URL + "/wiki", edition: editionCloud}, http: server.Client()}
- pages, err := c.pages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})
+ pages, _, err := c.pages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})
if err != nil {
t.Fatal(err)
}
diff --git a/internal/datasource/connector/confluence/connector.go b/internal/datasource/connector/confluence/connector.go
index 7b10b9df9..b753484a8 100644
--- a/internal/datasource/connector/confluence/connector.go
+++ b/internal/datasource/connector/confluence/connector.go
@@ -10,6 +10,7 @@ import (
htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"
"github.com/Tencent/WeKnora/internal/datasource"
+ "github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
)
@@ -174,10 +175,16 @@ func (c *Connector) fetchStream(
resourceID,
)
}
- pages, err := client.pages(ctx, s)
+ pages, complete, err := client.pages(ctx, s)
if err != nil {
return nil, fmt.Errorf("list pages in Confluence space %s: %w", s.Key, err)
}
+ if !complete {
+ logger.Warnf(ctx,
+ "[Confluence] space %s: listing stopped after %d empty pages that still linked onward; "+
+ "imported %d pages and skipped deletion reconciliation",
+ s.Key, maxEmptyServerPages, len(pages))
+ }
priorPages, hadBaseline := baseline.SpacePages[resourceID]
if next.SpacePages[resourceID] == nil {
next.SpacePages[resourceID] = map[string]string{}
@@ -221,7 +228,16 @@ func (c *Connector) fetchStream(
return nil, err
}
}
- if hadBaseline {
+ if !complete {
+ // A cut-short listing proves nothing about the pages it did not
+ // reach: keep tracking them so a later complete listing can still
+ // reconcile them, and delete nothing now.
+ for id, version := range priorPages {
+ if _, exists := seen[id]; !exists {
+ next.SpacePages[resourceID][id] = version
+ }
+ }
+ } else if hadBaseline {
if len(pages) == 0 && len(priorPages) > 0 {
return nil, fmt.Errorf(
"refusing Confluence mirror deletion in %s: listing returned 0 pages against a %d-page baseline",
diff --git a/internal/datasource/connector/confluence/connector_stream_test.go b/internal/datasource/connector/confluence/connector_stream_test.go
index b86ef9792..07a024b0e 100644
--- a/internal/datasource/connector/confluence/connector_stream_test.go
+++ b/internal/datasource/connector/confluence/connector_stream_test.go
@@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"errors"
+ "fmt"
"io"
"net/http"
"strings"
@@ -30,6 +31,9 @@ type streamAPI struct {
bodyCalls int
bodyStatus map[string]int
listCalls int
+ // endlessNext mimics Server/Data Center builds that keep linking to a
+ // next page after the last one, returning empty results forever (#3596).
+ endlessNext bool
}
func (a *streamAPI) jsonResponse(value any) (*http.Response, error) {
@@ -66,8 +70,19 @@ func (a *streamAPI) response(req *http.Request) (*http.Response, error) {
})
case path == "/wiki/rest/api/space/ENG/content/page":
a.listCalls++
+ if !a.endlessNext {
+ return a.jsonResponse(map[string]any{
+ "results": a.pageMaps(),
+ })
+ }
+ results := a.pageMaps()
+ if a.listCalls > 1 {
+ results = []any{}
+ }
+ next := fmt.Sprintf("/rest/api/space/ENG/content/page?limit=100&start=%d", a.listCalls*100)
return a.jsonResponse(map[string]any{
- "results": a.pageMaps(),
+ "results": results,
+ "_links": map[string]any{"next": next},
})
case strings.HasPrefix(path, "/wiki/rest/api/content/"):
if req.URL.Query().Get("expand") != "body.view,version,space" {
@@ -387,3 +402,37 @@ func TestListResourcesKeepsInvalidWebUI(t *testing.T) {
t.Fatalf("ListResources() with hostile webui = %#v, %v", resources, err)
}
}
+
+func TestFetchStreamKeepsUnreachedPagesWhenServerListingNeverEnds(t *testing.T) {
+ for _, full := range []bool{false, true} {
+ t.Run(fmt.Sprintf("full=%v", full), func(t *testing.T) {
+ api := &streamAPI{endlessNext: true, pages: []streamPage{{id: "p1", title: "Page", version: 2}}}
+ h := &captureHandler{}
+ old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"})
+ connector := newStreamConnector(api)
+ fetch := connector.FetchStream
+ if full {
+ fetch = connector.FetchFullStream
+ }
+ got, err := fetch(context.Background(), streamConfig(), old, h)
+ if err != nil {
+ t.Fatalf("fetch: %v", err)
+ }
+ if api.listCalls != 1+maxEmptyServerPages {
+ t.Fatalf("listCalls = %d, want %d", api.listCalls, 1+maxEmptyServerPages)
+ }
+ for _, item := range h.items {
+ if item.IsDeleted {
+ t.Fatalf("cut-short listing emitted a deletion for %s", item.ExternalID)
+ }
+ }
+ if len(h.items) != 1 || h.items[0].ExternalID != "p1" {
+ t.Fatalf("items = %#v, want the changed page p1 only", h.items)
+ }
+ pages := decodeCursor(got).SpacePages["1"]
+ if pages["p1"] != "v:2" || pages["p2"] != "v:1" {
+ t.Fatalf("cursor pages = %#v, want p1 advanced and unreached p2 kept", pages)
+ }
+ })
+ }
+}