Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(confluence): confirm unlisted pages before deleting after a cut-short listing

wizardchen503f1abfb4e7
diff --git a/internal/datasource/connector/confluence/client.go b/internal/datasource/connector/confluence/client.go
index 05c9d977f..ec156e382 100644
--- a/internal/datasource/connector/confluence/client.go
+++ b/internal/datasource/connector/confluence/client.go
@@ -3,6 +3,7 @@ package confluence
import (
"context"
"encoding/json"
+ "errors"
"fmt"
"io"
"net/http"
@@ -26,6 +27,9 @@ const (
// next past the end of a space (#3596); permission filtering can also
// empty one page mid-listing, so a single empty page is not the end.
maxEmptyServerPages = 3
+ // maxDeletionProbes caps how many unlisted pages one sync looks up
+ // individually after a cut-short listing; the rest wait for the next run.
+ maxDeletionProbes = 200
)
type apiError struct {
@@ -367,6 +371,31 @@ func (c *client) pages(ctx context.Context, s space) (pages []page, complete boo
return all, complete, err
}
+// pageInSpace reports whether a Server page is still a current page of the
+// space. Confluence answers 404 for a page that was deleted or moved to the
+// trash; a page that is no longer current or now belongs to another space is
+// gone from this one too, matching what a complete listing would omit.
+func (c *client) pageInSpace(ctx context.Context, id, spaceKey string) (bool, error) {
+ var result struct {
+ Status string `json:"status"`
+ Space struct {
+ Key string `json:"key"`
+ } `json:"space"`
+ }
+ err := c.get(ctx, "/rest/api/content/"+url.PathEscape(id)+"?expand=space", &result)
+ var apiErr *apiError
+ if errors.As(err, &apiErr) && apiErr.status == http.StatusNotFound {
+ return false, nil
+ }
+ if err != nil {
+ return false, err
+ }
+ if result.Status != "" && result.Status != "current" {
+ return false, nil
+ }
+ return result.Space.Key == "" || result.Space.Key == spaceKey, nil
+}
+
func (c *client) body(ctx context.Context, id string) (pageBody, error) {
if c.cfg.cloud() {
var result pageBody
diff --git a/internal/datasource/connector/confluence/client_pages_test.go b/internal/datasource/connector/confluence/client_pages_test.go
index a0407e871..1a7df281c 100644
--- a/internal/datasource/connector/confluence/client_pages_test.go
+++ b/internal/datasource/connector/confluence/client_pages_test.go
@@ -177,3 +177,34 @@ func TestPagesCloudFollowsNextAndKeepsDepth(t *testing.T) {
t.Fatalf("depth query = %#v", depths)
}
}
+
+func TestPageInSpace(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Query().Get("expand") != "space" {
+ t.Errorf("expand = %q, want space", r.URL.Query().Get("expand"))
+ }
+ w.Header().Set("Content-Type", "application/json")
+ switch r.URL.Path {
+ case "/rest/api/content/current":
+ _, _ = w.Write([]byte(`{"id": "current", "status": "current", "space": {"key": "ENG"}}`))
+ case "/rest/api/content/trashed":
+ _, _ = w.Write([]byte(`{"id": "trashed", "status": "trashed", "space": {"key": "ENG"}}`))
+ case "/rest/api/content/moved":
+ _, _ = w.Write([]byte(`{"id": "moved", "status": "current", "space": {"key": "OPS"}}`))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ t.Cleanup(server.Close)
+
+ c := &client{cfg: config{baseURL: server.URL}, http: server.Client()}
+ for id, want := range map[string]bool{"current": true, "trashed": false, "moved": false, "deleted": false} {
+ got, err := c.pageInSpace(context.Background(), id, "ENG")
+ if err != nil {
+ t.Fatalf("%s: %v", id, err)
+ }
+ if got != want {
+ t.Errorf("pageInSpace(%s) = %v, want %v", id, got, want)
+ }
+ }
+}
diff --git a/internal/datasource/connector/confluence/connector.go b/internal/datasource/connector/confluence/connector.go
index b753484a8..fe2a6f5ca 100644
--- a/internal/datasource/connector/confluence/connector.go
+++ b/internal/datasource/connector/confluence/connector.go
@@ -182,7 +182,7 @@ func (c *Connector) fetchStream(
if !complete {
logger.Warnf(ctx,
"[Confluence] space %s: listing stopped after %d empty pages that still linked onward; "+
- "imported %d pages and skipped deletion reconciliation",
+ "listed %d pages, checking unlisted ones one by one before deleting",
s.Key, maxEmptyServerPages, len(pages))
}
priorPages, hadBaseline := baseline.SpacePages[resourceID]
@@ -229,13 +229,8 @@ func (c *Connector) fetchStream(
}
}
if !complete {
- // A cut-short listing proves nothing about the pages it did not
- // reach: keep tracking them so a later complete listing can still
- // reconcile them, and delete nothing now.
- for id, version := range priorPages {
- if _, exists := seen[id]; !exists {
- next.SpacePages[resourceID][id] = version
- }
+ if err := reconcileUnlisted(ctx, client, h, s, resourceID, priorPages, seen, &next); err != nil {
+ return nil, err
}
} else if hadBaseline {
if len(pages) == 0 && len(priorPages) > 0 {
@@ -260,15 +255,7 @@ func (c *Connector) fetchStream(
if _, exists := seen[id]; exists {
continue
}
- deleted := types.FetchedItem{
- ExternalID: id, IsDeleted: true, SourceResourceID: resourceID,
- Metadata: map[string]string{"channel": types.ChannelConfluence},
- }
- if err := h.Emit(ctx, deleted); err != nil {
- return nil, err
- }
- delete(next.SpacePages[resourceID], id)
- if err := h.Checkpoint(ctx, next.syncCursor()); err != nil {
+ if err := emitDeleted(ctx, h, &next, resourceID, id); err != nil {
return nil, err
}
}
@@ -279,6 +266,63 @@ func (c *Connector) fetchStream(
return next.syncCursor(), nil
}
+// reconcileUnlisted handles the pages a cut-short listing did not return. The
+// listing proves nothing about them, so each one is looked up on its own and
+// deleted only when Confluence confirms it is gone from the space. Pages that
+// are still there, could not be checked, or exceed the per-run probe budget
+// stay tracked with their prior version and are checked again next run.
+func reconcileUnlisted(
+ ctx context.Context,
+ c *client,
+ h datasource.StreamHandler,
+ s space,
+ resourceID string,
+ priorPages map[string]string,
+ seen map[string]struct{},
+ next *cursor,
+) error {
+ probed := 0
+ for id, version := range priorPages {
+ if _, exists := seen[id]; exists {
+ continue
+ }
+ if probed >= maxDeletionProbes {
+ next.SpacePages[resourceID][id] = version
+ continue
+ }
+ probed++
+ present, err := c.pageInSpace(ctx, id, s.Key)
+ if err != nil {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ logger.Warnf(ctx, "[Confluence] space %s: could not check unlisted page %s, keeping it: %v", s.Key, id, err)
+ next.SpacePages[resourceID][id] = version
+ continue
+ }
+ if present {
+ next.SpacePages[resourceID][id] = version
+ continue
+ }
+ if err := emitDeleted(ctx, h, next, resourceID, id); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func emitDeleted(ctx context.Context, h datasource.StreamHandler, next *cursor, resourceID, id string) error {
+ deleted := types.FetchedItem{
+ ExternalID: id, IsDeleted: true, SourceResourceID: resourceID,
+ Metadata: map[string]string{"channel": types.ChannelConfluence},
+ }
+ if err := h.Emit(ctx, deleted); err != nil {
+ return err
+ }
+ delete(next.SpacePages[resourceID], id)
+ return h.Checkpoint(ctx, next.syncCursor())
+}
+
func failedPageItem(resourceID string, summary page, err error) types.FetchedItem {
code, reason := classifyConfluenceError(err)
return types.FetchedItem{
diff --git a/internal/datasource/connector/confluence/connector_stream_test.go b/internal/datasource/connector/confluence/connector_stream_test.go
index 07a024b0e..2f38c787f 100644
--- a/internal/datasource/connector/confluence/connector_stream_test.go
+++ b/internal/datasource/connector/confluence/connector_stream_test.go
@@ -34,6 +34,16 @@ type streamAPI struct {
// endlessNext mimics Server/Data Center builds that keep linking to a
// next page after the last one, returning empty results forever (#3596).
endlessNext bool
+ // probe answers the per-page existence check (GET content/{id}?expand=space)
+ // for pages that are not in the listing: an HTTP status, or 200 with the
+ // space key the page now belongs to.
+ probe map[string]probeReply
+ probeCalls int
+}
+
+type probeReply struct {
+ status int
+ spaceKey string
}
func (a *streamAPI) jsonResponse(value any) (*http.Response, error) {
@@ -84,6 +94,23 @@ func (a *streamAPI) response(req *http.Request) (*http.Response, error) {
"results": results,
"_links": map[string]any{"next": next},
})
+ case strings.HasPrefix(path, "/wiki/rest/api/content/") && req.URL.Query().Get("expand") == "space":
+ a.probeCalls++
+ id := strings.TrimPrefix(path, "/wiki/rest/api/content/")
+ reply, ok := a.probe[id]
+ if !ok {
+ return nil, errors.New("unexpected existence check for page " + id)
+ }
+ if reply.status != http.StatusOK {
+ return &http.Response{
+ StatusCode: reply.status,
+ Header: make(http.Header),
+ Body: io.NopCloser(strings.NewReader("{}")),
+ }, nil
+ }
+ return a.jsonResponse(map[string]any{
+ "id": id, "status": "current", "space": map[string]any{"key": reply.spaceKey},
+ })
case strings.HasPrefix(path, "/wiki/rest/api/content/"):
if req.URL.Query().Get("expand") != "body.view,version,space" {
return nil, errors.New("page body did not request rendered body.view")
@@ -403,12 +430,23 @@ func TestListResourcesKeepsInvalidWebUI(t *testing.T) {
}
}
-func TestFetchStreamKeepsUnreachedPagesWhenServerListingNeverEnds(t *testing.T) {
+func TestFetchStreamChecksUnlistedPagesWhenServerListingNeverEnds(t *testing.T) {
for _, full := range []bool{false, true} {
t.Run(fmt.Sprintf("full=%v", full), func(t *testing.T) {
- api := &streamAPI{endlessNext: true, pages: []streamPage{{id: "p1", title: "Page", version: 2}}}
+ api := &streamAPI{
+ endlessNext: true,
+ pages: []streamPage{{id: "p1", title: "Page", version: 2}},
+ probe: map[string]probeReply{
+ "gone": {status: http.StatusNotFound},
+ "moved": {status: http.StatusOK, spaceKey: "OPS"},
+ "kept": {status: http.StatusOK, spaceKey: "ENG"},
+ "denied": {status: http.StatusForbidden},
+ },
+ }
h := &captureHandler{}
- old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"})
+ old := streamCursor(map[string]string{
+ "p1": "v:1", "gone": "v:1", "moved": "v:1", "kept": "v:3", "denied": "v:1",
+ })
connector := newStreamConnector(api)
fetch := connector.FetchStream
if full {
@@ -421,17 +459,27 @@ func TestFetchStreamKeepsUnreachedPagesWhenServerListingNeverEnds(t *testing.T)
if api.listCalls != 1+maxEmptyServerPages {
t.Fatalf("listCalls = %d, want %d", api.listCalls, 1+maxEmptyServerPages)
}
+ if api.probeCalls != 4 {
+ t.Fatalf("probeCalls = %d, want one per unlisted page", api.probeCalls)
+ }
+ deleted := map[string]bool{}
for _, item := range h.items {
if item.IsDeleted {
- t.Fatalf("cut-short listing emitted a deletion for %s", item.ExternalID)
+ deleted[item.ExternalID] = true
}
}
- if len(h.items) != 1 || h.items[0].ExternalID != "p1" {
- t.Fatalf("items = %#v, want the changed page p1 only", h.items)
+ if len(deleted) != 2 || !deleted["gone"] || !deleted["moved"] {
+ t.Fatalf("deleted = %v, want only the pages Confluence confirmed gone from the space", deleted)
}
pages := decodeCursor(got).SpacePages["1"]
- if pages["p1"] != "v:2" || pages["p2"] != "v:1" {
- t.Fatalf("cursor pages = %#v, want p1 advanced and unreached p2 kept", pages)
+ want := map[string]string{"p1": "v:2", "kept": "v:3", "denied": "v:1"}
+ if len(pages) != len(want) {
+ t.Fatalf("cursor pages = %#v, want %#v", pages, want)
+ }
+ for id, version := range want {
+ if pages[id] != version {
+ t.Fatalf("cursor pages = %#v, want %#v", pages, want)
+ }
}
})
}