Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(confluence): confirm unlisted pages before deleting after a cut-short listing
wizardchen503f1abfb4e7
diff --git a/internal/datasource/connector/confluence/client.go b/internal/datasource/connector/confluence/client.goindex 05c9d977f..ec156e382 100644--- a/internal/datasource/connector/confluence/client.go+++ b/internal/datasource/connector/confluence/client.go@@ -3,6 +3,7 @@ package confluenceimport ("context""encoding/json"+ "errors""fmt""io""net/http"@@ -26,6 +27,9 @@ const (// next past the end of a space (#3596); permission filtering can also// empty one page mid-listing, so a single empty page is not the end.maxEmptyServerPages = 3+ // maxDeletionProbes caps how many unlisted pages one sync looks up+ // individually after a cut-short listing; the rest wait for the next run.+ maxDeletionProbes = 200)type apiError struct {@@ -367,6 +371,31 @@ func (c *client) pages(ctx context.Context, s space) (pages []page, complete booreturn all, complete, err}+// pageInSpace reports whether a Server page is still a current page of the+// space. Confluence answers 404 for a page that was deleted or moved to the+// trash; a page that is no longer current or now belongs to another space is+// gone from this one too, matching what a complete listing would omit.+func (c *client) pageInSpace(ctx context.Context, id, spaceKey string) (bool, error) {+ var result struct {+ Status string `json:"status"`+ Space struct {+ Key string `json:"key"`+ } `json:"space"`+ }+ err := c.get(ctx, "/rest/api/content/"+url.PathEscape(id)+"?expand=space", &result)+ var apiErr *apiError+ if errors.As(err, &apiErr) && apiErr.status == http.StatusNotFound {+ return false, nil+ }+ if err != nil {+ return false, err+ }+ if result.Status != "" && result.Status != "current" {+ return false, nil+ }+ return result.Space.Key == "" || result.Space.Key == spaceKey, nil+}+func (c *client) body(ctx context.Context, id string) (pageBody, error) {if c.cfg.cloud() {var result pageBodydiff --git a/internal/datasource/connector/confluence/client_pages_test.go b/internal/datasource/connector/confluence/client_pages_test.goindex a0407e871..1a7df281c 100644--- a/internal/datasource/connector/confluence/client_pages_test.go+++ b/internal/datasource/connector/confluence/client_pages_test.go@@ -177,3 +177,34 @@ func TestPagesCloudFollowsNextAndKeepsDepth(t *testing.T) {t.Fatalf("depth query = %#v", depths)}}++func TestPageInSpace(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Query().Get("expand") != "space" {+ t.Errorf("expand = %q, want space", r.URL.Query().Get("expand"))+ }+ w.Header().Set("Content-Type", "application/json")+ switch r.URL.Path {+ case "/rest/api/content/current":+ _, _ = w.Write([]byte(`{"id": "current", "status": "current", "space": {"key": "ENG"}}`))+ case "/rest/api/content/trashed":+ _, _ = w.Write([]byte(`{"id": "trashed", "status": "trashed", "space": {"key": "ENG"}}`))+ case "/rest/api/content/moved":+ _, _ = w.Write([]byte(`{"id": "moved", "status": "current", "space": {"key": "OPS"}}`))+ default:+ http.NotFound(w, r)+ }+ }))+ t.Cleanup(server.Close)++ c := &client{cfg: config{baseURL: server.URL}, http: server.Client()}+ for id, want := range map[string]bool{"current": true, "trashed": false, "moved": false, "deleted": false} {+ got, err := c.pageInSpace(context.Background(), id, "ENG")+ if err != nil {+ t.Fatalf("%s: %v", id, err)+ }+ if got != want {+ t.Errorf("pageInSpace(%s) = %v, want %v", id, got, want)+ }+ }+}diff --git a/internal/datasource/connector/confluence/connector.go b/internal/datasource/connector/confluence/connector.goindex b753484a8..fe2a6f5ca 100644--- a/internal/datasource/connector/confluence/connector.go+++ b/internal/datasource/connector/confluence/connector.go@@ -182,7 +182,7 @@ func (c *Connector) fetchStream(if !complete {logger.Warnf(ctx,"[Confluence] space %s: listing stopped after %d empty pages that still linked onward; "+- "imported %d pages and skipped deletion reconciliation",+ "listed %d pages, checking unlisted ones one by one before deleting",s.Key, maxEmptyServerPages, len(pages))}priorPages, hadBaseline := baseline.SpacePages[resourceID]@@ -229,13 +229,8 @@ func (c *Connector) fetchStream(}}if !complete {- // A cut-short listing proves nothing about the pages it did not- // reach: keep tracking them so a later complete listing can still- // reconcile them, and delete nothing now.- for id, version := range priorPages {- if _, exists := seen[id]; !exists {- next.SpacePages[resourceID][id] = version- }+ if err := reconcileUnlisted(ctx, client, h, s, resourceID, priorPages, seen, &next); err != nil {+ return nil, err}} else if hadBaseline {if len(pages) == 0 && len(priorPages) > 0 {@@ -260,15 +255,7 @@ func (c *Connector) fetchStream(if _, exists := seen[id]; exists {continue}- deleted := types.FetchedItem{- ExternalID: id, IsDeleted: true, SourceResourceID: resourceID,- Metadata: map[string]string{"channel": types.ChannelConfluence},- }- if err := h.Emit(ctx, deleted); err != nil {- return nil, err- }- delete(next.SpacePages[resourceID], id)- if err := h.Checkpoint(ctx, next.syncCursor()); err != nil {+ if err := emitDeleted(ctx, h, &next, resourceID, id); err != nil {return nil, err}}@@ -279,6 +266,63 @@ func (c *Connector) fetchStream(return next.syncCursor(), nil}+// reconcileUnlisted handles the pages a cut-short listing did not return. The+// listing proves nothing about them, so each one is looked up on its own and+// deleted only when Confluence confirms it is gone from the space. Pages that+// are still there, could not be checked, or exceed the per-run probe budget+// stay tracked with their prior version and are checked again next run.+func reconcileUnlisted(+ ctx context.Context,+ c *client,+ h datasource.StreamHandler,+ s space,+ resourceID string,+ priorPages map[string]string,+ seen map[string]struct{},+ next *cursor,+) error {+ probed := 0+ for id, version := range priorPages {+ if _, exists := seen[id]; exists {+ continue+ }+ if probed >= maxDeletionProbes {+ next.SpacePages[resourceID][id] = version+ continue+ }+ probed+++ present, err := c.pageInSpace(ctx, id, s.Key)+ if err != nil {+ if ctx.Err() != nil {+ return ctx.Err()+ }+ logger.Warnf(ctx, "[Confluence] space %s: could not check unlisted page %s, keeping it: %v", s.Key, id, err)+ next.SpacePages[resourceID][id] = version+ continue+ }+ if present {+ next.SpacePages[resourceID][id] = version+ continue+ }+ if err := emitDeleted(ctx, h, next, resourceID, id); err != nil {+ return err+ }+ }+ return nil+}++func emitDeleted(ctx context.Context, h datasource.StreamHandler, next *cursor, resourceID, id string) error {+ deleted := types.FetchedItem{+ ExternalID: id, IsDeleted: true, SourceResourceID: resourceID,+ Metadata: map[string]string{"channel": types.ChannelConfluence},+ }+ if err := h.Emit(ctx, deleted); err != nil {+ return err+ }+ delete(next.SpacePages[resourceID], id)+ return h.Checkpoint(ctx, next.syncCursor())+}+func failedPageItem(resourceID string, summary page, err error) types.FetchedItem {code, reason := classifyConfluenceError(err)return types.FetchedItem{diff --git a/internal/datasource/connector/confluence/connector_stream_test.go b/internal/datasource/connector/confluence/connector_stream_test.goindex 07a024b0e..2f38c787f 100644--- a/internal/datasource/connector/confluence/connector_stream_test.go+++ b/internal/datasource/connector/confluence/connector_stream_test.go@@ -34,6 +34,16 @@ type streamAPI struct {// endlessNext mimics Server/Data Center builds that keep linking to a// next page after the last one, returning empty results forever (#3596).endlessNext bool+ // probe answers the per-page existence check (GET content/{id}?expand=space)+ // for pages that are not in the listing: an HTTP status, or 200 with the+ // space key the page now belongs to.+ probe map[string]probeReply+ probeCalls int+}++type probeReply struct {+ status int+ spaceKey string}func (a *streamAPI) jsonResponse(value any) (*http.Response, error) {@@ -84,6 +94,23 @@ func (a *streamAPI) response(req *http.Request) (*http.Response, error) {"results": results,"_links": map[string]any{"next": next},})+ case strings.HasPrefix(path, "/wiki/rest/api/content/") && req.URL.Query().Get("expand") == "space":+ a.probeCalls+++ id := strings.TrimPrefix(path, "/wiki/rest/api/content/")+ reply, ok := a.probe[id]+ if !ok {+ return nil, errors.New("unexpected existence check for page " + id)+ }+ if reply.status != http.StatusOK {+ return &http.Response{+ StatusCode: reply.status,+ Header: make(http.Header),+ Body: io.NopCloser(strings.NewReader("{}")),+ }, nil+ }+ return a.jsonResponse(map[string]any{+ "id": id, "status": "current", "space": map[string]any{"key": reply.spaceKey},+ })case strings.HasPrefix(path, "/wiki/rest/api/content/"):if req.URL.Query().Get("expand") != "body.view,version,space" {return nil, errors.New("page body did not request rendered body.view")@@ -403,12 +430,23 @@ func TestListResourcesKeepsInvalidWebUI(t *testing.T) {}}-func TestFetchStreamKeepsUnreachedPagesWhenServerListingNeverEnds(t *testing.T) {+func TestFetchStreamChecksUnlistedPagesWhenServerListingNeverEnds(t *testing.T) {for _, full := range []bool{false, true} {t.Run(fmt.Sprintf("full=%v", full), func(t *testing.T) {- api := &streamAPI{endlessNext: true, pages: []streamPage{{id: "p1", title: "Page", version: 2}}}+ api := &streamAPI{+ endlessNext: true,+ pages: []streamPage{{id: "p1", title: "Page", version: 2}},+ probe: map[string]probeReply{+ "gone": {status: http.StatusNotFound},+ "moved": {status: http.StatusOK, spaceKey: "OPS"},+ "kept": {status: http.StatusOK, spaceKey: "ENG"},+ "denied": {status: http.StatusForbidden},+ },+ }h := &captureHandler{}- old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"})+ old := streamCursor(map[string]string{+ "p1": "v:1", "gone": "v:1", "moved": "v:1", "kept": "v:3", "denied": "v:1",+ })connector := newStreamConnector(api)fetch := connector.FetchStreamif full {@@ -421,17 +459,27 @@ func TestFetchStreamKeepsUnreachedPagesWhenServerListingNeverEnds(t *testing.T)if api.listCalls != 1+maxEmptyServerPages {t.Fatalf("listCalls = %d, want %d", api.listCalls, 1+maxEmptyServerPages)}+ if api.probeCalls != 4 {+ t.Fatalf("probeCalls = %d, want one per unlisted page", api.probeCalls)+ }+ deleted := map[string]bool{}for _, item := range h.items {if item.IsDeleted {- t.Fatalf("cut-short listing emitted a deletion for %s", item.ExternalID)+ deleted[item.ExternalID] = true}}- if len(h.items) != 1 || h.items[0].ExternalID != "p1" {- t.Fatalf("items = %#v, want the changed page p1 only", h.items)+ if len(deleted) != 2 || !deleted["gone"] || !deleted["moved"] {+ t.Fatalf("deleted = %v, want only the pages Confluence confirmed gone from the space", deleted)}pages := decodeCursor(got).SpacePages["1"]- if pages["p1"] != "v:2" || pages["p2"] != "v:1" {- t.Fatalf("cursor pages = %#v, want p1 advanced and unreached p2 kept", pages)+ want := map[string]string{"p1": "v:2", "kept": "v:3", "denied": "v:1"}+ if len(pages) != len(want) {+ t.Fatalf("cursor pages = %#v, want %#v", pages, want)+ }+ for id, version := range want {+ if pages[id] != version {+ t.Fatalf("cursor pages = %#v, want %#v", pages, want)+ }}})}