Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
feat(confluence): add directory sync and private image assets (#3319)
jacentsaoe7c27ccc9e7f
diff --git a/frontend/src/api/datasource/index.ts b/frontend/src/api/datasource/index.tsindex 1ac7ac6d2..19096307f 100644--- a/frontend/src/api/datasource/index.ts+++ b/frontend/src/api/datasource/index.ts@@ -82,6 +82,8 @@ export interface Resource {url: stringparent_id?: stringhas_children?: boolean+ /** Connector-specific hints, e.g. Confluence Cloud's top-level-container limitation. */+ metadata?: Record<string, any>}// --- API calls ---diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.tsindex 1c7e6dcf1..681150b9f 100755--- a/frontend/src/i18n/locales/en-US.ts+++ b/frontend/src/i18n/locales/en-US.ts@@ -6979,6 +6979,9 @@ export default {authHeaders: 'Custom headers (optional)',authHeadersHint: 'For private feeds. One per line in "Name: Value" form, e.g. Authorization: Bearer xxxx'},+ confluence: {+ cloudFolderLimitation: 'Confluence Cloud cannot list pages stored directly under top-level folders in this space; selecting the whole space still syncs them.'+ },comingSoon: 'Coming soon',docHint: 'Get credentials at:',openDoc: 'Open documentation',diff --git a/frontend/src/i18n/locales/ja-JP.ts b/frontend/src/i18n/locales/ja-JP.tsindex ac272ac64..308b3c0f7 100644--- a/frontend/src/i18n/locales/ja-JP.ts+++ b/frontend/src/i18n/locales/ja-JP.ts@@ -6979,6 +6979,9 @@ export default {authHeaders: 'カスタムヘッダー(任意)',authHeadersHint: '非公開フィード用です。1行に1つ「名前: 値」の形式で入力します(例: Authorization: Bearer xxxx)'},+ confluence: {+ cloudFolderLimitation: 'Confluence Cloud では、スペース直下のフォルダーなどコンテナ配下のページをここに一覧できません。スペース全体を選択すれば同期されます。'+ },comingSoon: '近日対応予定',docHint: '認証情報の取得先:',openDoc: 'ドキュメントを開く',diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.tsindex ead7f7139..9bba3b96a 100755--- a/frontend/src/i18n/locales/ko-KR.ts+++ b/frontend/src/i18n/locales/ko-KR.ts@@ -1125,6 +1125,9 @@ export default {authHeaders: '사용자 지정 헤더 (선택)',authHeadersHint: '비공개 피드 접근용. 한 줄에 하나씩 「이름: 값」 형식으로 입력하세요. 예: Authorization: Bearer xxxx'},+ confluence: {+ cloudFolderLimitation: 'Confluence Cloud는 아직 스페이스 최상위 폴더 등 컨테이너 아래의 페이지를 여기에 나열할 수 없습니다. 스페이스 전체를 선택하면 함께 동기화됩니다.'+ },connectorDesc: {feishu: '페이슈 위키에서 문서, 스프레드시트, 파일 동기화',lark: 'Lark 위키에서 문서, 스프레드시트, 파일 동기화',diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.tsindex 4f075e0bd..d5b8fb166 100755--- a/frontend/src/i18n/locales/ru-RU.ts+++ b/frontend/src/i18n/locales/ru-RU.ts@@ -1125,6 +1125,9 @@ export default {authHeaders: 'Пользовательские заголовки (необязательно)',authHeadersHint: 'Для приватных лент. По одному в строке в формате «Имя: Значение», например Authorization: Bearer xxxx'},+ confluence: {+ cloudFolderLimitation: 'Confluence Cloud пока не может показать здесь страницы из папок верхнего уровня пространства; при выборе всего пространства они будут синхронизированы.'+ },connectorDesc: {feishu: 'Синхронизация документов, таблиц и файлов из Feishu Wiki',lark: 'Синхронизация документов, таблиц и файлов из Lark Wiki',diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.tsindex 073f86492..5497981a0 100755--- a/frontend/src/i18n/locales/zh-CN.ts+++ b/frontend/src/i18n/locales/zh-CN.ts@@ -1127,6 +1127,9 @@ export default {authHeaders: '自定义请求头(可选)',authHeadersHint: '用于访问私有订阅源,每行一个,格式为「名称: 值」,例如 Authorization: Bearer xxxx'},+ confluence: {+ cloudFolderLimitation: 'Confluence Cloud 暂不支持在此列出空间顶层文件夹等容器下的页面;选择整个空间仍会同步它们。'+ },connectorDesc: {feishu: '同步飞书知识库中的文档、表格、文件',lark: '同步 Lark 知识库中的文档、表格、文件(飞书国际版)',diff --git a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.test.ts b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.test.tsindex 43c46c765..f9f2d1267 100644--- a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.test.ts+++ b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.test.ts@@ -157,3 +157,25 @@ test('new GitLab data sources continue to test credentials without persistence',assert.deepEqual(f.calls.map(call => call.method), ['validateCredentials'])} finally { f.close() }})++test('Cloud hierarchy limitation stays visible after an empty space expansion', async () => {+ const f = await fixture()+ try {+ f.vm.resources = [{+ external_id: 'space-1',+ name: 'Cloud space',+ type: 'space',+ has_children: true,+ metadata: { hierarchy_limitation: 'cloud_top_level_containers' },+ }]+ f.vm.expandedResourceIds = new Set(['space-1'])+ await nextTick()+ assert.equal(f.vm.visibleTree.some((row: any) => row.noticeAfter), true)++ f.vm.expandedResourceIds = new Set()+ f.vm.loadedChildrenIds = new Set(['space-1'])+ f.vm.resources = [{ ...f.vm.resources[0], has_children: false }]+ await nextTick()+ assert.equal(f.vm.visibleTree.some((row: any) => row.noticeAfter), true)+ } finally { f.close() }+})diff --git a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vueindex 44b16aa43..cb60d2bb6 100644--- a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue+++ b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue@@ -449,6 +449,16 @@ async function ensureChildrenLoaded(id: string) {if (!existing.has(c.external_id)) merged.push(c)}resources.value = merged+ } else {+ // Connectors may conservatively advertise HasChildren to avoid an N+1+ // probe. Once lazy loading proves this is a leaf, collapse it and retain+ // that fact so another expand cannot trigger another empty request.+ resources.value = resources.value.map(r => r.external_id === id+ ? { ...r, has_children: false }+ : r)+ const expanded = new Set(expandedResourceIds.value)+ expanded.delete(id)+ expandedResourceIds.value = expanded}loadedChildrenIds.value = new Set(loadedChildrenIds.value).add(id)} catch (e: any) {@@ -466,13 +476,21 @@ async function ensureChildrenLoaded(id: string) {const visibleTree = computed(() => {const roots = resources.value.filter(r => !r.parent_id)- const result: { resource: Resource; depth: number }[] = []+ const result: { resource: Resource; depth: number; noticeAfter?: boolean }[] = []function walk(items: Resource[], depth: number) {for (const r of items) {result.push({ resource: r, depth })if (r.has_children && expandedResourceIds.value.has(r.external_id)) {walk(childrenMap.value.get(r.external_id) || [], depth + 1)}+ // Keep this visible after an empty root listing is recognized as a leaf.+ if (+ r.metadata?.hierarchy_limitation === 'cloud_top_level_containers' &&+ (expandedResourceIds.value.has(r.external_id) ||+ (!r.has_children && loadedChildrenIds.value.has(r.external_id)))+ ) {+ result.push({ resource: r, depth: depth + 1, noticeAfter: true })+ }}}walk(roots, 0)@@ -1720,9 +1738,12 @@ const drawerConfirmText = computed(() => {</div></div><div class="resource-picker__list" role="tree">+ <template+ v-for="{ resource: r, depth, noticeAfter } in visibleTree"+ :key="noticeAfter ? `${r.external_id}__notice` : r.external_id"+ ><div- v-for="{ resource: r, depth } in visibleTree"- :key="r.external_id"+ v-if="!noticeAfter"class="resource-picker__row":class="{'is-checked': resourceRowState(r.external_id) === 'checked',@@ -1787,6 +1808,14 @@ const drawerConfirmText = computed(() => {>{{ resourceTypeLabel(r.type) }}</span></span></div>+ <p+ v-else+ class="resource-picker__notice"+ :style="{ '--depth': depth }"+ >+ {{ t('datasource.confluence.cloudFolderLimitation') }}+ </p>+ </template></div></div><div v-else class="ds-resource-empty">@@ -2576,6 +2605,19 @@ const drawerConfirmText = computed(() => {background: color-mix(in srgb, var(--td-text-color-placeholder) 8%, transparent);}+.resource-picker__notice {+ --depth: 0;+ margin: 0 0 2px;+ padding: 4px 8px 4px calc(8px + var(--depth) * 14px);+ font-size: var(--app-text-xs);+ line-height: 1.5;+ color: var(--td-text-color-placeholder);+}++.resource-picker__notice:last-child {+ margin-bottom: 0;+}+/* --- Step 2: empty state --- */.ds-resource-empty {text-align: center;diff --git a/internal/datasource/connector/confluence/asset_resolver.go b/internal/datasource/connector/confluence/asset_resolver.gonew file mode 100644index 000000000..f8b06f606--- /dev/null+++ b/internal/datasource/connector/confluence/asset_resolver.go@@ -0,0 +1,301 @@+package confluence++import (+ "context"+ "encoding/base64"+ "errors"+ "html"+ "net/url"+ "regexp"+ "strconv"+ "strings"+ "sync"++ "github.com/Tencent/WeKnora/internal/logger"+)++const (+ // maxPageImages is the total data URI image budget one page may carry into the+ // downstream docparser, which enforces a single per-page limit (maxRemoteImages)+ // across every data:image URI regardless of origin. The budget is shared: it+ // covers data URI images already present in the page plus the ones the resolver+ // generates, and a repeated image consumes it per occurrence, not per unique URL.+ // Downloads stay deduplicated by URL, and only occurrences within the remaining+ // budget are collected, so the connector never emits more data URIs than the+ // pipeline will keep.+ maxPageImages = 30+ // maxPageInlineBytes caps the total size of the data URIs one page may carry,+ // both in the download cache and in the rewritten HTML. Without it an extreme+ // page (maxPageImages occurrences of maxImageBytes images) would inline hundreds+ // of MB of base64, copied again by every downstream conversion. Images past the+ // budget keep their original src, like any other image the resolver skips.+ maxPageInlineBytes = 50 << 20+ // imageDownloadConcurrency bounds in-flight image downloads per page so an+ // image-heavy page cannot exhaust HTTP connections or spike memory.+ imageDownloadConcurrency = 4+)++// errImagePageBudget marks an image dropped because the page's inline budget+// (maxPageInlineBytes) was already used up.+var errImagePageBudget = errors.New("confluence page image budget exhausted")++// imgTagRe matches a full <img ...> start tag, tolerating '>' inside quoted+// attribute values. The \b keeps it from matching <image>.+var imgTagRe = regexp.MustCompile(`(?i)<img\b(?:[^>"']|"[^"]*"|'[^']*')*>`)++// srcAttrRe captures the src attribute value. The leading \s and the value group+// keep it from matching data-src/srcset, and support double-quoted,+// single-quoted, or bare values. Group 1 is the "src=" prefix, group 2 the value.+var srcAttrRe = regexp.MustCompile(`(?i)(\ssrc\s*=\s*)("[^"]*"|'[^']*'|[^\s"'>]+)`)++type imgAction uint8++const (+ imgKeep imgAction = iota // empty, data:, external, or non-http scheme: leave untouched+ imgDownload // same-origin private candidate+)++// assetResolver rewrites same-origin private Confluence images in a page's+// body.view HTML into inline data URIs. Persistence and resource:// rewriting+// are left entirely to the downstream docparser image pipeline; the resolver+// only performs the one step that pipeline cannot do on its own: an+// authenticated fetch of Confluence-private bytes.+type assetResolver struct {+ client *client+}++func newAssetResolver(c *client) *assetResolver { return &assetResolver{client: c} }++// Resolve returns content with same-origin private <img src> occurrences replaced+// by data URIs, within the per-page budget that Resolve shares with any data URI+// images already in the page. It is best-effort: an image that is skipped, falls+// past the remaining budget, or fails to download keeps its original src, and image+// problems never fail the page. Downloads are deduplicated by URL, but the budget+// counts written data URIs (occurrences) to match the downstream pipeline, which+// caps every data:image URI per page regardless of where it came from.+func (r *assetResolver) Resolve(ctx context.Context, content string) string {+ if r == nil || r.client == nil {+ return content+ }+ // Data URI images already in the page consume the same downstream budget as the+ // ones we generate, so reserve room for them before inlining any private image.+ available := maxPageImages - countDataURIImages(content)+ if available <= 0 {+ return content+ }+ candidates := r.collect(content, available)+ if len(candidates) == 0 {+ return content+ }+ cache := r.downloadAll(ctx, candidates)+ if len(cache) == 0 {+ return content+ }+ inlined, inlinedBytes := 0, 0+ budgetSpent := false+ return imgTagRe.ReplaceAllStringFunc(content, func(tag string) string {+ if inlined >= available || budgetSpent {+ return tag+ }+ src, ok := extractImageSrc(tag)+ if !ok {+ return tag+ }+ action, abs := r.classify(src)+ if action != imgDownload {+ return tag+ }+ dataURI := cache[abs]+ if dataURI == "" {+ return tag+ }+ if inlinedBytes+len(dataURI) > maxPageInlineBytes {+ budgetSpent = true+ return tag+ }+ inlined+++ inlinedBytes += len(dataURI)+ return replaceImageSrc(tag, dataURI)+ })+}++// collect returns the unique same-origin download candidates whose first+// occurrence falls within the remaining inline budget, in document order. Because+// Resolve writes at most budget data URIs, a URL first seen after that many+// occurrences can never be inlined, so it is not downloaded. Duplicates collapse to+// one entry, so a repeated image is fetched once yet may be inlined at every+// occurrence inside the budget.+func (r *assetResolver) collect(content string, budget int) []string {+ tags := imgTagRe.FindAllString(content, -1)+ order := make([]string, 0, len(tags))+ seen := make(map[string]struct{}, len(tags))+ occurrences := 0+ for _, tag := range tags {+ src, ok := extractImageSrc(tag)+ if !ok {+ continue+ }+ action, abs := r.classify(src)+ if action != imgDownload {+ continue+ }+ if occurrences >= budget {+ break+ }+ occurrences+++ if _, dup := seen[abs]; dup {+ continue+ }+ seen[abs] = struct{}{}+ order = append(order, abs)+ }+ return order+}++// countDataURIImages returns how many <img> tags in content already carry a+// data:image URI. The downstream docparser counts every data:image occurrence+// against the same per-page limit whether it came from the page or from us, so+// these must be reserved before the resolver inlines any private Confluence image.+func countDataURIImages(content string) int {+ count := 0+ for _, tag := range imgTagRe.FindAllString(content, -1) {+ src, ok := extractImageSrc(tag)+ if !ok {+ continue+ }+ if strings.HasPrefix(strings.ToLower(src), "data:image/") {+ count+++ }+ }+ return count+}++// downloadAll fetches each candidate with bounded concurrency and returns a+// URL-keyed cache of the successful ones as data URIs, holding at most+// maxPageInlineBytes of them. Failures and images past the budget are logged+// (redacted) and omitted, so the rewrite pass leaves their original src in place.+func (r *assetResolver) downloadAll(ctx context.Context, urls []string) map[string]string {+ cache := make(map[string]string, len(urls))+ cachedBytes := 0+ var mu sync.Mutex+ var wg sync.WaitGroup+ sem := make(chan struct{}, imageDownloadConcurrency)+ for _, u := range urls {+ if ctx.Err() != nil {+ break+ }+ wg.Add(1)+ sem <- struct{}{}+ go func(abs string) {+ defer wg.Done()+ defer func() { <-sem }()+ data, mime, err := r.client.downloadImage(ctx, abs)+ if err != nil {+ logImageFailure(ctx, abs, err)+ return+ }+ uri := "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(data)+ mu.Lock()+ fits := cachedBytes+len(uri) <= maxPageInlineBytes+ if fits {+ cache[abs] = uri+ cachedBytes += len(uri)+ }+ mu.Unlock()+ if !fits {+ logImageFailure(ctx, abs, errImagePageBudget)+ }+ }(u)+ }+ wg.Wait()+ return cache+}++// classify decides whether src is a same-origin private candidate to download or+// should be left untouched, returning the normalized absolute URL for downloads.+func (r *assetResolver) classify(src string) (imgAction, string) {+ s := strings.TrimSpace(src)+ if s == "" || strings.HasPrefix(strings.ToLower(s), "data:") {+ return imgKeep, ""+ }+ u, err := url.Parse(s)+ if err != nil {+ return imgKeep, ""+ }+ if u.Scheme != "" && u.Scheme != "http" && u.Scheme != "https" {+ // file:, javascript:, and any other scheme are never fetched.+ return imgKeep, ""+ }+ if u.Host != "" {+ if u.Scheme == "" {+ // Protocol-relative //host/path: only same-origin is eligible.+ base, berr := url.Parse(r.client.cfg.baseURL)+ if berr != nil || !strings.EqualFold(base.Host, u.Host) {+ return imgKeep, ""+ }+ } else if !r.client.sameOrigin(u) {+ // Absolute external http(s): keep, never send Confluence credentials.+ return imgKeep, ""+ }+ }+ abs, err := r.client.resolveEndpoint(s)+ if err != nil {+ return imgKeep, ""+ }+ return imgDownload, abs+}++// logImageFailure records a redacted warning: host and path only, never the+// query (which may carry signed tokens), the Authorization header, or the body.+func logImageFailure(ctx context.Context, abs string, err error) {+ host, path := abs, ""+ if u, perr := url.Parse(abs); perr == nil {+ host, path = u.Host, u.Path+ }+ reason := "download_failed"+ switch {+ case errors.Is(err, errImageTooLarge):+ reason = "oversize"+ case errors.Is(err, errImageNonImageContent):+ reason = "non_image_content_type"+ case errors.Is(err, errImageRedirectOffOrigin):+ reason = "cross_origin_redirect"+ case errors.Is(err, errImagePageBudget):+ reason = "page_budget"+ }+ var api *apiError+ if errors.As(err, &api) {+ reason = "status_" + strconv.Itoa(api.status)+ }+ logger.Warnf(ctx, "[Confluence] failed to resolve image host=%s path=%s reason=%s", host, path, reason)+}++// extractImageSrc returns the HTML-unescaped src value of an <img> tag.+func extractImageSrc(tag string) (string, bool) {+ m := srcAttrRe.FindStringSubmatch(tag)+ if m == nil {+ return "", false+ }+ value := strings.TrimSpace(m[2])+ if len(value) >= 2 {+ if q := value[0]; (q == '"' || q == '\'') && value[len(value)-1] == q {+ value = value[1 : len(value)-1]+ }+ }+ value = strings.TrimSpace(html.UnescapeString(value))+ if value == "" {+ return "", false+ }+ return value, true+}++// replaceImageSrc swaps only the src attribute value for a double-quoted data+// URI, preserving the rest of the tag byte-for-byte.+func replaceImageSrc(tag, dataURI string) string {+ loc := srcAttrRe.FindStringSubmatchIndex(tag)+ if loc == nil {+ return tag+ }+ start, end := loc[4], loc[5]+ return tag[:start] + `"` + dataURI + `"` + tag[end:]+}diff --git a/internal/datasource/connector/confluence/asset_resolver_test.go b/internal/datasource/connector/confluence/asset_resolver_test.gonew file mode 100644index 000000000..98b124831--- /dev/null+++ b/internal/datasource/connector/confluence/asset_resolver_test.go@@ -0,0 +1,612 @@+package confluence++import (+ "bytes"+ "context"+ "encoding/base64"+ "errors"+ "io"+ "net/http"+ "strings"+ "sync"+ "testing"+)++const testBaseURL = "https://confluence.test/wiki"++func newImageTestClient(rt roundTripper) *client {+ return &client{+ cfg: config{baseURL: testBaseURL, username: "reader", secret: "secret"},+ http: &http.Client{Transport: rt},+ }+}++func imageResp(status int, contentType string, body []byte) *http.Response {+ header := make(http.Header)+ if contentType != "" {+ header.Set("Content-Type", contentType)+ }+ return &http.Response{+ StatusCode: status,+ Header: header,+ Body: io.NopCloser(bytes.NewReader(body)),+ }+}++func redirectResp(location string) *http.Response {+ header := make(http.Header)+ header.Set("Location", location)+ return &http.Response{+ StatusCode: http.StatusFound,+ Header: header,+ Body: io.NopCloser(bytes.NewReader(nil)),+ }+}++// reqLog records outbound requests. Image downloads run concurrently, so every+// access is mutex-guarded.+type reqLog struct {+ mu sync.Mutex+ reqs []reqRecord+}++type reqRecord struct {+ host string+ path string+ hasAuth bool+}++func newReqLog() *reqLog { return &reqLog{} }++func (l *reqLog) add(req *http.Request) {+ l.mu.Lock()+ defer l.mu.Unlock()+ l.reqs = append(l.reqs, reqRecord{+ host: req.URL.Host,+ path: req.URL.Path,+ hasAuth: req.Header.Get("Authorization") != "",+ })+}++func (l *reqLog) count() int {+ l.mu.Lock()+ defer l.mu.Unlock()+ return len(l.reqs)+}++func (l *reqLog) all() []reqRecord {+ l.mu.Lock()+ defer l.mu.Unlock()+ return append([]reqRecord(nil), l.reqs...)+}++func (l *reqLog) hasHost(host string) bool {+ for _, r := range l.all() {+ if r.host == host {+ return true+ }+ }+ return false+}++func (l *reqLog) hasPath(path string) bool {+ for _, r := range l.all() {+ if r.path == path {+ return true+ }+ }+ return false+}++func (l *reqLog) anyWithoutAuth() bool {+ for _, r := range l.all() {+ if !r.hasAuth {+ return true+ }+ }+ return false+}++var testPNG = []byte("private-png-bytes")++func testDataURI(body []byte) string {+ return "data:image/png;base64," + base64.StdEncoding.EncodeToString(body)+}++func TestAssetResolverKeepsPublicURL(t *testing.T) {+ log := newReqLog()+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {+ log.add(req)+ return imageResp(http.StatusOK, "image/png", testPNG), nil+ })+ in := `<p>hi</p><img src="https://cdn.example.com/a.png">`+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)+ if out != in {+ t.Fatalf("public URL was modified:\n%s", out)+ }+ if log.count() != 0 {+ t.Fatalf("public image was downloaded: %v", log.all())+ }+}++func TestAssetResolverKeepsBase64(t *testing.T) {+ log := newReqLog()+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {+ log.add(req)+ return imageResp(http.StatusOK, "image/png", testPNG), nil+ })+ in := `<img src="data:image/png;base64,QUJDRA==">`+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)+ if out != in {+ t.Fatalf("base64 image was modified:\n%s", out)+ }+ if log.count() != 0 {+ t.Fatalf("base64 image triggered a download: %v", log.all())+ }+}++func TestAssetResolverKeepsNonHTTPScheme(t *testing.T) {+ log := newReqLog()+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {+ log.add(req)+ return imageResp(http.StatusOK, "image/png", testPNG), nil+ })+ in := `<img src="file:///etc/passwd"><img src="javascript:void(0)">`+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)+ if out != in {+ t.Fatalf("non-http scheme was modified:\n%s", out)+ }+ if log.count() != 0 {+ t.Fatalf("non-http scheme triggered a download: %v", log.all())+ }+}++func TestAssetResolverInlinesRelativeConfluenceImage(t *testing.T) {+ log := newReqLog()+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {+ log.add(req)+ return imageResp(http.StatusOK, "image/png", testPNG), nil+ })+ in := `<img src="/wiki/download/attachments/123/a.png">`+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)++ want := testDataURI(testPNG)+ if !strings.Contains(out, `src="`+want+`"`) {+ t.Fatalf("relative image not inlined as data URI:\n%s", out)+ }+ if strings.Contains(out, "/wiki/download/attachments") {+ t.Fatalf("original private URL survived:\n%s", out)+ }+ reqs := log.all()+ if len(reqs) != 1 {+ t.Fatalf("expected 1 download, got %d: %v", len(reqs), reqs)+ }+ if reqs[0].host != "confluence.test" || !reqs[0].hasAuth {+ t.Fatalf("download missing credentials or wrong host: %+v", reqs[0])+ }+}++func TestAssetResolverInlinesSameOriginPrivateImage(t *testing.T) {+ log := newReqLog()+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {+ log.add(req)+ return imageResp(http.StatusOK, "image/png", testPNG), nil+ })+ in := `<img src="https://confluence.test/wiki/download/attachments/123/a.png" alt="x"/>`+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)++ if !strings.Contains(out, `src="`+testDataURI(testPNG)+`"`) {+ t.Fatalf("same-origin image not inlined:\n%s", out)+ }+ // The rest of the tag must be preserved.+ if !strings.Contains(out, `alt="x"`) {+ t.Fatalf("sibling attribute lost:\n%s", out)+ }+ if log.count() != 1 {+ t.Fatalf("expected 1 download, got %d", log.count())+ }+}++func TestAssetResolverDoesNotSendCredentialsCrossOrigin(t *testing.T) {+ log := newReqLog()+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {+ log.add(req)+ return imageResp(http.StatusOK, "image/png", testPNG), nil+ })+ in := `<img src="https://cdn.example.com/a.png">` ++ `<img src="/wiki/download/attachments/1/a.png">`+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)++ if log.hasHost("cdn.example.com") {+ t.Fatalf("external host received a request: %v", log.all())+ }+ if !strings.Contains(out, "https://cdn.example.com/a.png") {+ t.Fatalf("external URL was modified:\n%s", out)+ }+ if !strings.Contains(out, testDataURI(testPNG)) {+ t.Fatalf("same-origin image was not inlined:\n%s", out)+ }+ if log.count() != 1 || log.anyWithoutAuth() {+ t.Fatalf("expected exactly one authenticated same-origin download: %v", log.all())+ }+}++func TestAssetResolverRejectsCrossOriginRedirect(t *testing.T) {+ log := newReqLog()+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {+ log.add(req)+ if req.URL.Host == "confluence.test" {+ return redirectResp("https://evil.example.com/a.png"), nil+ }+ return imageResp(http.StatusOK, "image/png", testPNG), nil+ })+ in := `<img src="/wiki/download/attachments/1/a.png">`+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)++ if out != in {+ t.Fatalf("cross-origin redirect was followed and inlined:\n%s", out)+ }+ if log.hasHost("evil.example.com") {+ t.Fatalf("credentials could have leaked to evil host: %v", log.all())+ }+ if log.count() != 1 {+ t.Fatalf("expected only the original same-origin request, got %d: %v", log.count(), log.all())+ }+}++func TestAssetResolverRejectsSameOriginRedirectOutsideContextPath(t *testing.T) {+ log := newReqLog()+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {+ log.add(req)+ // The base URL is https://confluence.test/wiki. A redirect to another app on+ // the same host stays same-origin but escapes the /wiki context path, and Go+ // keeps sending Basic Auth on a same-host hop. It must be refused before the+ // credentialed request is ever sent, not merely discarded afterward.+ if strings.HasPrefix(req.URL.Path, "/wiki/") {+ return redirectResp("https://confluence.test/other-app/leak.png"), nil+ }+ return imageResp(http.StatusOK, "image/png", testPNG), nil+ })+ in := `<img src="/wiki/download/attachments/1/a.png">`+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)++ if out != in {+ t.Fatalf("same-host redirect outside the context path was followed and inlined:\n%s", out)+ }+ if log.hasPath("/other-app/leak.png") {+ t.Fatalf("credentials followed a redirect outside the context path: %v", log.all())+ }+ if log.count() != 1 {+ t.Fatalf("expected only the original same-origin request, got %d: %v", log.count(), log.all())+ }+}++func TestAssetResolverRejectsOversizedImage(t *testing.T) {+ log := newReqLog()+ big := bytes.Repeat([]byte("a"), int(maxImageBytes)+1)+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {+ log.add(req)+ return imageResp(http.StatusOK, "image/png", big), nil+ })+ in := `<img src="/wiki/download/attachments/1/big.png">`+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)++ if out != in {+ t.Fatalf("oversized image was inlined:\n%s", out)+ }+ if strings.Contains(out, "data:image") {+ t.Fatalf("oversized image produced a data URI")+ }+}++func TestAssetResolverRejectsNonImageResponse(t *testing.T) {+ rt := roundTripper(func(_ *http.Request) (*http.Response, error) {+ return imageResp(http.StatusOK, "text/html; charset=utf-8", []byte("<html>login required</html>")), nil+ })+ in := `<img src="/wiki/download/attachments/1/a.png">`+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)++ if out != in {+ t.Fatalf("login HTML was inlined as an image:\n%s", out)+ }+}++func TestAssetResolverKeepsOriginalURLOnDownloadFailure(t *testing.T) {+ for _, status := range []int{http.StatusForbidden, http.StatusNotFound, http.StatusInternalServerError} {+ t.Run(http.StatusText(status), func(t *testing.T) {+ rt := roundTripper(func(_ *http.Request) (*http.Response, error) {+ return imageResp(status, "image/png", testPNG), nil+ })+ in := `<img src="/wiki/download/attachments/1/a.png">`+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)+ if out != in {+ t.Fatalf("status %d modified the document:\n%s", status, out)+ }+ })+ }+}++func TestAssetResolverDeduplicatesSamePageImages(t *testing.T) {+ log := newReqLog()+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {+ log.add(req)+ return imageResp(http.StatusOK, "image/png", testPNG), nil+ })+ src := `<img src="/wiki/download/attachments/1/a.png">`+ in := strings.Repeat(src, 5)+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)++ if got := strings.Count(out, "data:image/png;base64,"); got != 5 {+ t.Fatalf("expected 5 inlined copies, got %d:\n%s", got, out)+ }+ if log.count() != 1 {+ t.Fatalf("duplicate src downloaded %d times, want 1", log.count())+ }+}++func TestAssetResolverRespectsPerPageImageCap(t *testing.T) {+ log := newReqLog()+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {+ log.add(req)+ return imageResp(http.StatusOK, "image/png", testPNG), nil+ })+ var sb strings.Builder+ total := maxPageImages + 5+ for i := 0; i < total; i++ {+ sb.WriteString(`<img src="/wiki/download/attachments/1/img`)+ sb.WriteString(strings.Repeat("x", i+1)) // unique path per image+ sb.WriteString(`.png">`)+ }+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), sb.String())++ if got := strings.Count(out, "data:image/png;base64,"); got != maxPageImages {+ t.Fatalf("inlined %d images, want cap %d", got, maxPageImages)+ }+ if log.count() != maxPageImages {+ t.Fatalf("downloaded %d images, want cap %d", log.count(), maxPageImages)+ }+ if got := strings.Count(out, "/wiki/download/attachments"); got != 5 {+ t.Fatalf("expected 5 images past the cap to keep original src, got %d", got)+ }+}++func TestAssetResolverCapsOccurrencesNotUniqueDownloads(t *testing.T) {+ log := newReqLog()+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {+ log.add(req)+ return imageResp(http.StatusOK, "image/png", testPNG), nil+ })+ // Two unique URLs but 35 occurrences. Counting the cap by unique URL would+ // download cold.png too and inline 30 + 5 data URIs, overflowing the downstream+ // per-page limit that counts every occurrence. The budget must be occurrences.+ hot := `<img src="/wiki/download/attachments/1/hot.png">`+ cold := `<img src="/wiki/download/attachments/1/cold.png">`+ in := strings.Repeat(hot, maxPageImages) + strings.Repeat(cold, 5)+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)++ if got := strings.Count(out, "data:image/png;base64,"); got != maxPageImages {+ t.Fatalf("inlined %d data URIs, want cap %d", got, maxPageImages)+ }+ if got := strings.Count(out, "/wiki/download/attachments/1/cold.png"); got != 5 {+ t.Fatalf("expected the 5 images past the cap to keep original src, got %d", got)+ }+ // The repeated image downloads once; the image past the budget never downloads.+ if log.count() != 1 {+ t.Fatalf("downloaded %d images, want 1 (dedup + no over-budget fetch): %v", log.count(), log.all())+ }+ if log.hasPath("/wiki/download/attachments/1/cold.png") {+ t.Fatalf("image past the occurrence cap was downloaded: %v", log.all())+ }+}++// largeTestImage is just under maxImageBytes, so a handful of them overflow+// maxPageInlineBytes long before the per-page occurrence cap.+var largeTestImage = bytes.Repeat([]byte("a"), int(maxImageBytes)-1024)++func TestAssetResolverStopsInliningPastPageByteBudget(t *testing.T) {+ rt := roundTripper(func(_ *http.Request) (*http.Response, error) {+ return imageResp(http.StatusOK, "image/png", largeTestImage), nil+ })+ want := maxPageInlineBytes / len(testDataURI(largeTestImage))+ total := want + 3+ src := `<img src="/wiki/download/attachments/1/big.png">`+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), strings.Repeat(src, total))++ if got := strings.Count(out, "data:image/png;base64,"); got != want {+ t.Fatalf("inlined %d copies, want %d within the %d-byte page budget", got, want, maxPageInlineBytes)+ }+ if got := strings.Count(out, "/wiki/download/attachments/1/big.png"); got != total-want {+ t.Fatalf("expected %d copies past the budget to keep original src, got %d", total-want, got)+ }+}++func TestAssetResolverDownloadCacheRespectsPageByteBudget(t *testing.T) {+ rt := roundTripper(func(_ *http.Request) (*http.Response, error) {+ return imageResp(http.StatusOK, "image/png", largeTestImage), nil+ })+ want := maxPageInlineBytes / len(testDataURI(largeTestImage))+ urls := make([]string, 0, want+3)+ for i := 0; i < want+3; i++ {+ urls = append(urls, testBaseURL+"/download/attachments/1/img"+strings.Repeat("x", i+1)+".png")+ }+ cache := newAssetResolver(newImageTestClient(rt)).downloadAll(context.Background(), urls)++ if len(cache) != want {+ t.Fatalf("cached %d images, want %d within the %d-byte page budget", len(cache), want, maxPageInlineBytes)+ }+}++func TestAssetResolverSharesCapWithExistingDataURIImages(t *testing.T) {+ log := newReqLog()+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {+ log.add(req)+ return imageResp(http.StatusOK, "image/png", testPNG), nil+ })+ // Data URI images already in the page share the downstream 30-image budget with+ // the ones we generate. With 10 existing, only 20 of the 25 private images may+ // be inlined; the rest keep their original URL instead of overflowing the cap.+ var sb strings.Builder+ for i := 0; i < 10; i++ {+ sb.WriteString(`<img src="data:image/gif;base64,EXISTING">`)+ }+ for i := 0; i < 25; i++ {+ sb.WriteString(`<img src="/wiki/download/attachments/1/img`)+ sb.WriteString(strings.Repeat("x", i+1)) // unique path per image+ sb.WriteString(`.png">`)+ }+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), sb.String())++ // The 10 pre-existing data URIs are preserved untouched.+ if got := strings.Count(out, "data:image/gif;base64,EXISTING"); got != 10 {+ t.Fatalf("preserved %d existing data URIs, want 10", got)+ }+ // Exactly 20 private images inlined: 30 shared budget - 10 already present.+ if got := strings.Count(out, testDataURI(testPNG)); got != 20 {+ t.Fatalf("inlined %d private images, want 20", got)+ }+ // The 5 private images past the shared budget keep their original URL.+ if got := strings.Count(out, "/wiki/download/attachments"); got != 5 {+ t.Fatalf("expected 5 private images past the cap to keep original src, got %d", got)+ }+ // Only the 20 inlinable private images are downloaded.+ if log.count() != 20 {+ t.Fatalf("downloaded %d images, want 20: %v", log.count(), log.all())+ }+}++func TestAssetResolverIgnoresDataSrcAttribute(t *testing.T) {+ log := newReqLog()+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {+ log.add(req)+ return imageResp(http.StatusOK, "image/png", testPNG), nil+ })+ // data-src must not be mistaken for src; there is no real src here.+ in := `<img data-src="/wiki/download/attachments/1/a.png">`+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)+ if out != in {+ t.Fatalf("data-src was treated as src:\n%s", out)+ }+ if log.count() != 0 {+ t.Fatalf("data-src triggered a download: %v", log.all())+ }+}++func TestAssetResolverToleratesGreaterThanInAttribute(t *testing.T) {+ log := newReqLog()+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {+ log.add(req)+ return imageResp(http.StatusOK, "image/png", testPNG), nil+ })+ in := `<img alt="a > b" src="/wiki/download/attachments/1/a.png">`+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)+ if !strings.Contains(out, testDataURI(testPNG)) {+ t.Fatalf("image with '>' in alt was not inlined:\n%s", out)+ }+ if !strings.Contains(out, `alt="a > b"`) {+ t.Fatalf("alt attribute was corrupted:\n%s", out)+ }+}++func TestAssetResolverUnescapesAmpersandInQuery(t *testing.T) {+ var gotPath string+ var gotQuery string+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {+ gotPath = req.URL.Path+ gotQuery = req.URL.RawQuery+ return imageResp(http.StatusOK, "image/png", testPNG), nil+ })+ in := `<img src="/wiki/download/attachments/1/a.png?api=v2&v=3">`+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)+ if !strings.Contains(out, "data:image/png;base64,") {+ t.Fatalf("image with & query was not inlined:\n%s", out)+ }+ if gotPath != "/wiki/download/attachments/1/a.png" || gotQuery != "api=v2&v=3" {+ t.Fatalf("query not unescaped: path=%q query=%q", gotPath, gotQuery)+ }+}++func TestDownloadPrivateImageLimitsBodySize(t *testing.T) {+ big := bytes.Repeat([]byte("a"), int(maxImageBytes)+1)+ rt := roundTripper(func(_ *http.Request) (*http.Response, error) {+ return imageResp(http.StatusOK, "image/png", big), nil+ })+ c := newImageTestClient(rt)+ _, _, err := c.downloadImage(context.Background(), "https://confluence.test/wiki/download/attachments/1/big.png")+ if !errors.Is(err, errImageTooLarge) {+ t.Fatalf("downloadImage() error = %v, want errImageTooLarge", err)+ }+}++func TestDownloadPrivateImageValidatesContentType(t *testing.T) {+ rt := roundTripper(func(_ *http.Request) (*http.Response, error) {+ return imageResp(http.StatusOK, "application/json", []byte(`{"error":"unauthorized"}`)), nil+ })+ c := newImageTestClient(rt)+ _, _, err := c.downloadImage(context.Background(), "https://confluence.test/wiki/download/attachments/1/a.png")+ if !errors.Is(err, errImageNonImageContent) {+ t.Fatalf("downloadImage() error = %v, want errImageNonImageContent", err)+ }+}++func TestDownloadPrivateImageSniffsWhenContentTypeMissing(t *testing.T) {+ png := append([]byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}, bytes.Repeat([]byte{0}, 64)...)+ rt := roundTripper(func(_ *http.Request) (*http.Response, error) {+ return imageResp(http.StatusOK, "", png), nil+ })+ c := newImageTestClient(rt)+ body, mime, err := c.downloadImage(+ context.Background(),+ "https://confluence.test/wiki/download/attachments/1/a.png",+ )+ if err != nil {+ t.Fatalf("downloadImage() error = %v", err)+ }+ if mime != "image/png" {+ t.Fatalf("sniffed mime = %q, want image/png", mime)+ }+ if !bytes.Equal(body, png) {+ t.Fatalf("body mismatch: %d bytes", len(body))+ }+}++func TestDownloadPrivateImageRejectsCrossOriginRedirect(t *testing.T) {+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {+ if req.URL.Host == "confluence.test" {+ return redirectResp("https://evil.example.com/a.png"), nil+ }+ return imageResp(http.StatusOK, "image/png", testPNG), nil+ })+ c := newImageTestClient(rt)+ _, _, err := c.downloadImage(context.Background(), "https://confluence.test/wiki/download/attachments/1/a.png")+ if !errors.Is(err, errImageRedirectOffOrigin) {+ t.Fatalf("downloadImage() error = %v, want errImageRedirectOffOrigin", err)+ }+}++// TestFetchStreamInlinesPrivateImageAsDataURI is the connector integration test:+// a body.view containing a same-origin private image flows through FetchStream+// and the emitted FetchedItem.Content carries an inline data URI (which the+// downstream docparser pipeline later persists and rewrites to resource://).+func TestFetchStreamInlinesPrivateImageAsDataURI(t *testing.T) {+ api := &streamAPI{+ pages: []streamPage{{id: "p1", title: "Page", version: 1}},+ bodyHTML: map[string]string{"p1": `<p>hi</p><img src="/wiki/download/attachments/p1/a.png">`},+ }+ h := &captureHandler{}+ if _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), nil, h); err != nil {+ t.Fatal(err)+ }+ if len(h.items) != 1 {+ t.Fatalf("emitted %d items, want 1", len(h.items))+ }+ content := string(h.items[0].Content)+ if !strings.Contains(content, "data:image/png;base64,") {+ t.Fatalf("private image was not inlined into Content:\n%s", content)+ }+ if strings.Contains(content, "/wiki/download/attachments") {+ t.Fatalf("original private Confluence URL leaked into Content:\n%s", content)+ }+ if api.imageCalls != 1 {+ t.Fatalf("image download calls = %d, want 1", api.imageCalls)+ }+}diff --git a/internal/datasource/connector/confluence/client.go b/internal/datasource/connector/confluence/client.goindex ec156e382..66fa29ab4 100644--- a/internal/datasource/connector/confluence/client.go+++ b/internal/datasource/connector/confluence/client.go@@ -17,19 +17,29 @@ import ()const (- maxJSONResponseBytes int64 = 20 << 20- requestAttempts = 4- maxErrorResponseRunes = 1000- maxRetryDelay = 60 * time.Second- maxPaginationHops = 10000+ maxJSONResponseBytes int64 = 20 << 20+ maxImageBytes int64 = 10 << 20+ maxImageRedirects = 10+ requestAttempts = 4+ maxErrorResponseRunes = 1000+ maxRetryDelay = 60 * time.Second+ maxPaginationHops = 10000+ maxTraversalNodes = 100000+ maxTransparentTraversalNodes = 10000+ maxTransparentTraversalDepth = 32// maxEmptyServerPages bounds how far a Server/Data Center page listing- // follows _links.next through empty pages. Some builds keep returning- // next past the end of a space (#3596); permission filtering can also- // empty one page mid-listing, so a single empty page is not the end.+ // follows _links.next through empty pages.maxEmptyServerPages = 3- // maxDeletionProbes caps how many unlisted pages one sync looks up- // individually after a cut-short listing; the rest wait for the next run.- maxDeletionProbes = 200+ maxDeletionProbes = 200+)++var (+ errImageTooLarge = errors.New("confluence image exceeds size limit")+ errImageNonImageContent = errors.New("confluence image response is not an image")+ // errImageRedirectOffOrigin fires whenever a redirect target leaves the base+ // URL boundary resolveEndpoint enforces — a different origin, or the same host+ // but a path outside the configured context path (e.g. /wiki → /other-app).+ errImageRedirectOffOrigin = errors.New("confluence image redirect left the configured base URL"))type apiError struct {@@ -232,6 +242,121 @@ func (c *client) resourceURL(link string) string {return resolved}+// sameOrigin reports whether u shares scheme and host with the configured base+// URL. Only same-origin URLs may carry Confluence credentials.+func (c *client) sameOrigin(u *url.URL) bool {+ if u == nil {+ return false+ }+ base, err := url.Parse(c.cfg.baseURL)+ if err != nil {+ return false+ }+ return strings.EqualFold(base.Scheme, u.Scheme) && strings.EqualFold(base.Host, u.Host)+}++// withinImageScope reports whether u stays inside the boundary resolveEndpoint+// enforces for the initial image URL: the same origin AND a path inside the+// configured context path. Redirect targets are validated with this same rule,+// not merely sameOrigin, because Go forwards Basic Auth on a same-host redirect —+// so a hop like /wiki/download → /other-app/image must fail closed too.+func (c *client) withinImageScope(u *url.URL) bool {+ if u == nil {+ return false+ }+ _, err := c.resolveEndpoint(u.String())+ return err == nil+}++// downloadImage fetches a same-origin private image using the connector's+// credentials. It streams with a hard size cap, fails closed if a redirect left+// the Confluence origin, and validates the response really is an image. It is a+// best-effort single attempt: a failed image degrades to keeping its original+// URL rather than retrying or failing the page.+func (c *client) downloadImage(ctx context.Context, absURL string) ([]byte, string, error) {+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, absURL, nil)+ if err != nil {+ return nil, "", err+ }+ req.SetBasicAuth(c.cfg.username, c.cfg.secret)+ req.Header.Set("Accept", "image/*,*/*;q=0.8")+ resp, err := c.imageHTTPClient().Do(req)+ if err != nil {+ return nil, "", err+ }+ defer func() { _ = resp.Body.Close() }()+ // Backstop for the redirect policy: imageHTTPClient's CheckRedirect already+ // refuses any hop that leaves the base before it is followed, so this only+ // re-verifies the final URL when the transport populated resp.Request. A custom+ // RoundTripper may leave it nil, in which case there is no followed redirect to+ // re-check and the download is allowed to proceed.+ if resp.Request != nil && !c.withinImageScope(resp.Request.URL) {+ return nil, "", errImageRedirectOffOrigin+ }+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {+ return nil, "", &apiError{endpoint: "image", status: resp.StatusCode}+ }+ body, err := io.ReadAll(io.LimitReader(resp.Body, maxImageBytes+1))+ if err != nil {+ return nil, "", err+ }+ if int64(len(body)) > maxImageBytes {+ return nil, "", errImageTooLarge+ }+ mime, ok := imageContentType(resp.Header.Get("Content-Type"), body)+ if !ok {+ return nil, "", errImageNonImageContent+ }+ return body, mime, nil+}++// imageHTTPClient reuses the connector's SSRF-safe transport and timeout but+// pins redirects to the configured Confluence base, so a private-image fetch can+// never be bounced to a third party or to another app on the same host. The+// redirect check fails closed: any hop that resolveEndpoint would reject — a+// foreign origin, or a same-host path outside the context path (which Go would+// still send Basic Auth to) — is refused outright rather than merely stripped of+// credentials.+func (c *client) imageHTTPClient() *http.Client {+ return &http.Client{+ Transport: c.http.Transport,+ Timeout: c.http.Timeout,+ CheckRedirect: func(req *http.Request, via []*http.Request) error {+ if len(via) >= maxImageRedirects {+ return errors.New("confluence image stopped after too many redirects")+ }+ if !c.withinImageScope(req.URL) {+ return errImageRedirectOffOrigin+ }+ return nil+ },+ }+}++// imageContentType normalizes a response Content-Type to an image MIME type. An+// empty or generic binary type falls back to content sniffing; any explicit+// non-image type (for example the text/html login page Confluence returns when a+// token expired) is rejected so it is never stored as an image.+func imageContentType(header string, body []byte) (string, bool) {+ ct := strings.ToLower(strings.TrimSpace(header))+ if i := strings.Index(ct, ";"); i >= 0 {+ ct = strings.TrimSpace(ct[:i])+ }+ switch {+ case strings.HasPrefix(ct, "image/"):+ return ct, true+ case ct != "" && ct != "application/octet-stream":+ return "", false+ }+ if sniff := strings.ToLower(http.DetectContentType(body)); strings.HasPrefix(sniff, "image/") {+ if i := strings.Index(sniff, ";"); i >= 0 {+ sniff = strings.TrimSpace(sniff[:i])+ }+ return sniff, true+ }+ return "", false+}+func (c *client) ping(ctx context.Context) error {if c.cfg.cloud() {return c.get(ctx, "/api/v2/spaces?limit=1", nil)@@ -308,10 +433,6 @@ func withCloudPageQuery(next string) string {return parsed.String()}-// pages lists every current page in a space. complete is false when a Server-// listing was cut short after maxEmptyServerPages empty pages that still-// linked onward; the caller must not treat pages missing from such a listing-// as deleted.func (c *client) pages(ctx context.Context, s space) (pages []page, complete bool, err error) {if c.cfg.cloud() {var all []page@@ -371,10 +492,6 @@ func (c *client) pages(ctx context.Context, s space) (pages []page, complete booreturn all, complete, err}-// pageInSpace reports whether a Server page is still a current page of the-// space. Confluence answers 404 for a page that was deleted or moved to the-// trash; a page that is no longer current or now belongs to another space is-// gone from this one too, matching what a complete listing would omit.func (c *client) pageInSpace(ctx context.Context, id, spaceKey string) (bool, error) {var result struct {Status string `json:"status"`@@ -406,3 +523,349 @@ func (c *client) body(ctx context.Context, id string) (pageBody, error) {err := c.get(ctx, "/rest/api/content/"+url.PathEscape(id)+"?expand=body.view,version,space", &result)return result, err}++// pageDetail obtains identity, ownership, and version without requiring a+// rendered body. Scope enumeration must remain complete when a body is+// temporarily unavailable, otherwise a transient content error would be+// indistinguishable from a missing page during deletion reconciliation.+func (c *client) pageDetail(ctx context.Context, id string) (page, error) {+ var result page+ if c.cfg.cloud() {+ err := c.get(ctx, "/api/v2/pages/"+url.PathEscape(id), &result)+ return result, err+ }+ err := c.get(ctx, "/rest/api/content/"+url.PathEscape(id)+"?expand=version,space", &result)+ return result, err+}++// validatePageOwnership performs the constant-cost check required before a+// picker expansion. Ancestors are deliberately not fetched here: they are only+// needed by ResolveResourceAncestors and buildSyncPlan.+func (c *client) validatePageOwnership(ctx context.Context, s space, pageID string) error {+ full, err := c.pageDetail(ctx, pageID)+ if err != nil {+ return err+ }+ if c.cfg.cloud() {+ if full.SpaceID != s.ID {+ return fmt.Errorf("page %s belongs to space %s, not %s", pageID, full.SpaceID, s.ID)+ }+ return nil+ }+ if full.Space.Key == "" || full.Space.Key != s.Key {+ return fmt.Errorf("page %s does not belong to space %s", pageID, s.ID)+ }+ return nil+}++func serverChildPagesEndpoint(pageID string) string {+ query := url.Values{"expand": []string{serverPageExpand}, "limit": []string{"100"}}+ return "/rest/api/content/" + url.PathEscape(pageID) + "/child/page?" + query.Encode()+}++func serverTopLevelPagesEndpoint(spaceKey string) string {+ query := url.Values{"depth": []string{"root"}, "expand": []string{serverPageExpand}, "limit": []string{"100"}}+ return "/rest/api/space/" + url.PathEscape(spaceKey) + "/content/page?" + query.Encode()+}++func (c *client) listServerPages(ctx context.Context, endpoint string) ([]page, error) {+ start, _ := url.Parse(endpoint)+ startQuery := start.Query()+ var all []page+ err := c.paginate(ctx, endpoint, func(pageURL string) (string, error) {+ var result pageList+ if err := c.get(ctx, pageURL, &result); err != nil {+ return "", err+ }+ all = append(all, result.Results...)+ next := result.Links.Next+ if next != "" {+ next = withServerListQuery(next, startQuery)+ }+ return next, nil+ })+ return all, err+}++// Server pagination links commonly retain only start/limit. Restore the query+// that defines the listing scope (notably depth=root) as well as expand.+func withServerListQuery(next string, required url.Values) string {+ parsed, err := url.Parse(next)+ if err != nil {+ return next+ }+ query := parsed.Query()+ for key, values := range required {+ if query.Get(key) == "" && len(values) > 0 {+ query[key] = values+ }+ }+ parsed.RawQuery = query.Encode()+ return parsed.String()+}++func (c *client) listCloudHierarchy(ctx context.Context, endpoint string) ([]page, error) {+ var all []page+ err := c.paginate(ctx, endpoint, func(pageURL string) (string, error) {+ var result cloudHierarchyList+ if err := c.get(ctx, pageURL, &result); err != nil {+ return "", err+ }+ for _, item := range result.Results {+ all = append(+ all,+ page{+ ID: item.ID,+ Title: item.Title,+ Status: item.Status,+ Kind: item.Type,+ SpaceID: item.SpaceID,+ Links: item.Links,+ },+ )+ }+ return result.Links.Next, nil+ })+ return all, err+}++// topLevelPages retrieves one level only. The Server/DC depth=root endpoint+// is paginated by the server; it deliberately never falls back to scanning all+// pages because that would defeat lazy selection for large spaces.+//+// Cloud applies the same lazy contract: depth=0 returns only root-level pages,+// and Cloud exposes no endpoint to enumerate a space's top-level folders+// (CONFCLOUD-84275), so pages nested directly under those containers stay+// unselectable in the picker for now. Recovering them would require a+// whole-space scan per expand, which is exactly what lazy loading forbids;+// selecting the whole space still syncs them. The connector flags this+// limitation through resource metadata so the picker can surface it.+func (c *client) topLevelPages(ctx context.Context, s space) ([]page, error) {+ var pages []page+ var err error+ if c.cfg.cloud() {+ pages, err = c.listCloudHierarchy(+ ctx,+ "/api/v2/spaces/"+url.PathEscape(s.ID)+"/pages?status=current&depth=0&limit=250",+ )+ if err == nil {+ visible := pages[:0]+ for _, page := range pages {+ if page.Kind == "" || page.Kind == "page" {+ visible = append(visible, page)+ }+ }+ pages = visible+ }+ } else {+ pages, err = c.listServerPages(ctx, serverTopLevelPagesEndpoint(s.Key))+ }+ if err == nil {+ return pages, nil+ }+ // The depth=root navigation hint is Server/DC-specific; a Cloud 400/404 has+ // different causes and must surface the raw API error for diagnosis.+ if !c.cfg.cloud() {+ var api *apiError+ if errors.As(err, &api) && (api.status == http.StatusBadRequest || api.status == http.StatusNotFound) {+ return nil, fmt.Errorf(+ "confluence server does not support complete top-level page navigation for this space; "++ "select the Space to sync all pages: %w",+ err,+ )+ }+ }+ return nil, err+}++func (c *client) directChildPages(ctx context.Context, _ space, pageID string) ([]page, error) {+ if c.cfg.cloud() {+ return c.visibleCloudPageChildren(ctx, pageID)+ }+ return c.listServerPages(ctx, serverChildPagesEndpoint(pageID))+}++func cloudDirectChildrenEndpoint(kind, id string) (string, error) {+ plural := map[string]string{+ "page": "pages", "folder": "folders", "database": "databases", "whiteboard": "whiteboards", "embed": "embeds",+ }[kind]+ if plural == "" {+ return "", fmt.Errorf("unsupported Confluence Cloud hierarchy type %q", kind)+ }+ return "/api/v2/" + plural + "/" + url.PathEscape(id) + "/direct-children?limit=250", nil+}++// visibleCloudPageChildren projects Cloud's mixed-content hierarchy onto a+// Page-only picker. Containers remain connector-internal and are traversed+// until the next visible Page is found.+func (c *client) visibleCloudPageChildren(ctx context.Context, pageID string) ([]page, error) {+ return c.walkCloudPageHierarchy(ctx, pageID, false)+}++// walkCloudPageHierarchy traverses Cloud's mixed content tree through+// direct-children endpoints. Picker expansion stops at visible pages, while a+// sync walk continues through them to return the complete page subtree.+func (c *client) walkCloudPageHierarchy(ctx context.Context, pageID string, descendPages bool) ([]page, error) {+ type queuedNode struct {+ id, kind string+ depth int+ }+ queue := []queuedNode{{id: pageID, kind: "page"}}+ visited := map[string]struct{}{"page:" + pageID: {}}+ pages := make([]page, 0)+ nodeLimit := maxTransparentTraversalNodes+ if descendPages {+ nodeLimit = maxTraversalNodes+ }+ for len(queue) > 0 {+ current := queue[0]+ queue = queue[1:]+ endpoint, err := cloudDirectChildrenEndpoint(current.kind, current.id)+ if err != nil {+ return nil, err+ }+ children, err := c.listCloudHierarchy(ctx, endpoint)+ if err != nil {+ return nil, err+ }+ for _, child := range children {+ kind := child.Kind+ if kind == "" {+ kind = "page"+ }+ key := kind + ":" + child.ID+ if child.ID == "" || (kind == "page" && child.ID == pageID) {+ continue+ }+ if _, exists := visited[key]; exists {+ continue+ }+ if len(visited) >= nodeLimit {+ return nil, fmt.Errorf(+ "confluence Cloud hierarchy traversal exceeds %d nodes",+ nodeLimit,+ )+ }+ visited[key] = struct{}{}+ if kind == "page" {+ pages = append(pages, child)+ if !descendPages {+ continue+ }+ }+ depth := 0+ if kind != "page" {+ depth = current.depth + 1+ if current.kind == "page" {+ depth = 1+ }+ }+ if depth > maxTransparentTraversalDepth {+ return nil, fmt.Errorf(+ "confluence Cloud transparent traversal exceeds depth %d",+ maxTransparentTraversalDepth,+ )+ }+ if _, err := cloudDirectChildrenEndpoint(kind, child.ID); err != nil {+ return nil, err+ }+ queue = append(queue, queuedNode{id: child.ID, kind: kind, depth: depth})+ }+ }+ return pages, nil+}++// pageAncestors validates that pageID belongs to s and returns its path from+// the space root. A selected page is fetched even if it is top-level, so a+// forged page:{space}:{page} ID cannot cross a space boundary.+func (c *client) pageAncestors(ctx context.Context, s space, pageID string) ([]page, error) {+ if c.cfg.cloud() {+ full, err := c.pageDetail(ctx, pageID)+ if err != nil {+ return nil, err+ }+ if full.SpaceID != s.ID {+ return nil, fmt.Errorf("page %s belongs to space %s, not %s", pageID, full.SpaceID, s.ID)+ }+ ancestors, err := c.listCloudHierarchy(ctx, "/api/v2/pages/"+url.PathEscape(pageID)+"/ancestors?limit=250")+ if err != nil {+ return nil, err+ }+ return ancestors, nil+ }+ var full page+ if err := c.get(ctx, "/rest/api/content/"+url.PathEscape(pageID)+"?expand=ancestors,space", &full); err != nil {+ return nil, err+ }+ if full.Space.Key == "" || full.Space.Key != s.Key {+ return nil, fmt.Errorf("page %s does not belong to space %s", pageID, s.ID)+ }+ return full.Ancestors, nil+}++// pageSubtree uses direct-child traversal for Server/DC because that endpoint+// is stable across supported versions. Cloud uses the same complete+// mixed-content hierarchy walk as the picker, but continues through visible+// pages to include every page in the selected subtree.+func (c *client) pageSubtree(ctx context.Context, s space, pageID string) ([]page, error) {+ full, err := c.pageDetail(ctx, pageID)+ if err != nil {+ return nil, err+ }+ if c.cfg.cloud() {+ if full.SpaceID != s.ID {+ return nil, fmt.Errorf("page %s does not belong to space %s", pageID, s.ID)+ }+ } else if full.Space.Key == "" || full.Space.Key != s.Key {+ return nil, fmt.Errorf("page %s does not belong to space %s", pageID, s.ID)+ }+ root := full+ if c.cfg.cloud() {+ descendants, err := c.walkCloudPageHierarchy(ctx, pageID, true)+ if err != nil {+ return nil, err+ }+ out := []page{root}+ for _, descendant := range descendants {+ // Direct-child responses are intentionally compact. Hydrate each page+ // before version comparison; an unknown version must never be treated+ // as an unchanged stable token.+ detail, err := c.pageDetail(ctx, descendant.ID)+ if err != nil {+ return nil, err+ }+ if detail.SpaceID != s.ID {+ return nil, fmt.Errorf("page %s does not belong to space %s", descendant.ID, s.ID)+ }+ out = append(out, detail)+ }+ return out, nil+ }+ queue, out := []page{root}, []page{root}+ seen := map[string]bool{root.ID: true}+ for len(queue) > 0 {+ current := queue[0]+ queue = queue[1:]+ children, err := c.directChildPages(ctx, s, current.ID)+ if err != nil {+ return nil, err+ }+ for _, child := range children {+ if child.ID == "" || seen[child.ID] {+ continue+ }+ if len(seen) >= maxTraversalNodes {+ return nil, fmt.Errorf("confluence page subtree exceeds %d nodes", maxTraversalNodes)+ }+ seen[child.ID] = true+ if child.SpaceID != "" && child.SpaceID != s.ID {+ return nil, fmt.Errorf("page %s does not belong to space %s", child.ID, s.ID)+ }+ if !c.cfg.cloud() && child.Space.Key != "" && child.Space.Key != s.Key {+ return nil, fmt.Errorf("page %s does not belong to space %s", child.ID, s.ID)+ }+ queue, out = append(queue, child), append(out, child)+ }+ }+ return out, nil+}diff --git a/internal/datasource/connector/confluence/client_pages_test.go b/internal/datasource/connector/confluence/client_pages_test.goindex 1a7df281c..1113a35d6 100644--- a/internal/datasource/connector/confluence/client_pages_test.go+++ b/internal/datasource/connector/confluence/client_pages_test.go@@ -6,6 +6,7 @@ import ("fmt""net/http""net/http/httptest"+ "strings""testing")@@ -145,6 +146,70 @@ func TestServerSpacePageListPrefersFlatResults(t *testing.T) {}}+func TestTopLevelPagesServerRestoresScopeOnNext(t *testing.T) {+ var depths, expands []string+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/rest/api/space/ENG/content/page" {+ http.NotFound(w, r)+ return+ }+ depths = append(depths, r.URL.Query().Get("depth"))+ expands = append(expands, r.URL.Query().Get("expand"))+ w.Header().Set("Content-Type", "application/json")+ if r.URL.Query().Get("start") == "100" {+ _, _ = w.Write([]byte(`{+ "results": [{"id": "2", "title": "Second", "version": {"number": 2}}]+ }`))+ return+ }+ _, _ = w.Write([]byte(`{+ "_links": {"next": "/rest/api/space/ENG/content/page?limit=100&start=100"},+ "results": [{"id": "1", "title": "Root", "version": {"number": 1}}]+ }`))+ }))+ t.Cleanup(server.Close)++ c := &client{cfg: config{baseURL: server.URL}, http: server.Client()}+ pages, err := c.topLevelPages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})+ if err != nil {+ t.Fatal(err)+ }+ if len(pages) != 2 || pages[0].ID != "1" || pages[1].ID != "2" {+ t.Fatalf("pages = %#v", pages)+ }+ // The server's _links.next drops depth and expand; the connector must+ // rebuild the listing scope or later pages would look versionless.+ if len(depths) != 2 || depths[0] != "root" || depths[1] != "root" {+ t.Fatalf("depth query = %#v", depths)+ }+ if len(expands) != 2 || expands[0] != serverPageExpand || expands[1] != serverPageExpand {+ t.Fatalf("expand query = %#v", expands)+ }+}++func TestTopLevelPagesCloudSurfacesRawAPIError(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ w.WriteHeader(http.StatusNotFound)+ _, _ = w.Write([]byte(`{"message":"space not found"}`))+ }))+ t.Cleanup(server.Close)++ cloud := &client{cfg: config{baseURL: server.URL + "/wiki", edition: editionCloud}, http: server.Client()}+ _, err := cloud.topLevelPages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})+ if err == nil {+ t.Fatal("cloud topLevelPages unexpectedly succeeded")+ }+ if strings.Contains(err.Error(), "does not support") {+ t.Fatalf("cloud 404 reused the Server/DC navigation hint: %v", err)+ }++ srv := &client{cfg: config{baseURL: server.URL + "/wiki"}, http: server.Client()}+ _, err = srv.topLevelPages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})+ if err == nil || !strings.Contains(err.Error(), "confluence server does not support") {+ t.Fatalf("server error = %v; want the Server/DC navigation hint", err)+ }+}+func TestPagesCloudFollowsNextAndKeepsDepth(t *testing.T) {var depths []stringserver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {diff --git a/internal/datasource/connector/confluence/connector.go b/internal/datasource/connector/confluence/connector.goindex fe2a6f5ca..083d5bf13 100644--- a/internal/datasource/connector/confluence/connector.go+++ b/internal/datasource/connector/confluence/connector.go@@ -3,10 +3,14 @@ package confluenceimport ("context"+ "crypto/sha256""errors""fmt""net/http"+ "sort""strings"+ "sync"+ "time"htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2""github.com/Tencent/WeKnora/internal/datasource"@@ -21,12 +25,64 @@ var (// Connector implements datasource.StreamingConnector for Confluence.type Connector struct {- newClient func(config) (*client, error)+ newClient func(config) (*client, error)+ spaceCache spaceCache+}++const pickerSpaceCacheTTL = time.Minute++type spaceCache struct {+ mu sync.Mutex+ entries map[string]cachedSpaces+}++type cachedSpaces struct {+ values []space+ expires time.Time}// NewConnector creates a Confluence connector.func NewConnector() *Connector { return &Connector{newClient: newClient} }+func (c *Connector) pickerSpaces(ctx context.Context, client *client, cfg config) ([]space, error) {+ key := pickerSpaceCacheKey(cfg)+ now := time.Now()+ c.spaceCache.mu.Lock()+ if cached, ok := c.spaceCache.entries[key]; ok && now.Before(cached.expires) {+ values := append([]space(nil), cached.values...)+ c.spaceCache.mu.Unlock()+ return values, nil+ }+ c.spaceCache.mu.Unlock()+ spaces, err := client.spaces(ctx)+ if err != nil {+ return nil, err+ }+ c.spaceCache.mu.Lock()+ if c.spaceCache.entries == nil {+ c.spaceCache.entries = make(map[string]cachedSpaces)+ }+ for key, cached := range c.spaceCache.entries {+ if !now.Before(cached.expires) {+ delete(c.spaceCache.entries, key)+ }+ }+ c.spaceCache.entries[key] = cachedSpaces{+ values: append([]space(nil), spaces...),+ expires: now.Add(pickerSpaceCacheTTL),+ }+ c.spaceCache.mu.Unlock()+ return spaces, nil+}++// pickerSpaceCacheKey partitions cached space visibility by every credential+// component that can affect permissions. The secret is fingerprinted so it+// never becomes a map key retained in process memory or appears in diagnostics.+func pickerSpaceCacheKey(cfg config) string {+ fingerprint := sha256.Sum256([]byte(cfg.secret))+ return cfg.edition + "\x00" + cfg.baseURL + "\x00" + cfg.username + "\x00" + fmt.Sprintf("%x", fingerprint)+}+// Type returns the connector type identifier.func (*Connector) Type() string { return types.ConnectorTypeConfluence }@@ -52,38 +108,142 @@ func (c *Connector) Validate(ctx context.Context, ds *types.DataSourceConfig) erreturn client.ping(ctx)}-// ListResources returns the Confluence spaces the credentials can see.+// ListResources exposes a lazy Space → Page tree. It never scans a whole+// space merely to render the picker.func (c *Connector) ListResources(ctx context.Context, ds *types.DataSourceConfig, parentID string,) ([]types.Resource, error) {- if parentID != "" {- return nil, nil+ client, cfg, err := c.configured(ds)+ if err != nil {+ return nil, err}- client, _, err := c.configured(ds)+ if parentID == "" {+ spaces, err := c.pickerSpaces(ctx, client, cfg)+ if err != nil {+ return nil, err+ }+ out := make([]types.Resource, 0, len(spaces))+ for _, s := range spaces {+ metadata := map[string]interface{}{"space_key": s.Key}+ if cfg.cloud() {+ // Cloud cannot enumerate a space's top-level folders, so pages under+ // them are unselectable in the picker; the frontend surfaces this+ // limitation when the space is expanded.+ metadata["hierarchy_limitation"] = "cloud_top_level_containers"+ }+ out = append(out, types.Resource{+ ExternalID: s.ID, Name: s.Name, Type: "space",+ URL: client.resourceURL(s.Links.WebUI), HasChildren: true,+ Metadata: metadata,+ })+ }+ return out, nil+ }+ ref, err := parseResourceID(parentID)if err != nil {return nil, err}- spaces, err := client.spaces(ctx)+ spaces, err := c.pickerSpaces(ctx, client, cfg)if err != nil {return nil, err}- out := make([]types.Resource, 0, len(spaces))+ var selected space+ found := falsefor _, s := range spaces {- out = append(out, types.Resource{- ExternalID: s.ID,- Name: s.Name,- Type: "space",- URL: client.resourceURL(s.Links.WebUI),- HasChildren: false,- Metadata: map[string]interface{}{"space_key": s.Key},- })+ if s.ID == ref.SpaceID {+ selected, found = s, true+ break+ }+ }+ if !found {+ return nil, fmt.Errorf("confluence space %s is unavailable", ref.SpaceID)+ }+ var pages []page+ if ref.Kind == resourceSpace {+ pages, err = client.topLevelPages(ctx, selected)+ } else {+ if ref.Kind == resourcePage {+ if err := client.validatePageOwnership(ctx, selected, ref.PageID); err != nil {+ return nil, fmt.Errorf("validate Confluence page %s: %w", ref.PageID, err)+ }+ }+ pages, err = client.directChildPages(ctx, selected, ref.PageID)+ }+ if err != nil {+ return nil, fmt.Errorf("list Confluence resources under %s: %w", parentID, err)+ }+ out := make([]types.Resource, 0, len(pages))+ for _, p := range pages {+ out = append(out, pageResource(client, selected, parentID, p))}return out, nil}-// ResolveResourceAncestors is a no-op: Confluence spaces are a flat list.-func (*Connector) ResolveResourceAncestors(context.Context, *types.DataSourceConfig, []string) ([]string, error) {- return nil, nil+func pageResource(client *client, s space, parentID string, p page) types.Resource {+ return types.Resource{+ ExternalID: makePageResourceID(s.ID, p.ID), ParentID: parentID, Name: p.Title, Type: "page",+ URL: client.resourceURL(p.Links.WebUI), HasChildren: true,+ Metadata: map[string]interface{}{"space_id": s.ID, "page_id": p.ID},+ }+}++// ResolveResourceAncestors expands only the O(depth) path required to reveal+// persisted page selections in the lazy picker. Resolution is best effort:+// a deleted page, an unavailable space, or a malformed id skips just its own+// reveal path instead of failing every saved selection; the sync path keeps+// its strict validation in buildSyncPlan.+func (c *Connector) ResolveResourceAncestors(+ ctx context.Context,+ ds *types.DataSourceConfig,+ resourceIDs []string,+) ([]string, error) {+ client, _, err := c.configured(ds)+ if err != nil {+ return nil, err+ }+ spaces, err := client.spaces(ctx)+ if err != nil {+ return nil, err+ }+ byID := make(map[string]space, len(spaces))+ for _, s := range spaces {+ byID[s.ID] = s+ }+ seen, out := map[string]bool{}, []string{}+ add := func(id string) {+ if id != "" && !seen[id] {+ seen[id] = true+ out = append(out, id)+ }+ }+ for _, id := range resourceIDs {+ ref, err := parseResourceID(id)+ if err != nil {+ logger.Warnf(ctx, "skip malformed Confluence resource %q during ancestor resolution: %v", id, err)+ continue+ }+ if ref.Kind == resourceSpace {+ continue+ }+ s, ok := byID[ref.SpaceID]+ if !ok {+ logger.Warnf(ctx, "skip Confluence resource %s: space %s is unavailable", id, ref.SpaceID)+ continue+ }+ ancestors, err := client.pageAncestors(ctx, s, ref.PageID)+ if err != nil {+ logger.Warnf(ctx, "skip Confluence resource %s during ancestor resolution: %v", id, err)+ continue+ }+ add(s.ID)+ for _, ancestor := range ancestors {+ if ancestor.Kind != "" && ancestor.Kind != "page" {+ continue+ }+ add(makePageResourceID(s.ID, ancestor.ID))+ }+ }+ return out, nil}// FetchAll syncs every selected space. Deletion reconciliation needs FetchStream.@@ -158,51 +318,90 @@ func (c *Connector) fetchStream(if err != nil {return nil, err}- spaces, err := client.spaces(ctx)+ plan, err := c.buildSyncPlan(ctx, client, ds.ResourceIDs)if err != nil {- return nil, fmt.Errorf("list Confluence spaces: %w", err)+ return nil, err}- byID := make(map[string]space, len(spaces))- for _, s := range spaces {- byID[s.ID] = s+ currentRoots := make(map[string]bool, len(plan.Roots))+ for _, root := range plan.Roots {+ currentRoots[root.ResourceID] = true}baseline, next := prepareSyncCursors(old, forceFull)- for _, resourceID := range ds.ResourceIDs {- s, found := byID[resourceID]- if !found {- return nil, fmt.Errorf(- "selected Confluence space %s is unavailable; refusing destructive reconciliation",- resourceID,- )+ // A prior successful body is reusable when ownership moves to a different+ // selected root and this run cannot fetch the new body. Keep that version in+ // the new root rather than dropping it during ownership reconciliation.+ priorVersionByPage := make(map[string]string)+ for _, pages := range baseline.SpacePages {+ for id, version := range pages {+ if _, exists := priorVersionByPage[id]; !exists {+ priorVersionByPage[id] = version+ }+ }+ }+ // A page may be returned by multiple overlapping roots (for example during+ // a configuration change). Fetch it once, while retaining a cursor entry+ // per scope root so existing cursor JSON remains compatible.+ seenPages := make(map[string]struct{})+ incompleteRoots := make(map[string]bool)+ incompleteSpaces := make([]space, 0)+ for _, root := range plan.Roots {+ resourceID, s := root.ResourceID, root.Space+ var pages []page+ complete := true+ if root.Kind == syncWholeSpace {+ pages, complete, err = client.pages(ctx, s)+ } else {+ pages, err = client.pageSubtree(ctx, s, root.PageID)}- pages, complete, err := client.pages(ctx, s)if err != nil {- return nil, fmt.Errorf("list pages in Confluence space %s: %w", s.Key, err)+ return nil, fmt.Errorf("list pages in Confluence scope %s: %w", resourceID, err)}+ priorPages := baseline.SpacePages[resourceID]if !complete {- logger.Warnf(ctx,- "[Confluence] space %s: listing stopped after %d empty pages that still linked onward; "+- "listed %d pages, checking unlisted ones one by one before deleting",- s.Key, maxEmptyServerPages, len(pages))+ logger.Warnf(+ ctx,+ "[Confluence] space %s listing was cut short; checking unlisted pages before deleting",+ s.Key,+ )+ incompleteRoots[resourceID] = true+ incompleteSpaces = append(incompleteSpaces, s)}- priorPages, hadBaseline := baseline.SpacePages[resourceID]if next.SpacePages[resourceID] == nil {next.SpacePages[resourceID] = map[string]string{}}- seen := make(map[string]struct{}, len(pages))+ if complete && len(pages) == 0 && len(priorPages) > 0 {+ return nil, fmt.Errorf(+ "refusing Confluence mirror deletion in %s: listing returned 0 pages against a %d-page baseline",+ s.Key, len(priorPages),+ )+ }for _, summary := range pages {if err := ctx.Err(); err != nil {return nil, err}- seen[summary.ID] = struct{}{}- version := pageVersion(summary)- if forceFull {- if next.SpacePages[resourceID][summary.ID] == version {- continue+ if _, duplicate := seenPages[summary.ID]; duplicate {+ // The first scope already owns the emitted document; preserve this+ // root's version record so a later selection change stays incremental.+ if !forceFull {+ if version, ok := priorPages[summary.ID]; ok {+ next.SpacePages[resourceID][summary.ID] = version+ }}- } else if priorPages[summary.ID] == version {continue}+ seenPages[summary.ID] = struct{}{}+ version, versionKnown := pageVersion(summary)+ // An unknown version cannot prove a page unchanged, so it must+ // always refresh the body instead of skipping on a stable "t:".+ if versionKnown {+ if forceFull {+ if next.SpacePages[resourceID][summary.ID] == version {+ continue+ }+ } else if priorPages[summary.ID] == version {+ continue+ }+ }full, err := client.body(ctx, summary.ID)if err != nil {if ctx.Err() != nil {@@ -211,13 +410,23 @@ func (c *Connector) fetchStream(if emitErr := h.Emit(ctx, failedPageItem(resourceID, summary, err)); emitErr != nil {return nil, emitErr}+ if !forceFull {+ if prior, ok := priorVersionByPage[summary.ID]; ok {+ next.SpacePages[resourceID][summary.ID] = prior+ }+ }continue}- item, err := markdownItem(client, resourceID, summary, full)+ item, err := markdownItem(ctx, client, resourceID, summary, full)if err != nil {if emitErr := h.Emit(ctx, failedPageItem(resourceID, summary, err)); emitErr != nil {return nil, emitErr}+ if !forceFull {+ if prior, ok := priorVersionByPage[summary.ID]; ok {+ next.SpacePages[resourceID][summary.ID] = prior+ }+ }continue}if err := h.Emit(ctx, item); err != nil {@@ -228,99 +437,117 @@ func (c *Connector) fetchStream(return nil, err}}- if !complete {- if err := reconcileUnlisted(ctx, client, h, s, resourceID, priorPages, seen, &next); err != nil {- return nil, err+ }+ // Reconcile previous owners only after every current scope has been enumerated.+ // A page may move between roots, so a missing page must be checked against every+ // scope whose listing was cut short before a tombstone can be emitted.+ previousPages := make(map[string]string)+ for owner, pages := range baseline.SpacePages {+ for id := range pages {+ if _, exists := previousPages[id]; !exists {+ previousPages[id] = owner}- } else if hadBaseline {- if len(pages) == 0 && len(priorPages) > 0 {+ }+ }+ missing := make([]string, 0)+ probeCount := 0+ for id := range previousPages {+ if _, present := seenPages[id]; present {+ continue+ }+ owner := previousPages[id]+ for _, s := range incompleteSpaces {+ if probeCount >= maxDeletionProbes {return nil, fmt.Errorf(- "refusing Confluence mirror deletion in %s: listing returned 0 pages against a %d-page baseline",- s.Key, len(priorPages),+ "Confluence deletion reconciliation deferred: more than %d pages need verification",+ maxDeletionProbes,)}- missing := 0- for id := range priorPages {- if _, exists := seen[id]; !exists {- missing+++ probeCount+++ present, err := client.pageInSpace(ctx, id, s.Key)+ if err != nil {+ if ctx.Err() != nil {+ return nil, ctx.Err()}- }- if missing >= 20 && missing*100 >= len(priorPages)*80 {return nil, fmt.Errorf(- "refusing Confluence mirror deletion in %s: %d/%d pages disappeared",- s.Key, missing, len(priorPages),+ "Confluence deletion reconciliation deferred: could not verify page %s in space %s: %w",+ id, s.Key, err,)}- for id := range priorPages {- if _, exists := seen[id]; exists {- continue- }- if err := emitDeleted(ctx, h, &next, resourceID, id); err != nil {- return nil, err+ if present {+ version := baseline.SpacePages[owner][id]+ if next.SpacePages[s.ID] == nil {+ next.SpacePages[s.ID] = make(map[string]string)}+ next.SpacePages[s.ID][id] = version+ seenPages[id] = struct{}{}+ break}}- }- next.FullSync = false- next.FullSyncBaseline = nil- return next.syncCursor(), nil-}--// reconcileUnlisted handles the pages a cut-short listing did not return. The-// listing proves nothing about them, so each one is looked up on its own and-// deleted only when Confluence confirms it is gone from the space. Pages that-// are still there, could not be checked, or exceed the per-run probe budget-// stay tracked with their prior version and are checked again next run.-func reconcileUnlisted(- ctx context.Context,- c *client,- h datasource.StreamHandler,- s space,- resourceID string,- priorPages map[string]string,- seen map[string]struct{},- next *cursor,-) error {- probed := 0- for id, version := range priorPages {- if _, exists := seen[id]; exists {+ if _, present := seenPages[id]; present {continue}- if probed >= maxDeletionProbes {- next.SpacePages[resourceID][id] = version+ missing = append(missing, id)+ }+ // Protect still-selected roots from a suspiciously large shrink. Removed+ // roots are an explicit scope change, so their out-of-scope pages can be+ // tombstoned once every current scope has been enumerated safely.+ owners := make([]string, 0, len(baseline.SpacePages))+ for owner := range baseline.SpacePages {+ owners = append(owners, owner)+ }+ sort.Strings(owners)+ for _, owner := range owners {+ // A deliberate scope change may remove most of an old root. The guard is+ // only for unexpected shrinkage of a still-selected, completely listed root.+ if !currentRoots[owner] || incompleteRoots[owner] {continue}- probed++- present, err := c.pageInSpace(ctx, id, s.Key)- if err != nil {- if ctx.Err() != nil {- return ctx.Err()+ pages := baseline.SpacePages[owner]+ missingInRoot := 0+ for id := range pages {+ if _, present := seenPages[id]; !present {+ missingInRoot++}- logger.Warnf(ctx, "[Confluence] space %s: could not check unlisted page %s, keeping it: %v", s.Key, id, err)- next.SpacePages[resourceID][id] = version- continue}- if present {- next.SpacePages[resourceID][id] = version- continue- }- if err := emitDeleted(ctx, h, next, resourceID, id); err != nil {- return err+ if len(pages) > 0 && missingInRoot >= 20 && missingInRoot*100 >= len(pages)*80 {+ return nil, fmt.Errorf(+ "refusing Confluence mirror deletion: %d/%d previously synced pages would be removed from scope %s",+ missingInRoot, len(pages), owner,+ )}}- return nil-}--func emitDeleted(ctx context.Context, h datasource.StreamHandler, next *cursor, resourceID, id string) error {- deleted := types.FetchedItem{- ExternalID: id, IsDeleted: true, SourceResourceID: resourceID,- Metadata: map[string]string{"channel": types.ChannelConfluence},+ sort.Strings(missing)+ for _, id := range missing {+ deleted := types.FetchedItem{+ ExternalID: id, IsDeleted: true, SourceResourceID: previousPages[id],+ Metadata: map[string]string{"channel": types.ChannelConfluence},+ }+ if err := h.Emit(ctx, deleted); err != nil {+ return nil, err+ }+ // Drop the tombstoned page before checkpointing so the cursor records+ // the deletion itself; an interrupted run resumes with the remaining+ // candidates instead of re-emitting completed tombstones.+ removePageOwnership(&next, id)+ if err := h.Checkpoint(ctx, next.syncCursor()); err != nil {+ return nil, err+ }}- if err := h.Emit(ctx, deleted); err != nil {- return err+ // Commit exactly the current ownership roots only after reconciliation. This+ // removes cancelled scopes while keeping the on-disk cursor field compatible.+ committedRoots := make(map[string]map[string]string, len(plan.Roots))+ for _, root := range plan.Roots {+ if pages := next.SpacePages[root.ResourceID]; pages != nil {+ committedRoots[root.ResourceID] = pages+ } else {+ committedRoots[root.ResourceID] = map[string]string{}+ }}- delete(next.SpacePages[resourceID], id)- return h.Checkpoint(ctx, next.syncCursor())+ next.SpacePages = committedRoots+ next.FullSync = false+ next.FullSyncBaseline = nil+ return next.syncCursor(), nil}func failedPageItem(resourceID string, summary page, err error) types.FetchedItem {@@ -361,8 +588,18 @@ func classifyConfluenceError(err error) (code, reason string) {return "confluence_sync_failed", "Confluence page could not be synced; see server logs"}-func markdownItem(client *client, resourceID string, summary page, full pageBody) (types.FetchedItem, error) {+func markdownItem(+ ctx context.Context,+ client *client,+ resourceID string,+ summary page,+ full pageBody,+) (types.FetchedItem, error) {html := full.Body.View.Value+ // Inline same-origin private Confluence images as data URIs so the generic+ // docparser image pipeline can persist them and rewrite to resource:// URLs.+ // Best-effort: a failed image keeps its original src and never fails the page.+ html = newAssetResolver(client).Resolve(ctx, html)markdown, err := htmltomd.ConvertString(html)if err != nil {return types.FetchedItem{}, errdiff --git a/internal/datasource/connector/confluence/connector_stream_test.go b/internal/datasource/connector/confluence/connector_stream_test.goindex 2f38c787f..0197a55dd 100644--- a/internal/datasource/connector/confluence/connector_stream_test.go+++ b/internal/datasource/connector/confluence/connector_stream_test.go@@ -10,6 +10,7 @@ import ("net/http""strings""testing"+ "time""github.com/Tencent/WeKnora/internal/datasource""github.com/Tencent/WeKnora/internal/types"@@ -26,19 +27,18 @@ type streamPage struct {}type streamAPI struct {- cloud bool- pages []streamPage- bodyCalls int- bodyStatus map[string]int- listCalls int- // endlessNext mimics Server/Data Center builds that keep linking to a- // next page after the last one, returning empty results forever (#3596).- endlessNext bool- // probe answers the per-page existence check (GET content/{id}?expand=space)- // for pages that are not in the listing: an HTTP status, or 200 with the- // space key the page now belongs to.- probe map[string]probeReply- probeCalls int+ cloud bool+ pages []streamPage+ bodyCalls int+ bodyStatus map[string]int+ bodyHTML map[string]string+ imageCalls int+ listCalls int+ ancestorCalls int+ spaceCalls int+ endlessNext bool+ probe map[string]probeReply+ probeCalls int}type probeReply struct {@@ -71,7 +71,17 @@ func (a *streamAPI) pageMaps() []any {func (a *streamAPI) response(req *http.Request) (*http.Response, error) {path := req.URL.Pathswitch {+ case strings.HasPrefix(path, "/wiki/download/"):+ a.imageCalls+++ header := make(http.Header)+ header.Set("Content-Type", "image/png")+ return &http.Response{+ StatusCode: http.StatusOK,+ Header: header,+ Body: io.NopCloser(bytes.NewReader([]byte("private-png-bytes"))),+ }, nilcase path == "/wiki/rest/api/space":+ a.spaceCalls++return a.jsonResponse(map[string]any{"results": []any{map[string]any{"id": 1, "key": "ENG", "name": "Engineering",@@ -80,19 +90,23 @@ func (a *streamAPI) response(req *http.Request) (*http.Response, error) {})case path == "/wiki/rest/api/space/ENG/content/page":a.listCalls++- if !a.endlessNext {+ if req.URL.Query().Get("depth") == "root" {+ return a.jsonResponse(map[string]any{"results": a.pageMaps()})+ }+ if a.endlessNext {+ results := a.pageMaps()+ if a.listCalls > 1 {+ results = []any{}+ }return a.jsonResponse(map[string]any{- "results": a.pageMaps(),+ "results": results,+ "_links": map[string]any{+ "next": fmt.Sprintf("/rest/api/space/ENG/content/page?limit=100&start=%d", a.listCalls*100),+ },})}- results := a.pageMaps()- if a.listCalls > 1 {- results = []any{}- }- next := fmt.Sprintf("/rest/api/space/ENG/content/page?limit=100&start=%d", a.listCalls*100)return a.jsonResponse(map[string]any{- "results": results,- "_links": map[string]any{"next": next},+ "results": a.pageMaps(),})case strings.HasPrefix(path, "/wiki/rest/api/content/") && req.URL.Query().Get("expand") == "space":a.probeCalls++@@ -108,10 +122,36 @@ func (a *streamAPI) response(req *http.Request) (*http.Response, error) {Body: io.NopCloser(strings.NewReader("{}")),}, nil}- return a.jsonResponse(map[string]any{- "id": id, "status": "current", "space": map[string]any{"key": reply.spaceKey},- })+ return a.jsonResponse(+ map[string]any{"id": id, "status": "current", "space": map[string]any{"key": reply.spaceKey}},+ )+ case strings.HasPrefix(path, "/wiki/rest/api/content/") && strings.HasSuffix(path, "/child/page"):+ return a.jsonResponse(map[string]any{"results": []any{}})case strings.HasPrefix(path, "/wiki/rest/api/content/"):+ if req.URL.Query().Get("expand") == "ancestors,space" {+ a.ancestorCalls+++ return a.jsonResponse(map[string]any{+ "id": strings.TrimPrefix(path, "/wiki/rest/api/content/"),+ "space": map[string]any{"key": "ENG", "name": "Engineering"},+ "ancestors": []any{},+ })+ }+ if req.URL.Query().Get("expand") == "version,space" {+ id := strings.TrimPrefix(path, "/wiki/rest/api/content/")+ for _, page := range a.pages {+ if page.id == id {+ return a.jsonResponse(+ map[string]any{+ "id": page.id,+ "title": page.title,+ "version": map[string]any{"number": page.version},+ "space": map[string]any{"key": "ENG", "name": "Engineering"},+ },+ )+ }+ }+ return nil, errors.New("missing page")+ }if req.URL.Query().Get("expand") != "body.view,version,space" {return nil, errors.New("page body did not request rendered body.view")}@@ -137,8 +177,14 @@ func (a *streamAPI) response(req *http.Request) (*http.Response, error) {})}return a.jsonResponse(map[string]any{"results": pages})+ case strings.HasPrefix(path, "/wiki/api/v2/pages/") && strings.HasSuffix(path, "/ancestors"):+ return a.jsonResponse(map[string]any{"results": []any{}})+ case strings.HasPrefix(path, "/wiki/api/v2/pages/") && strings.HasSuffix(path, "/descendants"):+ return a.jsonResponse(map[string]any{"results": []any{}})+ case strings.HasPrefix(path, "/wiki/api/v2/pages/") && strings.HasSuffix(path, "/direct-children"):+ return a.jsonResponse(map[string]any{"results": []any{}})case strings.HasPrefix(path, "/wiki/api/v2/pages/"):- if req.URL.Query().Get("body-format") != "view" {+ if req.URL.Query().Get("body-format") != "" && req.URL.Query().Get("body-format") != "view" {return nil, errors.New("cloud page body did not request body-format=view")}return a.pageBody(strings.TrimPrefix(path, "/wiki/api/v2/pages/"))@@ -158,12 +204,17 @@ func (a *streamAPI) pageBody(id string) (*http.Response, error) {}for _, page := range a.pages {if page.id == id {+ value := "<p>" + page.title + "</p>"+ if custom, ok := a.bodyHTML[id]; ok {+ value = custom+ }return a.jsonResponse(map[string]any{"id": page.id, "title": page.title,+ "spaceId": "1","version": map[string]any{"number": page.version, "by": map[string]any{"displayName": "Ada"}},"space": map[string]any{"key": "ENG", "name": "Engineering"},"_links": map[string]any{"webui": "/wiki/pages/" + page.id},- "body": map[string]any{"view": map[string]any{"value": "<p>" + page.title + "</p>"}},+ "body": map[string]any{"view": map[string]any{"value": value}},})}}@@ -196,10 +247,11 @@ func cloudStreamConfig() *types.DataSourceConfig {}type captureHandler struct {- items []types.FetchedItem- checkpoints []*types.SyncCursor- emitErr error- succeedFirst int+ items []types.FetchedItem+ checkpoints []*types.SyncCursor+ emitErr error+ succeedFirst int+ checkpointErrAt int}func (h *captureHandler) Emit(_ context.Context, item types.FetchedItem) error {@@ -214,6 +266,9 @@ func (h *captureHandler) Emit(_ context.Context, item types.FetchedItem) error {}func (h *captureHandler) Checkpoint(_ context.Context, c *types.SyncCursor) error {+ if h.checkpointErrAt > 0 && len(h.checkpoints)+1 == h.checkpointErrAt {+ return errors.New("checkpoint failed")+ }raw, _ := json.Marshal(c)var stored types.SyncCursor_ = json.Unmarshal(raw, &stored)@@ -248,16 +303,144 @@ func TestFetchStreamSkipsUnchangedPages(t *testing.T) {}}+func TestFetchStreamUnknownVersionDoesNotSkipBody(t *testing.T) {+ // version.number=0 with no when/createdAt leaves the version unknown; the+ // legacy stable "t:" cursor value must not suppress the body refresh.+ api := &streamAPI{pages: []streamPage{{id: "p1", title: "Page", version: 0}}}+ h := &captureHandler{}+ old := streamCursor(map[string]string{"p1": "t:"})+ next, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h)+ if err != nil {+ t.Fatal(err)+ }+ if api.bodyCalls != 1 || len(h.items) != 1 || h.items[0].IsDeleted {+ t.Fatalf("unknown-version page fetched=%d items=%#v", api.bodyCalls, h.items)+ }+ if decoded := decodeCursor(next); decoded.SpacePages["1"]["p1"] != "" {+ t.Fatalf("cursor stored a stable unknown version: %#v", decoded)+ }+}+func TestFetchStreamDetectsDeletedPages(t *testing.T) {api := &streamAPI{pages: []streamPage{{id: "p1", title: "Page", version: 1}}}h := &captureHandler{}old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"})- if _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h); err != nil {+ next, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h)+ if err != nil {+ t.Fatal(err)+ }+ if len(h.items) != 1 || !h.items[0].IsDeleted || h.items[0].ExternalID != "p2" {+ t.Fatalf("deletion items = %#v", h.items)+ }+ if decoded := decodeCursor(next); decoded.SpacePages["1"]["p2"] != "" {+ t.Fatalf("deleted page stayed in cursor as a ghost: %#v", decoded)+ }+}++func TestFetchStreamRemovesDeletedPageFromCursor(t *testing.T) {+ api := &streamAPI{pages: []streamPage{{id: "p1", title: "Page", version: 1}}}+ h := &captureHandler{}+ old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"})+ next, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h)+ if err != nil {t.Fatal(err)}if len(h.items) != 1 || !h.items[0].IsDeleted || h.items[0].ExternalID != "p2" {t.Fatalf("deletion items = %#v", h.items)}+ decoded := decodeCursor(next)+ if decoded.SpacePages["1"]["p1"] != "v:1" {+ t.Fatalf("surviving page missing from cursor: %#v", decoded)+ }+ for _, checkpoint := range h.checkpoints {+ if stored := decodeCursor(checkpoint); stored.SpacePages["1"]["p2"] != "" {+ t.Fatalf("checkpoint recorded a not-yet-deleted page as deleted: %#v", stored)+ }+ }+ // A follow-up run must not re-emit the already tombstoned page.+ followUp := &captureHandler{}+ if _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), next, followUp); err != nil {+ t.Fatal(err)+ }+ for _, item := range followUp.items {+ if item.IsDeleted {+ t.Fatalf("ghost page re-tombstoned on the next run: %#v", followUp.items)+ }+ }+}++func TestFetchStreamTombstoneResumeSkipsCompletedDeletions(t *testing.T) {+ api := &streamAPI{pages: []streamPage{{id: "p1", title: "Page", version: 1}}}+ old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1", "p3": "v:1"})+ first := &captureHandler{succeedFirst: 1}+ if _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, first); err == nil {+ t.Fatal("interrupted tombstone run unexpectedly succeeded")+ }+ if len(first.items) != 1 || !first.items[0].IsDeleted || first.items[0].ExternalID != "p2" {+ t.Fatalf("first run items = %#v", first.items)+ }+ if len(first.checkpoints) != 1 {+ t.Fatalf("first run checkpoints = %d; want 1", len(first.checkpoints))+ }+ second := &captureHandler{}+ next, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), first.checkpoints[0], second)+ if err != nil {+ t.Fatal(err)+ }+ if len(second.items) != 1 || !second.items[0].IsDeleted || second.items[0].ExternalID != "p3" {+ t.Fatalf("resume must tombstone only the remaining page: %#v", second.items)+ }+ if decoded := decodeCursor(next); decoded.SpacePages["1"]["p1"] != "v:1" || decoded.SpacePages["1"]["p3"] != "" {+ t.Fatalf("resume cursor = %#v", decoded)+ }+}++func TestFetchFullStreamTombstoneResumeSkipsCompletedDeletions(t *testing.T) {+ api := &streamAPI{pages: []streamPage{{id: "p1", title: "Page", version: 1}}}+ old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1", "p3": "v:1"})+ // p1 re-fetch succeeds and the p2 tombstone is checkpointed before the+ // worker dies on the p3 tombstone.+ first := &captureHandler{succeedFirst: 2}+ if _, err := newStreamConnector(api).FetchFullStream(context.Background(), streamConfig(), old, first); err == nil {+ t.Fatal("interrupted full-sync tombstone run unexpectedly succeeded")+ }+ if len(first.items) != 2 || first.items[1].ExternalID != "p2" || !first.items[1].IsDeleted {+ t.Fatalf("first run items = %#v", first.items)+ }+ if len(first.checkpoints) != 2 {+ t.Fatalf("first run checkpoints = %d; want 2", len(first.checkpoints))+ }+ deletionCheckpoint := decodeCursor(first.checkpoints[1])+ if !deletionCheckpoint.FullSync {+ t.Fatalf("mid-full-sync checkpoint lost the resume flag: %#v", deletionCheckpoint)+ }+ if baseline := deletionCheckpoint.FullSyncBaseline["1"]; baseline["p1"] != "v:1" || baseline["p2"] != "" ||+ baseline["p3"] != "v:1" {+ t.Fatalf("tombstone checkpoint did not advance the deletion baseline: %#v", deletionCheckpoint)+ }++ resumeAPI := &streamAPI{pages: api.pages}+ second := &captureHandler{}+ next, err := newStreamConnector(+ resumeAPI,+ ).FetchFullStream(context.Background(), streamConfig(), first.checkpoints[1], second)+ if err != nil {+ t.Fatal(err)+ }+ if resumeAPI.bodyCalls != 0 {+ t.Fatalf("resume re-fetched %d already synced page bodies", resumeAPI.bodyCalls)+ }+ if len(second.items) != 1 || !second.items[0].IsDeleted || second.items[0].ExternalID != "p3" {+ t.Fatalf("resume must tombstone only the remaining page: %#v", second.items)+ }+ decoded := decodeCursor(next)+ if decoded.FullSync || decoded.FullSyncBaseline != nil {+ t.Fatalf("completed full sync left resume fields set: %#v", decoded)+ }+ if decoded.SpacePages["1"]["p1"] != "v:1" || decoded.SpacePages["1"]["p2"] != "" ||+ decoded.SpacePages["1"]["p3"] != "" {+ t.Fatalf("completed resume cursor = %#v", decoded)+ }}func TestFetchStreamRefusesMassDeletion(t *testing.T) {@@ -275,6 +458,67 @@ func TestFetchStreamRefusesMassDeletion(t *testing.T) {}}+func TestFetchStreamRefusesMassDeletionInSurvivingScope(t *testing.T) {+ prior := make(map[string]string, 100)+ for i := 0; i < 100; i++ {+ prior[fmt.Sprintf("p%03d", i)] = "v:1"+ }+ // The selection is unchanged; a broken listing suddenly returns only five+ // of the hundred previously synced pages.+ listed := make([]streamPage, 0, 5)+ for i := 0; i < 5; i++ {+ listed = append(listed, streamPage{id: fmt.Sprintf("p%03d", i), title: "Page", version: 1})+ }+ api := &streamAPI{pages: listed}+ h := &captureHandler{}+ _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), streamCursor(prior), h)+ if err == nil || len(h.items) != 0 {+ t.Fatalf("surviving-scope mass deletion err=%v items=%#v", err, h.items)+ }+}++func TestFetchStreamAllowsExplicitSpaceToSubtreeShrink(t *testing.T) {+ prior := make(map[string]string, 100)+ for i := 0; i < 100; i++ {+ prior[fmt.Sprintf("p%03d", i)] = "v:1"+ }+ // The user deliberately narrows a whole-space selection down to a page+ // subtree. The mass-deletion guard must only protect a surviving root.+ api := &streamAPI{pages: []streamPage{{id: "erp", title: "ERP", version: 2}}}+ ds := streamConfig()+ ds.ResourceIDs = []string{"page:1:erp"}+ h := &captureHandler{}+ next, err := newStreamConnector(api).FetchStream(context.Background(), ds, streamCursor(prior), h)+ if err != nil {+ t.Fatalf("explicit scope shrink: %v", err)+ }+ if len(h.items) != len(prior)+1 || h.items[0].ExternalID != "erp" {+ t.Fatalf("scope shrink items = %d, want selected page plus %d tombstones", len(h.items), len(prior))+ }+ deleted := 0+ for _, item := range h.items[1:] {+ if !item.IsDeleted {+ t.Fatalf("scope shrink emitted a non-tombstone outside the new scope: %#v", item)+ }+ deleted+++ }+ if deleted != len(prior) {+ t.Fatalf("scope shrink tombstones = %d, want %d", deleted, len(prior))+ }+ decoded := decodeCursor(next)+ if len(decoded.SpacePages) != 1 || decoded.SpacePages["page:1:erp"]["erp"] != "v:2" {+ t.Fatalf("scope shrink cursor retained cancelled roots: %#v", decoded.SpacePages)+ }+ if _, oldRootPresent := decoded.SpacePages["1"]; oldRootPresent {+ t.Fatalf("cancelled space root returned to cursor: %#v", decoded.SpacePages)+ }+ for _, item := range h.items[1:] {+ if item.SourceResourceID != "1" {+ t.Fatalf("tombstone source = %q, want old root 1", item.SourceResourceID)+ }+ }+}+func TestFetchStreamRefusesEmptyListingWhenBaselineExists(t *testing.T) {prior := map[string]string{"p1": "v:1", "p2": "v:1"}api := &streamAPI{}@@ -349,6 +593,50 @@ func TestFetchFullStreamResumesFromCheckpoint(t *testing.T) {}}+func TestFetchFullStreamRetryRefetchesFailedPage(t *testing.T) {+ pages := []streamPage{+ {id: "p1", title: "One", version: 1},+ {id: "p2", title: "Broken then healthy", version: 2},+ {id: "p3", title: "Three", version: 1},+ }+ api := &streamAPI{pages: pages, bodyStatus: map[string]int{"p2": http.StatusForbidden}}+ first := &captureHandler{checkpointErrAt: 2}+ old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1", "p3": "v:1"})+ _, err := newStreamConnector(api).FetchFullStream(context.Background(), streamConfig(), old, first)+ if err == nil {+ t.Fatal("interrupted full sync unexpectedly succeeded")+ }+ if len(first.checkpoints) != 1 {+ t.Fatalf("first full-sync checkpoints = %d; want one successful checkpoint", len(first.checkpoints))+ }+ checkpoint := decodeCursor(first.checkpoints[0])+ if !checkpoint.FullSync || checkpoint.FullSyncBaseline["1"]["p2"] != "v:1" ||+ checkpoint.SpacePages["1"]["p2"] != "" {+ t.Fatalf("failed page was not left pending in the full-sync checkpoint: %#v", checkpoint)+ }++ resumeAPI := &streamAPI{pages: pages}+ second := &captureHandler{}+ next, err := newStreamConnector(resumeAPI).FetchFullStream(+ context.Background(), streamConfig(), first.checkpoints[0], second,+ )+ if err != nil {+ t.Fatal(err)+ }+ if resumeAPI.bodyCalls != 2 {+ t.Fatalf("retry fetched %d bodies, want failed p2 and uncheckpointed p3", resumeAPI.bodyCalls)+ }+ if len(second.items) != 2 || second.items[0].ExternalID != "p2" || second.items[1].ExternalID != "p3" {+ t.Fatalf("retry items = %#v; failed page was skipped", second.items)+ }+ decoded := decodeCursor(next)+ if decoded.FullSync || decoded.FullSyncBaseline != nil || len(decoded.SpacePages) != 1 ||+ decoded.SpacePages["1"]["p1"] != "v:1" || decoded.SpacePages["1"]["p2"] != "v:2" ||+ decoded.SpacePages["1"]["p3"] != "v:1" {+ t.Fatalf("completed full-sync cursor = %#v", decoded)+ }+}+func TestFetchStreamContinuesAfterPageBodyError(t *testing.T) {api := &streamAPI{pages: []streamPage{@@ -430,57 +718,536 @@ func TestListResourcesKeepsInvalidWebUI(t *testing.T) {}}-func TestFetchStreamChecksUnlistedPagesWhenServerListingNeverEnds(t *testing.T) {- for _, full := range []bool{false, true} {- t.Run(fmt.Sprintf("full=%v", full), func(t *testing.T) {- api := &streamAPI{- endlessNext: true,- pages: []streamPage{{id: "p1", title: "Page", version: 2}},- probe: map[string]probeReply{- "gone": {status: http.StatusNotFound},- "moved": {status: http.StatusOK, spaceKey: "OPS"},- "kept": {status: http.StatusOK, spaceKey: "ENG"},- "denied": {status: http.StatusForbidden},- },- }- h := &captureHandler{}- old := streamCursor(map[string]string{- "p1": "v:1", "gone": "v:1", "moved": "v:1", "kept": "v:3", "denied": "v:1",- })- connector := newStreamConnector(api)- fetch := connector.FetchStream- if full {- fetch = connector.FetchFullStream- }- got, err := fetch(context.Background(), streamConfig(), old, h)- if err != nil {- t.Fatalf("fetch: %v", err)- }- if api.listCalls != 1+maxEmptyServerPages {- t.Fatalf("listCalls = %d, want %d", api.listCalls, 1+maxEmptyServerPages)+func TestListResourcesLoadsConfluencePagesLazily(t *testing.T) {+ api := &streamAPI{pages: []streamPage{{id: "p1", title: "Home", version: 1}}}+ connector := newStreamConnector(api)+ spaces, err := connector.ListResources(context.Background(), streamConfig(), "")+ if err != nil || len(spaces) != 1 || !spaces[0].HasChildren {+ t.Fatalf("spaces = %#v, %v", spaces, err)+ }+ pages, err := connector.ListResources(context.Background(), streamConfig(), "1")+ if err != nil || len(pages) != 1 {+ t.Fatalf("top pages = %#v, %v", pages, err)+ }+ if pages[0].ExternalID != "page:1:p1" || pages[0].ParentID != "1" || pages[0].Type != "page" ||+ !pages[0].HasChildren {+ t.Fatalf("page resource = %#v", pages[0])+ }+ children, err := connector.ListResources(context.Background(), streamConfig(), "page:1:p1")+ if err != nil || len(children) != 0 {+ t.Fatalf("children = %#v, %v", children, err)+ }+ if api.listCalls != 1 {+ t.Fatalf("space expansion made %d whole-space list calls", api.listCalls)+ }+ if api.ancestorCalls != 0 {+ t.Fatalf("page expansion made %d unnecessary ancestor calls", api.ancestorCalls)+ }+}++func TestPickerSpaceCacheIsPartitionedByCredentialFingerprint(t *testing.T) {+ api := &streamAPI{}+ connector := newStreamConnector(api)+ client, cfg, err := connector.configured(streamConfig())+ if err != nil {+ t.Fatal(err)+ }+ if _, err := connector.pickerSpaces(context.Background(), client, cfg); err != nil {+ t.Fatal(err)+ }+ if _, err := connector.pickerSpaces(context.Background(), client, cfg); err != nil {+ t.Fatal(err)+ }+ rotated := cfg+ rotated.secret = "rotated-password"+ if _, err := connector.pickerSpaces(context.Background(), client, rotated); err != nil {+ t.Fatal(err)+ }+ if api.spaceCalls != 2 {+ t.Fatalf("space list calls = %d; want 2 for distinct credentials", api.spaceCalls)+ }+}++func TestPickerSpaceCachePrunesExpiredCredentialEntries(t *testing.T) {+ api := &streamAPI{}+ connector := newStreamConnector(api)+ client, cfg, err := connector.configured(streamConfig())+ if err != nil {+ t.Fatal(err)+ }+ expired := cfg+ expired.secret = "expired-password"+ connector.spaceCache.entries = map[string]cachedSpaces{+ pickerSpaceCacheKey(expired): {expires: time.Now().Add(-time.Second)},+ }+ if _, err := connector.pickerSpaces(context.Background(), client, cfg); err != nil {+ t.Fatal(err)+ }+ if len(connector.spaceCache.entries) != 1 {+ t.Fatalf("cache entries = %#v; expired credential entry was retained", connector.spaceCache.entries)+ }+}++func TestFetchStreamSyncsSelectedPageSubtree(t *testing.T) {+ api := &streamAPI{pages: []streamPage{{id: "p1", title: "Selected", version: 2}}}+ ds := streamConfig()+ ds.ResourceIDs = []string{"page:1:p1"}+ h := &captureHandler{}+ next, err := newStreamConnector(api).FetchStream(context.Background(), ds, nil, h)+ if err != nil {+ t.Fatal(err)+ }+ if len(h.items) != 1 || h.items[0].SourceResourceID != "page:1:p1" || h.items[0].ExternalID != "p1" {+ t.Fatalf("items = %#v", h.items)+ }+ if decoded := decodeCursor(next); decoded.SpacePages["page:1:p1"]["p1"] != "v:2" {+ t.Fatalf("cursor = %#v", decoded)+ }+}++func TestFetchStreamReconcilesCancelledSpaceScope(t *testing.T) {+ api := &streamAPI{pages: []streamPage{{id: "p1", title: "Selected", version: 2}}}+ ds := streamConfig()+ ds.ResourceIDs = []string{"page:1:p1"}+ old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"})+ h := &captureHandler{}+ next, err := newStreamConnector(api).FetchStream(context.Background(), ds, old, h)+ if err != nil {+ t.Fatal(err)+ }+ if len(h.items) != 2 || !h.items[1].IsDeleted || h.items[1].ExternalID != "p2" {+ t.Fatalf("items = %#v", h.items)+ }+ decoded := decodeCursor(next)+ if _, oldScopePresent := decoded.SpacePages["1"]; oldScopePresent ||+ decoded.SpacePages["page:1:p1"]["p1"] != "v:2" {+ t.Fatalf("cursor = %#v", decoded)+ }+}++func TestFetchStreamPreservesMovedPageWhenBodyFails(t *testing.T) {+ api := &streamAPI{+ pages: []streamPage{{id: "p1", title: "Selected", version: 2}},+ bodyStatus: map[string]int{"p1": http.StatusNotFound},+ }+ ds := streamConfig()+ ds.ResourceIDs = []string{"page:1:p1"}+ old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"})+ h := &captureHandler{}+ next, err := newStreamConnector(api).FetchStream(context.Background(), ds, old, h)+ if err != nil {+ t.Fatal(err)+ }+ decoded := decodeCursor(next)+ if decoded.SpacePages["page:1:p1"]["p1"] != "v:1" {+ t.Fatalf("prior version was not preserved: %#v", decoded)+ }+ deletedP2 := false+ for _, item := range h.items {+ if item.IsDeleted && item.ExternalID == "p1" {+ t.Fatalf("failed page was deleted: %#v", h.items)+ }+ deletedP2 = deletedP2 || (item.IsDeleted && item.ExternalID == "p2")+ }+ if !deletedP2 {+ t.Fatalf("cancelled scope page was not deleted: %#v", h.items)+ }+}++func TestFetchStreamDoesNotDeleteWhenNewScopeCannotEnumerate(t *testing.T) {+ api := &streamAPI{pages: []streamPage{{id: "p1", title: "Selected", version: 2}}}+ connector := newStreamConnector(api)+ originalFactory := connector.newClient+ connector.newClient = func(cfg config) (*client, error) {+ c, err := originalFactory(cfg)+ if err != nil {+ return nil, err+ }+ originalTransport := c.http.Transport+ c.http.Transport = roundTripper(func(req *http.Request) (*http.Response, error) {+ if req.URL.Path == "/wiki/rest/api/content/p1" && req.URL.Query().Get("expand") == "version,space" {+ return &http.Response{+ StatusCode: http.StatusNotFound,+ Header: make(http.Header),+ Body: io.NopCloser(strings.NewReader("missing")),+ }, nil}- if api.probeCalls != 4 {- t.Fatalf("probeCalls = %d, want one per unlisted page", api.probeCalls)+ return originalTransport.RoundTrip(req)+ })+ return c, nil+ }+ ds := streamConfig()+ ds.ResourceIDs = []string{"page:1:p1"}+ h := &captureHandler{}+ _, err := connector.FetchStream(+ context.Background(),+ ds,+ streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"}),+ h,+ )+ if err == nil {+ t.Fatal("scope enumeration failure unexpectedly succeeded")+ }+ for _, item := range h.items {+ if item.IsDeleted {+ t.Fatalf("incomplete scope emitted a tombstone: %#v", h.items)+ }+ }+}++func TestBuildSyncPlanNormalizesOverlappingScopes(t *testing.T) {+ api := &streamAPI{}+ client := &client{+ cfg: config{baseURL: "https://confluence.test/wiki"},+ http: &http.Client{Transport: roundTripper(func(req *http.Request) (*http.Response, error) {+ switch req.URL.Path {+ case "/wiki/rest/api/space":+ return api.jsonResponse(+ map[string]any{"results": []any{map[string]any{"id": 1, "key": "ENG", "name": "Engineering"}}},+ )+ case "/wiki/rest/api/content/parent":+ return api.jsonResponse(+ map[string]any{"id": "parent", "space": map[string]any{"key": "ENG"}, "ancestors": []any{}},+ )+ case "/wiki/rest/api/content/child":+ return api.jsonResponse(+ map[string]any{+ "id": "child",+ "space": map[string]any{"key": "ENG"},+ "ancestors": []any{map[string]any{"id": "parent"}},+ },+ )+ default:+ return nil, errors.New("unexpected endpoint: " + req.URL.String())}- deleted := map[string]bool{}- for _, item := range h.items {- if item.IsDeleted {- deleted[item.ExternalID] = true+ })},+ }+ connector := NewConnector()+ plan, err := connector.buildSyncPlan(context.Background(), client, []string{"page:1:child", "page:1:parent"})+ if err != nil {+ t.Fatal(err)+ }+ if len(plan.Roots) != 1 || plan.Roots[0].ResourceID != "page:1:parent" {+ t.Fatalf("page overlap plan = %#v", plan)+ }+ plan, err = connector.buildSyncPlan(context.Background(), client, []string{"1", "page:1:child"})+ if err != nil {+ t.Fatal(err)+ }+ if len(plan.Roots) != 1 || plan.Roots[0].Kind != syncWholeSpace || plan.Roots[0].ResourceID != "1" {+ t.Fatalf("space overlap plan = %#v", plan)+ }+}++func TestFetchStreamCloudSyncsSelectedPageSubtree(t *testing.T) {+ api := &streamAPI{cloud: true, pages: []streamPage{{id: "p1", title: "Cloud", version: 3}}}+ ds := cloudStreamConfig()+ ds.ResourceIDs = []string{"page:1:p1"}+ h := &captureHandler{}+ next, err := newStreamConnector(api).FetchStream(context.Background(), ds, nil, h)+ if err != nil {+ t.Fatal(err)+ }+ if len(h.items) != 1 || h.items[0].ExternalID != "p1" || h.items[0].SourceResourceID != "page:1:p1" {+ t.Fatalf("items = %#v", h.items)+ }+ if decoded := decodeCursor(next); decoded.SpacePages["page:1:p1"]["p1"] != "v:3" {+ t.Fatalf("cursor = %#v", decoded)+ }+}++func TestListResourcesCloudStaysLazyWithContainerLimitation(t *testing.T) {+ api := &streamAPI{}+ var listDepths []string+ conn := &Connector{newClient: func(cfg config) (*client, error) {+ return &client{+ cfg: cfg,+ http: &http.Client{Transport: roundTripper(func(req *http.Request) (*http.Response, error) {+ switch req.URL.Path {+ case "/wiki/api/v2/spaces":+ return api.jsonResponse(+ map[string]any{+ "results": []any{map[string]any{"id": "1", "key": "ENG", "name": "Engineering"}},+ },+ )+ case "/wiki/api/v2/spaces/1/pages":+ listDepths = append(listDepths, req.URL.Query().Get("depth"))+ // depth=0 surfaces root pages only; the folder and the page nested+ // inside it are unreachable through this endpoint (CONFCLOUD-84275).+ return api.jsonResponse(map[string]any{"results": []any{+ map[string]any{"id": "top", "type": "page", "title": "Top"},+ map[string]any{"id": "folder", "type": "folder", "title": "Folder"},+ }})+ default:+ return nil, errors.New("unexpected endpoint: " + req.URL.String())}+ })},+ }, nil+ }}+ // Cloud spaces carry the limitation marker so the picker can explain why+ // pages under top-level containers are not selectable.+ spaces, err := conn.ListResources(context.Background(), cloudStreamConfig(), "")+ if err != nil {+ t.Fatal(err)+ }+ if len(spaces) != 1 || spaces[0].Metadata["hierarchy_limitation"] != "cloud_top_level_containers" {+ t.Fatalf("cloud space metadata = %#v", spaces)+ }+ resources, err := conn.ListResources(context.Background(), cloudStreamConfig(), "1")+ if err != nil {+ t.Fatal(err)+ }+ if len(resources) != 1 || resources[0].ExternalID != "page:1:top" {+ t.Fatalf("resources = %#v", resources)+ }+ // Expanding a space must stay a single shallow call, never a whole-space+ // scan, even though top-level folders hide some pages.+ if len(listDepths) != 1 || listDepths[0] != "0" {+ t.Fatalf("space expansion depths = %#v; want a single depth=0 call", listDepths)+ }+}++func TestResolveResourceAncestorsSkipsUnresolvableSelections(t *testing.T) {+ api := &streamAPI{}+ connector := &Connector{newClient: func(cfg config) (*client, error) {+ return &client{+ cfg: cfg,+ http: &http.Client{Transport: roundTripper(func(req *http.Request) (*http.Response, error) {+ switch req.URL.Path {+ case "/wiki/rest/api/space":+ return api.jsonResponse(map[string]any{"results": []any{map[string]any{+ "id": 1, "key": "ENG", "name": "Engineering",+ }}})+ case "/wiki/rest/api/content/alive":+ return api.jsonResponse(map[string]any{+ "id": "alive", "space": map[string]any{"key": "ENG"},+ "ancestors": []any{map[string]any{"id": "parent"}},+ })+ case "/wiki/rest/api/content/gone":+ return &http.Response{+ StatusCode: http.StatusNotFound,+ Header: make(http.Header),+ Body: io.NopCloser(strings.NewReader("missing page")),+ }, nil+ default:+ return nil, errors.New("unexpected endpoint: " + req.URL.String())+ }+ })},+ }, nil+ }}+ ancestors, err := connector.ResolveResourceAncestors(+ context.Background(), streamConfig(),+ []string{"page:1:alive", "page:1:gone", "page:9:elsewhere", "not-a-resource"},+ )+ if err != nil {+ t.Fatal(err)+ }+ want := []string{"1", "page:1:parent"}+ if len(ancestors) != len(want) || ancestors[0] != want[0] || ancestors[1] != want[1] {+ t.Fatalf("ancestors = %#v; want %#v", ancestors, want)+ }+}++func TestVisibleCloudPageChildrenTraversesContainers(t *testing.T) {+ api := &streamAPI{}+ client := &client{+ cfg: config{edition: editionCloud, baseURL: "https://confluence.test/wiki"},+ http: &http.Client{Transport: roundTripper(func(req *http.Request) (*http.Response, error) {+ switch req.URL.Path {+ case "/wiki/api/v2/pages/a/direct-children":+ return api.jsonResponse(map[string]any{"results": []any{+ map[string]any{"id": "b", "type": "page", "title": "B"},+ map[string]any{"id": "folder", "type": "folder", "title": "Folder"},+ }})+ case "/wiki/api/v2/folders/folder/direct-children":+ return api.jsonResponse(map[string]any{"results": []any{+ map[string]any{"id": "c", "type": "page", "title": "C"},+ map[string]any{"id": "db", "type": "database", "title": "DB"},+ }})+ case "/wiki/api/v2/databases/db/direct-children":+ return api.jsonResponse(+ map[string]any{"results": []any{map[string]any{"id": "e", "type": "page", "title": "E"}}},+ )+ default:+ return nil, errors.New("unexpected endpoint: " + req.URL.String())}- if len(deleted) != 2 || !deleted["gone"] || !deleted["moved"] {- t.Fatalf("deleted = %v, want only the pages Confluence confirmed gone from the space", deleted)- }- pages := decodeCursor(got).SpacePages["1"]- want := map[string]string{"p1": "v:2", "kept": "v:3", "denied": "v:1"}- if len(pages) != len(want) {- t.Fatalf("cursor pages = %#v, want %#v", pages, want)+ })},+ }+ pages, err := client.visibleCloudPageChildren(context.Background(), "a")+ if err != nil {+ t.Fatal(err)+ }+ if len(pages) != 3 || pages[0].ID != "b" || pages[1].ID != "c" || pages[2].ID != "e" {+ t.Fatalf("projected pages = %#v", pages)+ }+}++func TestCloudPageSubtreeWalksDeepMixedHierarchy(t *testing.T) {+ api := &streamAPI{}+ client := &client{+ cfg: config{edition: editionCloud, baseURL: "https://confluence.test/wiki"},+ http: &http.Client{Transport: roundTripper(func(req *http.Request) (*http.Response, error) {+ if req.Method != http.MethodGet {+ return nil, fmt.Errorf("unexpected method %s", req.Method)}- for id, version := range want {- if pages[id] != version {- t.Fatalf("cursor pages = %#v, want %#v", pages, want)- }+ switch req.URL.Path {+ case "/wiki/api/v2/pages/root":+ return api.jsonResponse(map[string]any{"id": "root", "spaceId": "1"})+ case "/wiki/api/v2/pages/p1":+ return api.jsonResponse(map[string]any{+ "id": "p1", "spaceId": "1", "version": map[string]any{"number": 2},+ })+ case "/wiki/api/v2/pages/p2":+ return api.jsonResponse(map[string]any{+ "id": "p2", "spaceId": "1", "version": map[string]any{"number": 3},+ })+ case "/wiki/api/v2/pages/p3":+ return api.jsonResponse(map[string]any{+ "id": "p3", "spaceId": "1", "version": map[string]any{"number": 4},+ })+ case "/wiki/api/v2/pages/root/direct-children":+ return api.jsonResponse(map[string]any{"results": []any{+ map[string]any{"id": "p1", "type": "page", "title": "First page"},+ map[string]any{"id": "folder", "type": "folder", "title": "Folder"},+ }})+ case "/wiki/api/v2/pages/p1/direct-children":+ return api.jsonResponse(map[string]any{"results": []any{+ map[string]any{"id": "db", "type": "database", "title": "Database"},+ }})+ case "/wiki/api/v2/folders/folder/direct-children":+ return api.jsonResponse(map[string]any{"results": []any{+ map[string]any{"id": "p2", "type": "page", "title": "Second page"},+ }})+ case "/wiki/api/v2/databases/db/direct-children":+ return api.jsonResponse(map[string]any{"results": []any{+ map[string]any{"id": "p3", "type": "page", "title": "Deep page"},+ }})+ case "/wiki/api/v2/pages/p2/direct-children", "/wiki/api/v2/pages/p3/direct-children":+ return api.jsonResponse(map[string]any{"results": []any{}})+ default:+ return nil, fmt.Errorf("unexpected endpoint: %s", req.URL.String())}- })+ })},+ }++ pages, err := client.pageSubtree(context.Background(), space{ID: "1"}, "root")+ if err != nil {+ t.Fatal(err)+ }+ want := []string{"root", "p1", "p2", "p3"}+ if len(pages) != len(want) {+ t.Fatalf("subtree pages = %#v; want IDs %v", pages, want)+ }+ for i, id := range want {+ if pages[i].ID != id || pages[i].SpaceID != "1" {+ t.Fatalf("subtree page[%d] = %#v; want ID %q in space 1", i, pages[i], id)+ }+ }+}++func TestFetchStreamChecksUnlistedPagesWhenServerListingNeverEnds(t *testing.T) {+ api := &streamAPI{+ endlessNext: true,+ pages: []streamPage{{id: "p1", title: "Page", version: 2}},+ probe: map[string]probeReply{+ "gone": {status: http.StatusNotFound},+ "kept": {status: http.StatusOK, spaceKey: "ENG"},+ },+ }+ h := &captureHandler{}+ old := streamCursor(map[string]string{"p1": "v:1", "gone": "v:1", "kept": "v:1"})+ got, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h)+ if err != nil {+ t.Fatal(err)+ }+ if api.listCalls != 1+maxEmptyServerPages || api.probeCalls != 2 {+ t.Fatalf("list/probe calls = %d/%d", api.listCalls, api.probeCalls)+ }+ if len(h.items) != 2 || !h.items[1].IsDeleted || h.items[1].ExternalID != "gone" {+ t.Fatalf("items = %#v", h.items)+ }+ pages := decodeCursor(got).SpacePages["1"]+ if pages["p1"] != "v:2" || pages["kept"] != "v:1" || len(pages) != 2 {+ t.Fatalf("cursor pages = %#v", pages)+ }+}++func TestFetchStreamDoesNotRestoreCancelledRootForPageFoundByPartialProbe(t *testing.T) {+ api := &streamAPI{+ endlessNext: true,+ probe: map[string]probeReply{+ "kept": {status: http.StatusOK, spaceKey: "ENG"},+ },+ }+ h := &captureHandler{}+ old := (&cursor{SpacePages: map[string]map[string]string{+ "page:1:old-root": {"kept": "v:1"},+ }}).syncCursor()+ next, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h)+ if err != nil {+ t.Fatal(err)+ }+ if len(h.items) != 0 {+ t.Fatalf("confirmed-present page emitted items: %#v", h.items)+ }+ decoded := decodeCursor(next)+ if len(decoded.SpacePages) != 1 || decoded.SpacePages["1"]["kept"] != "v:1" {+ t.Fatalf("reconciled cursor = %#v; expected only the current root", decoded.SpacePages)+ }+ if _, oldRootPresent := decoded.SpacePages["page:1:old-root"]; oldRootPresent {+ t.Fatalf("cancelled root was restored to cursor: %#v", decoded.SpacePages)+ }+}++func TestFetchStreamDoesNotTombstoneWhenPartialListingProbeFails(t *testing.T) {+ api := &streamAPI{+ endlessNext: true,+ probe: map[string]probeReply{+ "gone": {status: http.StatusNotFound},+ "unknown": {status: http.StatusForbidden},+ },+ }+ h := &captureHandler{}+ old := streamCursor(map[string]string{"gone": "v:1", "unknown": "v:1"})+ _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h)+ if err == nil {+ t.Fatal("unverified partial-listing candidate unexpectedly completed reconciliation")+ }+ for _, item := range h.items {+ if item.IsDeleted {+ t.Fatalf("probe failure allowed a tombstone: %#v", h.items)+ }+ }+ if len(h.checkpoints) != 0 {+ t.Fatalf("probe failure checkpointed reconciliation: %#v", h.checkpoints)+ }+ if api.probeCalls == 0 {+ t.Fatal("empty incomplete listing was rejected before checking prior pages")+ }+}++func TestFetchStreamDefersWhenPartialListingProbeBudgetIsExhausted(t *testing.T) {+ prior := make(map[string]string, maxDeletionProbes+1)+ probes := make(map[string]probeReply, maxDeletionProbes+1)+ for i := 0; i <= maxDeletionProbes; i++ {+ id := fmt.Sprintf("p%03d", i)+ prior[id] = "v:1"+ probes[id] = probeReply{status: http.StatusNotFound}+ }+ api := &streamAPI{endlessNext: true, probe: probes}+ h := &captureHandler{}+ _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), streamCursor(prior), h)+ if err == nil {+ t.Fatal("partial listing with unverified candidates unexpectedly completed")+ }+ if api.probeCalls != maxDeletionProbes {+ t.Fatalf("existence probes = %d; want capped at %d", api.probeCalls, maxDeletionProbes)+ }+ if len(h.items) != 0 || len(h.checkpoints) != 0 {+ t.Fatalf(+ "probe budget exhaustion emitted or checkpointed reconciliation: items=%#v checkpoints=%#v",+ h.items,+ h.checkpoints,+ )}}diff --git a/internal/datasource/connector/confluence/resource.go b/internal/datasource/connector/confluence/resource.gonew file mode 100644index 000000000..a34a16195--- /dev/null+++ b/internal/datasource/connector/confluence/resource.go@@ -0,0 +1,43 @@+package confluence++import (+ "fmt"+ "strings"+)++type resourceKind uint8++const (+ resourceSpace resourceKind = iota+ resourcePage+)++// resourceRef is the canonical, persisted representation of a selection.+// Space IDs deliberately remain unprefixed for backwards compatibility.+type resourceRef struct {+ Kind resourceKind+ SpaceID string+ PageID string+}++func makePageResourceID(spaceID, pageID string) string {+ return "page:" + spaceID + ":" + pageID+}++func parseResourceID(id string) (resourceRef, error) {+ id = strings.TrimSpace(id)+ if id == "" {+ return resourceRef{}, fmt.Errorf("invalid empty Confluence resource ID")+ }+ if !strings.HasPrefix(id, "page:") {+ if strings.Contains(id, ":") {+ return resourceRef{}, fmt.Errorf("invalid Confluence space resource ID %q", id)+ }+ return resourceRef{Kind: resourceSpace, SpaceID: id}, nil+ }+ parts := strings.Split(id, ":")+ if len(parts) != 3 || parts[0] != "page" || strings.TrimSpace(parts[1]) == "" || strings.TrimSpace(parts[2]) == "" {+ return resourceRef{}, fmt.Errorf("invalid Confluence page resource ID %q", id)+ }+ return resourceRef{Kind: resourcePage, SpaceID: parts[1], PageID: parts[2]}, nil+}diff --git a/internal/datasource/connector/confluence/resource_test.go b/internal/datasource/connector/confluence/resource_test.gonew file mode 100644index 000000000..b03e92f2d--- /dev/null+++ b/internal/datasource/connector/confluence/resource_test.go@@ -0,0 +1,23 @@+package confluence++import "testing"++func TestParseResourceID(t *testing.T) {+ space, err := parseResourceID("86802433")+ if err != nil || space.Kind != resourceSpace || space.SpaceID != "86802433" {+ t.Fatalf("space = %#v, %v", space, err)+ }+ page, err := parseResourceID("page:86802433:123456789")+ if err != nil || page.Kind != resourcePage || page.SpaceID != "86802433" || page.PageID != "123456789" {+ t.Fatalf("page = %#v, %v", page, err)+ }+ invalid := []string{+ "", "page:", "page:100", "page::200", "page:100:", "page:100:200:300",+ "node:86802433:folder-1", "space:100",+ }+ for _, id := range invalid {+ if _, err := parseResourceID(id); err == nil {+ t.Errorf("parseResourceID(%q) succeeded", id)+ }+ }+}diff --git a/internal/datasource/connector/confluence/sync_plan.go b/internal/datasource/connector/confluence/sync_plan.gonew file mode 100644index 000000000..c61484d92--- /dev/null+++ b/internal/datasource/connector/confluence/sync_plan.go@@ -0,0 +1,112 @@+package confluence++import (+ "context"+ "fmt"+ "sort"+)++type syncRootKind uint8++const (+ syncWholeSpace syncRootKind = iota+ syncPageSubtree+)++type syncRoot struct {+ ResourceID string+ SpaceID string+ Space space+ PageID string+ Kind syncRootKind+}++type syncPlan struct{ Roots []syncRoot }++// buildSyncPlan validates selections before any destructive reconciliation and+// removes scopes which are already covered by a selected ancestor.+func (c *Connector) buildSyncPlan(ctx context.Context, client *client, resourceIDs []string) (*syncPlan, error) {+ spaces, err := client.spaces(ctx)+ if err != nil {+ return nil, fmt.Errorf("list Confluence spaces: %w", err)+ }+ byID := make(map[string]space, len(spaces))+ for _, s := range spaces {+ byID[s.ID] = s+ }++ plan := &syncPlan{}+ wholeSpaces := make(map[string]bool)+ pageRoots := make(map[string]syncRoot)+ ancestorPaths := make(map[string][]page)+ for _, resourceID := range resourceIDs {+ ref, err := parseResourceID(resourceID)+ if err != nil {+ return nil, err+ }+ s, ok := byID[ref.SpaceID]+ if !ok {+ return nil, fmt.Errorf(+ "selected Confluence space %s is unavailable; refusing destructive reconciliation",+ ref.SpaceID,+ )+ }+ if ref.Kind == resourceSpace {+ wholeSpaces[ref.SpaceID] = true+ continue+ }+ // pageAncestors fetches the page itself on Server/DC and Cloud; that is+ // also the authoritative check that a client did not forge its space ID.+ ancestors, err := client.pageAncestors(ctx, s, ref.PageID)+ if err != nil {+ return nil, fmt.Errorf("validate selected Confluence page %s: %w", ref.PageID, err)+ }+ pageRoots[resourceID] = syncRoot{+ ResourceID: resourceID,+ SpaceID: ref.SpaceID,+ Space: s,+ PageID: ref.PageID,+ Kind: syncPageSubtree,+ }+ ancestorPaths[resourceID] = ancestors+ for _, ancestor := range ancestors {+ if ancestor.ID == ref.PageID {+ return nil, fmt.Errorf("invalid cyclic Confluence page ancestry for %s", ref.PageID)+ }+ }+ }+ spaceIDs := make([]string, 0, len(wholeSpaces))+ for id := range wholeSpaces {+ spaceIDs = append(spaceIDs, id)+ }+ sort.Strings(spaceIDs)+ for _, id := range spaceIDs {+ plan.Roots = append(plan.Roots, syncRoot{ResourceID: id, SpaceID: id, Space: byID[id], Kind: syncWholeSpace})+ }+ pageIDs := make([]string, 0, len(pageRoots))+ for id := range pageRoots {+ pageIDs = append(pageIDs, id)+ }+ sort.Strings(pageIDs)+ for _, id := range pageIDs {+ root := pageRoots[id]+ if wholeSpaces[root.SpaceID] {+ continue+ }+ ancestors := ancestorPaths[id]+ covered := false+ for _, ancestor := range ancestors {+ if ancestor.Kind != "" && ancestor.Kind != "page" {+ continue+ }+ if _, ok := pageRoots[makePageResourceID(root.SpaceID, ancestor.ID)]; ok {+ covered = true+ break+ }+ }+ if !covered {+ plan.Roots = append(plan.Roots, pageRoots[id])+ }+ }+ return plan, nil+}diff --git a/internal/datasource/connector/confluence/types.go b/internal/datasource/connector/confluence/types.goindex 7884f349d..498288083 100644--- a/internal/datasource/connector/confluence/types.go+++ b/internal/datasource/connector/confluence/types.go@@ -95,10 +95,11 @@ func isAtlassianCloudHost(host string) bool {}type space struct {- ID string `json:"id"`- Key string `json:"key"`- Name string `json:"name"`- Links struct {+ ID string `json:"id"`+ Key string `json:"key"`+ Name string `json:"name"`+ HomepageID string `json:"homepageId"`+ Links struct {WebUI string `json:"webui"`} `json:"_links"`}@@ -125,10 +126,13 @@ type serverSpaceList struct {}type page struct {- ID string `json:"id"`- Title string `json:"title"`- Status string `json:"status"`- Space struct {+ ID string `json:"id"`+ Title string `json:"title"`+ Status string `json:"status"`+ Kind string `json:"-"`+ SpaceID string `json:"spaceId"`+ Ancestors []page `json:"ancestors"`+ Space struct {Key string `json:"key"`Name string `json:"name"`} `json:"space"`@@ -189,6 +193,27 @@ type cloudPage struct {} `json:"_links"`}+// cloudHierarchyItem is shared by Cloud direct-children, descendants, and+// ancestors. Non-page items are intentionally retained by the API model so we+// can filter them explicitly at the navigation/import boundary.+type cloudHierarchyItem struct {+ ID string `json:"id"`+ Type string `json:"type"`+ Title string `json:"title"`+ SpaceID string `json:"spaceId"`+ Status string `json:"status"`+ Links struct {+ WebUI string `json:"webui"`+ } `json:"_links"`+}++type cloudHierarchyList struct {+ Results []cloudHierarchyItem `json:"results"`+ Links struct {+ Next string `json:"next"`+ } `json:"_links"`+}+type cloudPageList struct {Results []cloudPage `json:"results"`Links struct {@@ -278,14 +303,37 @@ func prepareSyncCursors(old *types.SyncCursor, forceFull bool) (baseline, next creturn baseline, next}-func pageVersion(p page) string {+// removePageOwnership drops a page from every root ownership in the cursor.+// It runs after each tombstone emit so the checkpoint mirrors the deletion;+// otherwise the page would linger as a ghost, be re-tombstoned on every later+// run, and eventually trip the mass-deletion guard. During a full sync the+// deletion baseline lives in FullSyncBaseline, not SpacePages (which is being+// rebuilt from scratch), so both maps must record the completed tombstone for+// a retry to resume with the remaining candidates only.+func removePageOwnership(c *cursor, pageID string) {+ removeFromPageMap(c.SpacePages, pageID)+ if c.FullSync {+ removeFromPageMap(c.FullSyncBaseline, pageID)+ }+}++func removeFromPageMap(m map[string]map[string]string, pageID string) {+ for _, pages := range m {+ delete(pages, pageID)+ }+}++func pageVersion(p page) (string, bool) {if p.Version.Number > 0 {- return fmt.Sprintf("v:%d", p.Version.Number)+ return fmt.Sprintf("v:%d", p.Version.Number), true}if p.Version.When != "" {- return "t:" + p.Version.When+ return "t:" + p.Version.When, true+ }+ if p.Version.CreatedAt != "" {+ return "t:" + p.Version.CreatedAt, true}- return "t:" + p.Version.CreatedAt+ return "", false}func pageUpdatedAt(p page) time.Time {diff --git a/internal/datasource/connector/confluence/types_test.go b/internal/datasource/connector/confluence/types_test.goindex 6a2247f03..bf62f7f34 100644--- a/internal/datasource/connector/confluence/types_test.go+++ b/internal/datasource/connector/confluence/types_test.go@@ -75,6 +75,34 @@ func TestPageFileNameIncludesID(t *testing.T) {}}+func TestPageVersionUnknown(t *testing.T) {+ withNumber := page{}+ withNumber.Version.Number = 7+ withWhen := page{}+ withWhen.Version.When = "2026-01-02T03:04:05Z"+ withCreatedAt := page{}+ withCreatedAt.Version.CreatedAt = "2026-01-02T03:04:05Z"++ for _, tc := range []struct {+ name string+ in page+ want string+ wantKnown bool+ }{+ {"number", withNumber, "v:7", true},+ {"when", withWhen, "t:2026-01-02T03:04:05Z", true},+ {"createdAt", withCreatedAt, "t:2026-01-02T03:04:05Z", true},+ {"unknown", page{}, "", false},+ } {+ t.Run(tc.name, func(t *testing.T) {+ got, known := pageVersion(tc.in)+ if got != tc.want || known != tc.wantKnown {+ t.Fatalf("pageVersion() = (%q, %v); want (%q, %v)", got, known, tc.want, tc.wantKnown)+ }+ })+ }+}+func TestPrepareSyncCursorsPreservesFullSyncBaseline(t *testing.T) {old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"})baseline, next := prepareSyncCursors(old, true)