Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

feat(confluence): add directory sync and private image assets (#3319)

jacentsaoe7c27ccc9e7f
diff --git a/frontend/src/api/datasource/index.ts b/frontend/src/api/datasource/index.ts
index 1ac7ac6d2..19096307f 100644
--- a/frontend/src/api/datasource/index.ts
+++ b/frontend/src/api/datasource/index.ts
@@ -82,6 +82,8 @@ export interface Resource {
url: string
parent_id?: string
has_children?: boolean
+ /** Connector-specific hints, e.g. Confluence Cloud's top-level-container limitation. */
+ metadata?: Record<string, any>
}
// --- API calls ---
diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts
index 1c7e6dcf1..681150b9f 100755
--- a/frontend/src/i18n/locales/en-US.ts
+++ b/frontend/src/i18n/locales/en-US.ts
@@ -6979,6 +6979,9 @@ export default {
authHeaders: 'Custom headers (optional)',
authHeadersHint: 'For private feeds. One per line in "Name: Value" form, e.g. Authorization: Bearer xxxx'
},
+ confluence: {
+ cloudFolderLimitation: 'Confluence Cloud cannot list pages stored directly under top-level folders in this space; selecting the whole space still syncs them.'
+ },
comingSoon: 'Coming soon',
docHint: 'Get credentials at:',
openDoc: 'Open documentation',
diff --git a/frontend/src/i18n/locales/ja-JP.ts b/frontend/src/i18n/locales/ja-JP.ts
index ac272ac64..308b3c0f7 100644
--- a/frontend/src/i18n/locales/ja-JP.ts
+++ b/frontend/src/i18n/locales/ja-JP.ts
@@ -6979,6 +6979,9 @@ export default {
authHeaders: 'カスタムヘッダー(任意)',
authHeadersHint: '非公開フィード用です。1行に1つ「名前: 値」の形式で入力します(例: Authorization: Bearer xxxx)'
},
+ confluence: {
+ cloudFolderLimitation: 'Confluence Cloud では、スペース直下のフォルダーなどコンテナ配下のページをここに一覧できません。スペース全体を選択すれば同期されます。'
+ },
comingSoon: '近日対応予定',
docHint: '認証情報の取得先:',
openDoc: 'ドキュメントを開く',
diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts
index ead7f7139..9bba3b96a 100755
--- a/frontend/src/i18n/locales/ko-KR.ts
+++ b/frontend/src/i18n/locales/ko-KR.ts
@@ -1125,6 +1125,9 @@ export default {
authHeaders: '사용자 지정 헤더 (선택)',
authHeadersHint: '비공개 피드 접근용. 한 줄에 하나씩 「이름: 값」 형식으로 입력하세요. 예: Authorization: Bearer xxxx'
},
+ confluence: {
+ cloudFolderLimitation: 'Confluence Cloud는 아직 스페이스 최상위 폴더 등 컨테이너 아래의 페이지를 여기에 나열할 수 없습니다. 스페이스 전체를 선택하면 함께 동기화됩니다.'
+ },
connectorDesc: {
feishu: '페이슈 위키에서 문서, 스프레드시트, 파일 동기화',
lark: 'Lark 위키에서 문서, 스프레드시트, 파일 동기화',
diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts
index 4f075e0bd..d5b8fb166 100755
--- a/frontend/src/i18n/locales/ru-RU.ts
+++ b/frontend/src/i18n/locales/ru-RU.ts
@@ -1125,6 +1125,9 @@ export default {
authHeaders: 'Пользовательские заголовки (необязательно)',
authHeadersHint: 'Для приватных лент. По одному в строке в формате «Имя: Значение», например Authorization: Bearer xxxx'
},
+ confluence: {
+ cloudFolderLimitation: 'Confluence Cloud пока не может показать здесь страницы из папок верхнего уровня пространства; при выборе всего пространства они будут синхронизированы.'
+ },
connectorDesc: {
feishu: 'Синхронизация документов, таблиц и файлов из Feishu Wiki',
lark: 'Синхронизация документов, таблиц и файлов из Lark Wiki',
diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts
index 073f86492..5497981a0 100755
--- a/frontend/src/i18n/locales/zh-CN.ts
+++ b/frontend/src/i18n/locales/zh-CN.ts
@@ -1127,6 +1127,9 @@ export default {
authHeaders: '自定义请求头(可选)',
authHeadersHint: '用于访问私有订阅源,每行一个,格式为「名称: 值」,例如 Authorization: Bearer xxxx'
},
+ confluence: {
+ cloudFolderLimitation: 'Confluence Cloud 暂不支持在此列出空间顶层文件夹等容器下的页面;选择整个空间仍会同步它们。'
+ },
connectorDesc: {
feishu: '同步飞书知识库中的文档、表格、文件',
lark: '同步 Lark 知识库中的文档、表格、文件(飞书国际版)',
diff --git a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.test.ts b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.test.ts
index 43c46c765..f9f2d1267 100644
--- a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.test.ts
+++ b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.test.ts
@@ -157,3 +157,25 @@ test('new GitLab data sources continue to test credentials without persistence',
assert.deepEqual(f.calls.map(call => call.method), ['validateCredentials'])
} finally { f.close() }
})
+
+test('Cloud hierarchy limitation stays visible after an empty space expansion', async () => {
+ const f = await fixture()
+ try {
+ f.vm.resources = [{
+ external_id: 'space-1',
+ name: 'Cloud space',
+ type: 'space',
+ has_children: true,
+ metadata: { hierarchy_limitation: 'cloud_top_level_containers' },
+ }]
+ f.vm.expandedResourceIds = new Set(['space-1'])
+ await nextTick()
+ assert.equal(f.vm.visibleTree.some((row: any) => row.noticeAfter), true)
+
+ f.vm.expandedResourceIds = new Set()
+ f.vm.loadedChildrenIds = new Set(['space-1'])
+ f.vm.resources = [{ ...f.vm.resources[0], has_children: false }]
+ await nextTick()
+ assert.equal(f.vm.visibleTree.some((row: any) => row.noticeAfter), true)
+ } finally { f.close() }
+})
diff --git a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue
index 44b16aa43..cb60d2bb6 100644
--- a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue
+++ b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue
@@ -449,6 +449,16 @@ async function ensureChildrenLoaded(id: string) {
if (!existing.has(c.external_id)) merged.push(c)
}
resources.value = merged
+ } else {
+ // Connectors may conservatively advertise HasChildren to avoid an N+1
+ // probe. Once lazy loading proves this is a leaf, collapse it and retain
+ // that fact so another expand cannot trigger another empty request.
+ resources.value = resources.value.map(r => r.external_id === id
+ ? { ...r, has_children: false }
+ : r)
+ const expanded = new Set(expandedResourceIds.value)
+ expanded.delete(id)
+ expandedResourceIds.value = expanded
}
loadedChildrenIds.value = new Set(loadedChildrenIds.value).add(id)
} catch (e: any) {
@@ -466,13 +476,21 @@ async function ensureChildrenLoaded(id: string) {
const visibleTree = computed(() => {
const roots = resources.value.filter(r => !r.parent_id)
- const result: { resource: Resource; depth: number }[] = []
+ const result: { resource: Resource; depth: number; noticeAfter?: boolean }[] = []
function walk(items: Resource[], depth: number) {
for (const r of items) {
result.push({ resource: r, depth })
if (r.has_children && expandedResourceIds.value.has(r.external_id)) {
walk(childrenMap.value.get(r.external_id) || [], depth + 1)
}
+ // Keep this visible after an empty root listing is recognized as a leaf.
+ if (
+ r.metadata?.hierarchy_limitation === 'cloud_top_level_containers' &&
+ (expandedResourceIds.value.has(r.external_id) ||
+ (!r.has_children && loadedChildrenIds.value.has(r.external_id)))
+ ) {
+ result.push({ resource: r, depth: depth + 1, noticeAfter: true })
+ }
}
}
walk(roots, 0)
@@ -1720,9 +1738,12 @@ const drawerConfirmText = computed(() => {
</div>
</div>
<div class="resource-picker__list" role="tree">
+ <template
+ v-for="{ resource: r, depth, noticeAfter } in visibleTree"
+ :key="noticeAfter ? `${r.external_id}__notice` : r.external_id"
+ >
<div
- v-for="{ resource: r, depth } in visibleTree"
- :key="r.external_id"
+ v-if="!noticeAfter"
class="resource-picker__row"
:class="{
'is-checked': resourceRowState(r.external_id) === 'checked',
@@ -1787,6 +1808,14 @@ const drawerConfirmText = computed(() => {
>{{ resourceTypeLabel(r.type) }}</span>
</span>
</div>
+ <p
+ v-else
+ class="resource-picker__notice"
+ :style="{ '--depth': depth }"
+ >
+ {{ t('datasource.confluence.cloudFolderLimitation') }}
+ </p>
+ </template>
</div>
</div>
<div v-else class="ds-resource-empty">
@@ -2576,6 +2605,19 @@ const drawerConfirmText = computed(() => {
background: color-mix(in srgb, var(--td-text-color-placeholder) 8%, transparent);
}
+.resource-picker__notice {
+ --depth: 0;
+ margin: 0 0 2px;
+ padding: 4px 8px 4px calc(8px + var(--depth) * 14px);
+ font-size: var(--app-text-xs);
+ line-height: 1.5;
+ color: var(--td-text-color-placeholder);
+}
+
+.resource-picker__notice:last-child {
+ margin-bottom: 0;
+}
+
/* --- Step 2: empty state --- */
.ds-resource-empty {
text-align: center;
diff --git a/internal/datasource/connector/confluence/asset_resolver.go b/internal/datasource/connector/confluence/asset_resolver.go
new file mode 100644
index 000000000..f8b06f606
--- /dev/null
+++ b/internal/datasource/connector/confluence/asset_resolver.go
@@ -0,0 +1,301 @@
+package confluence
+
+import (
+ "context"
+ "encoding/base64"
+ "errors"
+ "html"
+ "net/url"
+ "regexp"
+ "strconv"
+ "strings"
+ "sync"
+
+ "github.com/Tencent/WeKnora/internal/logger"
+)
+
+const (
+ // maxPageImages is the total data URI image budget one page may carry into the
+ // downstream docparser, which enforces a single per-page limit (maxRemoteImages)
+ // across every data:image URI regardless of origin. The budget is shared: it
+ // covers data URI images already present in the page plus the ones the resolver
+ // generates, and a repeated image consumes it per occurrence, not per unique URL.
+ // Downloads stay deduplicated by URL, and only occurrences within the remaining
+ // budget are collected, so the connector never emits more data URIs than the
+ // pipeline will keep.
+ maxPageImages = 30
+ // maxPageInlineBytes caps the total size of the data URIs one page may carry,
+ // both in the download cache and in the rewritten HTML. Without it an extreme
+ // page (maxPageImages occurrences of maxImageBytes images) would inline hundreds
+ // of MB of base64, copied again by every downstream conversion. Images past the
+ // budget keep their original src, like any other image the resolver skips.
+ maxPageInlineBytes = 50 << 20
+ // imageDownloadConcurrency bounds in-flight image downloads per page so an
+ // image-heavy page cannot exhaust HTTP connections or spike memory.
+ imageDownloadConcurrency = 4
+)
+
+// errImagePageBudget marks an image dropped because the page's inline budget
+// (maxPageInlineBytes) was already used up.
+var errImagePageBudget = errors.New("confluence page image budget exhausted")
+
+// imgTagRe matches a full <img ...> start tag, tolerating '>' inside quoted
+// attribute values. The \b keeps it from matching <image>.
+var imgTagRe = regexp.MustCompile(`(?i)<img\b(?:[^>"']|"[^"]*"|'[^']*')*>`)
+
+// srcAttrRe captures the src attribute value. The leading \s and the value group
+// keep it from matching data-src/srcset, and support double-quoted,
+// single-quoted, or bare values. Group 1 is the "src=" prefix, group 2 the value.
+var srcAttrRe = regexp.MustCompile(`(?i)(\ssrc\s*=\s*)("[^"]*"|'[^']*'|[^\s"'>]+)`)
+
+type imgAction uint8
+
+const (
+ imgKeep imgAction = iota // empty, data:, external, or non-http scheme: leave untouched
+ imgDownload // same-origin private candidate
+)
+
+// assetResolver rewrites same-origin private Confluence images in a page's
+// body.view HTML into inline data URIs. Persistence and resource:// rewriting
+// are left entirely to the downstream docparser image pipeline; the resolver
+// only performs the one step that pipeline cannot do on its own: an
+// authenticated fetch of Confluence-private bytes.
+type assetResolver struct {
+ client *client
+}
+
+func newAssetResolver(c *client) *assetResolver { return &assetResolver{client: c} }
+
+// Resolve returns content with same-origin private <img src> occurrences replaced
+// by data URIs, within the per-page budget that Resolve shares with any data URI
+// images already in the page. It is best-effort: an image that is skipped, falls
+// past the remaining budget, or fails to download keeps its original src, and image
+// problems never fail the page. Downloads are deduplicated by URL, but the budget
+// counts written data URIs (occurrences) to match the downstream pipeline, which
+// caps every data:image URI per page regardless of where it came from.
+func (r *assetResolver) Resolve(ctx context.Context, content string) string {
+ if r == nil || r.client == nil {
+ return content
+ }
+ // Data URI images already in the page consume the same downstream budget as the
+ // ones we generate, so reserve room for them before inlining any private image.
+ available := maxPageImages - countDataURIImages(content)
+ if available <= 0 {
+ return content
+ }
+ candidates := r.collect(content, available)
+ if len(candidates) == 0 {
+ return content
+ }
+ cache := r.downloadAll(ctx, candidates)
+ if len(cache) == 0 {
+ return content
+ }
+ inlined, inlinedBytes := 0, 0
+ budgetSpent := false
+ return imgTagRe.ReplaceAllStringFunc(content, func(tag string) string {
+ if inlined >= available || budgetSpent {
+ return tag
+ }
+ src, ok := extractImageSrc(tag)
+ if !ok {
+ return tag
+ }
+ action, abs := r.classify(src)
+ if action != imgDownload {
+ return tag
+ }
+ dataURI := cache[abs]
+ if dataURI == "" {
+ return tag
+ }
+ if inlinedBytes+len(dataURI) > maxPageInlineBytes {
+ budgetSpent = true
+ return tag
+ }
+ inlined++
+ inlinedBytes += len(dataURI)
+ return replaceImageSrc(tag, dataURI)
+ })
+}
+
+// collect returns the unique same-origin download candidates whose first
+// occurrence falls within the remaining inline budget, in document order. Because
+// Resolve writes at most budget data URIs, a URL first seen after that many
+// occurrences can never be inlined, so it is not downloaded. Duplicates collapse to
+// one entry, so a repeated image is fetched once yet may be inlined at every
+// occurrence inside the budget.
+func (r *assetResolver) collect(content string, budget int) []string {
+ tags := imgTagRe.FindAllString(content, -1)
+ order := make([]string, 0, len(tags))
+ seen := make(map[string]struct{}, len(tags))
+ occurrences := 0
+ for _, tag := range tags {
+ src, ok := extractImageSrc(tag)
+ if !ok {
+ continue
+ }
+ action, abs := r.classify(src)
+ if action != imgDownload {
+ continue
+ }
+ if occurrences >= budget {
+ break
+ }
+ occurrences++
+ if _, dup := seen[abs]; dup {
+ continue
+ }
+ seen[abs] = struct{}{}
+ order = append(order, abs)
+ }
+ return order
+}
+
+// countDataURIImages returns how many <img> tags in content already carry a
+// data:image URI. The downstream docparser counts every data:image occurrence
+// against the same per-page limit whether it came from the page or from us, so
+// these must be reserved before the resolver inlines any private Confluence image.
+func countDataURIImages(content string) int {
+ count := 0
+ for _, tag := range imgTagRe.FindAllString(content, -1) {
+ src, ok := extractImageSrc(tag)
+ if !ok {
+ continue
+ }
+ if strings.HasPrefix(strings.ToLower(src), "data:image/") {
+ count++
+ }
+ }
+ return count
+}
+
+// downloadAll fetches each candidate with bounded concurrency and returns a
+// URL-keyed cache of the successful ones as data URIs, holding at most
+// maxPageInlineBytes of them. Failures and images past the budget are logged
+// (redacted) and omitted, so the rewrite pass leaves their original src in place.
+func (r *assetResolver) downloadAll(ctx context.Context, urls []string) map[string]string {
+ cache := make(map[string]string, len(urls))
+ cachedBytes := 0
+ var mu sync.Mutex
+ var wg sync.WaitGroup
+ sem := make(chan struct{}, imageDownloadConcurrency)
+ for _, u := range urls {
+ if ctx.Err() != nil {
+ break
+ }
+ wg.Add(1)
+ sem <- struct{}{}
+ go func(abs string) {
+ defer wg.Done()
+ defer func() { <-sem }()
+ data, mime, err := r.client.downloadImage(ctx, abs)
+ if err != nil {
+ logImageFailure(ctx, abs, err)
+ return
+ }
+ uri := "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(data)
+ mu.Lock()
+ fits := cachedBytes+len(uri) <= maxPageInlineBytes
+ if fits {
+ cache[abs] = uri
+ cachedBytes += len(uri)
+ }
+ mu.Unlock()
+ if !fits {
+ logImageFailure(ctx, abs, errImagePageBudget)
+ }
+ }(u)
+ }
+ wg.Wait()
+ return cache
+}
+
+// classify decides whether src is a same-origin private candidate to download or
+// should be left untouched, returning the normalized absolute URL for downloads.
+func (r *assetResolver) classify(src string) (imgAction, string) {
+ s := strings.TrimSpace(src)
+ if s == "" || strings.HasPrefix(strings.ToLower(s), "data:") {
+ return imgKeep, ""
+ }
+ u, err := url.Parse(s)
+ if err != nil {
+ return imgKeep, ""
+ }
+ if u.Scheme != "" && u.Scheme != "http" && u.Scheme != "https" {
+ // file:, javascript:, and any other scheme are never fetched.
+ return imgKeep, ""
+ }
+ if u.Host != "" {
+ if u.Scheme == "" {
+ // Protocol-relative //host/path: only same-origin is eligible.
+ base, berr := url.Parse(r.client.cfg.baseURL)
+ if berr != nil || !strings.EqualFold(base.Host, u.Host) {
+ return imgKeep, ""
+ }
+ } else if !r.client.sameOrigin(u) {
+ // Absolute external http(s): keep, never send Confluence credentials.
+ return imgKeep, ""
+ }
+ }
+ abs, err := r.client.resolveEndpoint(s)
+ if err != nil {
+ return imgKeep, ""
+ }
+ return imgDownload, abs
+}
+
+// logImageFailure records a redacted warning: host and path only, never the
+// query (which may carry signed tokens), the Authorization header, or the body.
+func logImageFailure(ctx context.Context, abs string, err error) {
+ host, path := abs, ""
+ if u, perr := url.Parse(abs); perr == nil {
+ host, path = u.Host, u.Path
+ }
+ reason := "download_failed"
+ switch {
+ case errors.Is(err, errImageTooLarge):
+ reason = "oversize"
+ case errors.Is(err, errImageNonImageContent):
+ reason = "non_image_content_type"
+ case errors.Is(err, errImageRedirectOffOrigin):
+ reason = "cross_origin_redirect"
+ case errors.Is(err, errImagePageBudget):
+ reason = "page_budget"
+ }
+ var api *apiError
+ if errors.As(err, &api) {
+ reason = "status_" + strconv.Itoa(api.status)
+ }
+ logger.Warnf(ctx, "[Confluence] failed to resolve image host=%s path=%s reason=%s", host, path, reason)
+}
+
+// extractImageSrc returns the HTML-unescaped src value of an <img> tag.
+func extractImageSrc(tag string) (string, bool) {
+ m := srcAttrRe.FindStringSubmatch(tag)
+ if m == nil {
+ return "", false
+ }
+ value := strings.TrimSpace(m[2])
+ if len(value) >= 2 {
+ if q := value[0]; (q == '"' || q == '\'') && value[len(value)-1] == q {
+ value = value[1 : len(value)-1]
+ }
+ }
+ value = strings.TrimSpace(html.UnescapeString(value))
+ if value == "" {
+ return "", false
+ }
+ return value, true
+}
+
+// replaceImageSrc swaps only the src attribute value for a double-quoted data
+// URI, preserving the rest of the tag byte-for-byte.
+func replaceImageSrc(tag, dataURI string) string {
+ loc := srcAttrRe.FindStringSubmatchIndex(tag)
+ if loc == nil {
+ return tag
+ }
+ start, end := loc[4], loc[5]
+ return tag[:start] + `"` + dataURI + `"` + tag[end:]
+}
diff --git a/internal/datasource/connector/confluence/asset_resolver_test.go b/internal/datasource/connector/confluence/asset_resolver_test.go
new file mode 100644
index 000000000..98b124831
--- /dev/null
+++ b/internal/datasource/connector/confluence/asset_resolver_test.go
@@ -0,0 +1,612 @@
+package confluence
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "errors"
+ "io"
+ "net/http"
+ "strings"
+ "sync"
+ "testing"
+)
+
+const testBaseURL = "https://confluence.test/wiki"
+
+func newImageTestClient(rt roundTripper) *client {
+ return &client{
+ cfg: config{baseURL: testBaseURL, username: "reader", secret: "secret"},
+ http: &http.Client{Transport: rt},
+ }
+}
+
+func imageResp(status int, contentType string, body []byte) *http.Response {
+ header := make(http.Header)
+ if contentType != "" {
+ header.Set("Content-Type", contentType)
+ }
+ return &http.Response{
+ StatusCode: status,
+ Header: header,
+ Body: io.NopCloser(bytes.NewReader(body)),
+ }
+}
+
+func redirectResp(location string) *http.Response {
+ header := make(http.Header)
+ header.Set("Location", location)
+ return &http.Response{
+ StatusCode: http.StatusFound,
+ Header: header,
+ Body: io.NopCloser(bytes.NewReader(nil)),
+ }
+}
+
+// reqLog records outbound requests. Image downloads run concurrently, so every
+// access is mutex-guarded.
+type reqLog struct {
+ mu sync.Mutex
+ reqs []reqRecord
+}
+
+type reqRecord struct {
+ host string
+ path string
+ hasAuth bool
+}
+
+func newReqLog() *reqLog { return &reqLog{} }
+
+func (l *reqLog) add(req *http.Request) {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ l.reqs = append(l.reqs, reqRecord{
+ host: req.URL.Host,
+ path: req.URL.Path,
+ hasAuth: req.Header.Get("Authorization") != "",
+ })
+}
+
+func (l *reqLog) count() int {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ return len(l.reqs)
+}
+
+func (l *reqLog) all() []reqRecord {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ return append([]reqRecord(nil), l.reqs...)
+}
+
+func (l *reqLog) hasHost(host string) bool {
+ for _, r := range l.all() {
+ if r.host == host {
+ return true
+ }
+ }
+ return false
+}
+
+func (l *reqLog) hasPath(path string) bool {
+ for _, r := range l.all() {
+ if r.path == path {
+ return true
+ }
+ }
+ return false
+}
+
+func (l *reqLog) anyWithoutAuth() bool {
+ for _, r := range l.all() {
+ if !r.hasAuth {
+ return true
+ }
+ }
+ return false
+}
+
+var testPNG = []byte("private-png-bytes")
+
+func testDataURI(body []byte) string {
+ return "data:image/png;base64," + base64.StdEncoding.EncodeToString(body)
+}
+
+func TestAssetResolverKeepsPublicURL(t *testing.T) {
+ log := newReqLog()
+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {
+ log.add(req)
+ return imageResp(http.StatusOK, "image/png", testPNG), nil
+ })
+ in := `<p>hi</p><img src="https://cdn.example.com/a.png">`
+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
+ if out != in {
+ t.Fatalf("public URL was modified:\n%s", out)
+ }
+ if log.count() != 0 {
+ t.Fatalf("public image was downloaded: %v", log.all())
+ }
+}
+
+func TestAssetResolverKeepsBase64(t *testing.T) {
+ log := newReqLog()
+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {
+ log.add(req)
+ return imageResp(http.StatusOK, "image/png", testPNG), nil
+ })
+ in := `<img src="data:image/png;base64,QUJDRA==">`
+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
+ if out != in {
+ t.Fatalf("base64 image was modified:\n%s", out)
+ }
+ if log.count() != 0 {
+ t.Fatalf("base64 image triggered a download: %v", log.all())
+ }
+}
+
+func TestAssetResolverKeepsNonHTTPScheme(t *testing.T) {
+ log := newReqLog()
+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {
+ log.add(req)
+ return imageResp(http.StatusOK, "image/png", testPNG), nil
+ })
+ in := `<img src="file:///etc/passwd"><img src="javascript:void(0)">`
+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
+ if out != in {
+ t.Fatalf("non-http scheme was modified:\n%s", out)
+ }
+ if log.count() != 0 {
+ t.Fatalf("non-http scheme triggered a download: %v", log.all())
+ }
+}
+
+func TestAssetResolverInlinesRelativeConfluenceImage(t *testing.T) {
+ log := newReqLog()
+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {
+ log.add(req)
+ return imageResp(http.StatusOK, "image/png", testPNG), nil
+ })
+ in := `<img src="/wiki/download/attachments/123/a.png">`
+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
+
+ want := testDataURI(testPNG)
+ if !strings.Contains(out, `src="`+want+`"`) {
+ t.Fatalf("relative image not inlined as data URI:\n%s", out)
+ }
+ if strings.Contains(out, "/wiki/download/attachments") {
+ t.Fatalf("original private URL survived:\n%s", out)
+ }
+ reqs := log.all()
+ if len(reqs) != 1 {
+ t.Fatalf("expected 1 download, got %d: %v", len(reqs), reqs)
+ }
+ if reqs[0].host != "confluence.test" || !reqs[0].hasAuth {
+ t.Fatalf("download missing credentials or wrong host: %+v", reqs[0])
+ }
+}
+
+func TestAssetResolverInlinesSameOriginPrivateImage(t *testing.T) {
+ log := newReqLog()
+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {
+ log.add(req)
+ return imageResp(http.StatusOK, "image/png", testPNG), nil
+ })
+ in := `<img src="https://confluence.test/wiki/download/attachments/123/a.png" alt="x"/>`
+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
+
+ if !strings.Contains(out, `src="`+testDataURI(testPNG)+`"`) {
+ t.Fatalf("same-origin image not inlined:\n%s", out)
+ }
+ // The rest of the tag must be preserved.
+ if !strings.Contains(out, `alt="x"`) {
+ t.Fatalf("sibling attribute lost:\n%s", out)
+ }
+ if log.count() != 1 {
+ t.Fatalf("expected 1 download, got %d", log.count())
+ }
+}
+
+func TestAssetResolverDoesNotSendCredentialsCrossOrigin(t *testing.T) {
+ log := newReqLog()
+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {
+ log.add(req)
+ return imageResp(http.StatusOK, "image/png", testPNG), nil
+ })
+ in := `<img src="https://cdn.example.com/a.png">` +
+ `<img src="/wiki/download/attachments/1/a.png">`
+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
+
+ if log.hasHost("cdn.example.com") {
+ t.Fatalf("external host received a request: %v", log.all())
+ }
+ if !strings.Contains(out, "https://cdn.example.com/a.png") {
+ t.Fatalf("external URL was modified:\n%s", out)
+ }
+ if !strings.Contains(out, testDataURI(testPNG)) {
+ t.Fatalf("same-origin image was not inlined:\n%s", out)
+ }
+ if log.count() != 1 || log.anyWithoutAuth() {
+ t.Fatalf("expected exactly one authenticated same-origin download: %v", log.all())
+ }
+}
+
+func TestAssetResolverRejectsCrossOriginRedirect(t *testing.T) {
+ log := newReqLog()
+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {
+ log.add(req)
+ if req.URL.Host == "confluence.test" {
+ return redirectResp("https://evil.example.com/a.png"), nil
+ }
+ return imageResp(http.StatusOK, "image/png", testPNG), nil
+ })
+ in := `<img src="/wiki/download/attachments/1/a.png">`
+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
+
+ if out != in {
+ t.Fatalf("cross-origin redirect was followed and inlined:\n%s", out)
+ }
+ if log.hasHost("evil.example.com") {
+ t.Fatalf("credentials could have leaked to evil host: %v", log.all())
+ }
+ if log.count() != 1 {
+ t.Fatalf("expected only the original same-origin request, got %d: %v", log.count(), log.all())
+ }
+}
+
+func TestAssetResolverRejectsSameOriginRedirectOutsideContextPath(t *testing.T) {
+ log := newReqLog()
+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {
+ log.add(req)
+ // The base URL is https://confluence.test/wiki. A redirect to another app on
+ // the same host stays same-origin but escapes the /wiki context path, and Go
+ // keeps sending Basic Auth on a same-host hop. It must be refused before the
+ // credentialed request is ever sent, not merely discarded afterward.
+ if strings.HasPrefix(req.URL.Path, "/wiki/") {
+ return redirectResp("https://confluence.test/other-app/leak.png"), nil
+ }
+ return imageResp(http.StatusOK, "image/png", testPNG), nil
+ })
+ in := `<img src="/wiki/download/attachments/1/a.png">`
+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
+
+ if out != in {
+ t.Fatalf("same-host redirect outside the context path was followed and inlined:\n%s", out)
+ }
+ if log.hasPath("/other-app/leak.png") {
+ t.Fatalf("credentials followed a redirect outside the context path: %v", log.all())
+ }
+ if log.count() != 1 {
+ t.Fatalf("expected only the original same-origin request, got %d: %v", log.count(), log.all())
+ }
+}
+
+func TestAssetResolverRejectsOversizedImage(t *testing.T) {
+ log := newReqLog()
+ big := bytes.Repeat([]byte("a"), int(maxImageBytes)+1)
+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {
+ log.add(req)
+ return imageResp(http.StatusOK, "image/png", big), nil
+ })
+ in := `<img src="/wiki/download/attachments/1/big.png">`
+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
+
+ if out != in {
+ t.Fatalf("oversized image was inlined:\n%s", out)
+ }
+ if strings.Contains(out, "data:image") {
+ t.Fatalf("oversized image produced a data URI")
+ }
+}
+
+func TestAssetResolverRejectsNonImageResponse(t *testing.T) {
+ rt := roundTripper(func(_ *http.Request) (*http.Response, error) {
+ return imageResp(http.StatusOK, "text/html; charset=utf-8", []byte("<html>login required</html>")), nil
+ })
+ in := `<img src="/wiki/download/attachments/1/a.png">`
+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
+
+ if out != in {
+ t.Fatalf("login HTML was inlined as an image:\n%s", out)
+ }
+}
+
+func TestAssetResolverKeepsOriginalURLOnDownloadFailure(t *testing.T) {
+ for _, status := range []int{http.StatusForbidden, http.StatusNotFound, http.StatusInternalServerError} {
+ t.Run(http.StatusText(status), func(t *testing.T) {
+ rt := roundTripper(func(_ *http.Request) (*http.Response, error) {
+ return imageResp(status, "image/png", testPNG), nil
+ })
+ in := `<img src="/wiki/download/attachments/1/a.png">`
+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
+ if out != in {
+ t.Fatalf("status %d modified the document:\n%s", status, out)
+ }
+ })
+ }
+}
+
+func TestAssetResolverDeduplicatesSamePageImages(t *testing.T) {
+ log := newReqLog()
+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {
+ log.add(req)
+ return imageResp(http.StatusOK, "image/png", testPNG), nil
+ })
+ src := `<img src="/wiki/download/attachments/1/a.png">`
+ in := strings.Repeat(src, 5)
+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
+
+ if got := strings.Count(out, "data:image/png;base64,"); got != 5 {
+ t.Fatalf("expected 5 inlined copies, got %d:\n%s", got, out)
+ }
+ if log.count() != 1 {
+ t.Fatalf("duplicate src downloaded %d times, want 1", log.count())
+ }
+}
+
+func TestAssetResolverRespectsPerPageImageCap(t *testing.T) {
+ log := newReqLog()
+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {
+ log.add(req)
+ return imageResp(http.StatusOK, "image/png", testPNG), nil
+ })
+ var sb strings.Builder
+ total := maxPageImages + 5
+ for i := 0; i < total; i++ {
+ sb.WriteString(`<img src="/wiki/download/attachments/1/img`)
+ sb.WriteString(strings.Repeat("x", i+1)) // unique path per image
+ sb.WriteString(`.png">`)
+ }
+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), sb.String())
+
+ if got := strings.Count(out, "data:image/png;base64,"); got != maxPageImages {
+ t.Fatalf("inlined %d images, want cap %d", got, maxPageImages)
+ }
+ if log.count() != maxPageImages {
+ t.Fatalf("downloaded %d images, want cap %d", log.count(), maxPageImages)
+ }
+ if got := strings.Count(out, "/wiki/download/attachments"); got != 5 {
+ t.Fatalf("expected 5 images past the cap to keep original src, got %d", got)
+ }
+}
+
+func TestAssetResolverCapsOccurrencesNotUniqueDownloads(t *testing.T) {
+ log := newReqLog()
+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {
+ log.add(req)
+ return imageResp(http.StatusOK, "image/png", testPNG), nil
+ })
+ // Two unique URLs but 35 occurrences. Counting the cap by unique URL would
+ // download cold.png too and inline 30 + 5 data URIs, overflowing the downstream
+ // per-page limit that counts every occurrence. The budget must be occurrences.
+ hot := `<img src="/wiki/download/attachments/1/hot.png">`
+ cold := `<img src="/wiki/download/attachments/1/cold.png">`
+ in := strings.Repeat(hot, maxPageImages) + strings.Repeat(cold, 5)
+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
+
+ if got := strings.Count(out, "data:image/png;base64,"); got != maxPageImages {
+ t.Fatalf("inlined %d data URIs, want cap %d", got, maxPageImages)
+ }
+ if got := strings.Count(out, "/wiki/download/attachments/1/cold.png"); got != 5 {
+ t.Fatalf("expected the 5 images past the cap to keep original src, got %d", got)
+ }
+ // The repeated image downloads once; the image past the budget never downloads.
+ if log.count() != 1 {
+ t.Fatalf("downloaded %d images, want 1 (dedup + no over-budget fetch): %v", log.count(), log.all())
+ }
+ if log.hasPath("/wiki/download/attachments/1/cold.png") {
+ t.Fatalf("image past the occurrence cap was downloaded: %v", log.all())
+ }
+}
+
+// largeTestImage is just under maxImageBytes, so a handful of them overflow
+// maxPageInlineBytes long before the per-page occurrence cap.
+var largeTestImage = bytes.Repeat([]byte("a"), int(maxImageBytes)-1024)
+
+func TestAssetResolverStopsInliningPastPageByteBudget(t *testing.T) {
+ rt := roundTripper(func(_ *http.Request) (*http.Response, error) {
+ return imageResp(http.StatusOK, "image/png", largeTestImage), nil
+ })
+ want := maxPageInlineBytes / len(testDataURI(largeTestImage))
+ total := want + 3
+ src := `<img src="/wiki/download/attachments/1/big.png">`
+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), strings.Repeat(src, total))
+
+ if got := strings.Count(out, "data:image/png;base64,"); got != want {
+ t.Fatalf("inlined %d copies, want %d within the %d-byte page budget", got, want, maxPageInlineBytes)
+ }
+ if got := strings.Count(out, "/wiki/download/attachments/1/big.png"); got != total-want {
+ t.Fatalf("expected %d copies past the budget to keep original src, got %d", total-want, got)
+ }
+}
+
+func TestAssetResolverDownloadCacheRespectsPageByteBudget(t *testing.T) {
+ rt := roundTripper(func(_ *http.Request) (*http.Response, error) {
+ return imageResp(http.StatusOK, "image/png", largeTestImage), nil
+ })
+ want := maxPageInlineBytes / len(testDataURI(largeTestImage))
+ urls := make([]string, 0, want+3)
+ for i := 0; i < want+3; i++ {
+ urls = append(urls, testBaseURL+"/download/attachments/1/img"+strings.Repeat("x", i+1)+".png")
+ }
+ cache := newAssetResolver(newImageTestClient(rt)).downloadAll(context.Background(), urls)
+
+ if len(cache) != want {
+ t.Fatalf("cached %d images, want %d within the %d-byte page budget", len(cache), want, maxPageInlineBytes)
+ }
+}
+
+func TestAssetResolverSharesCapWithExistingDataURIImages(t *testing.T) {
+ log := newReqLog()
+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {
+ log.add(req)
+ return imageResp(http.StatusOK, "image/png", testPNG), nil
+ })
+ // Data URI images already in the page share the downstream 30-image budget with
+ // the ones we generate. With 10 existing, only 20 of the 25 private images may
+ // be inlined; the rest keep their original URL instead of overflowing the cap.
+ var sb strings.Builder
+ for i := 0; i < 10; i++ {
+ sb.WriteString(`<img src="data:image/gif;base64,EXISTING">`)
+ }
+ for i := 0; i < 25; i++ {
+ sb.WriteString(`<img src="/wiki/download/attachments/1/img`)
+ sb.WriteString(strings.Repeat("x", i+1)) // unique path per image
+ sb.WriteString(`.png">`)
+ }
+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), sb.String())
+
+ // The 10 pre-existing data URIs are preserved untouched.
+ if got := strings.Count(out, "data:image/gif;base64,EXISTING"); got != 10 {
+ t.Fatalf("preserved %d existing data URIs, want 10", got)
+ }
+ // Exactly 20 private images inlined: 30 shared budget - 10 already present.
+ if got := strings.Count(out, testDataURI(testPNG)); got != 20 {
+ t.Fatalf("inlined %d private images, want 20", got)
+ }
+ // The 5 private images past the shared budget keep their original URL.
+ if got := strings.Count(out, "/wiki/download/attachments"); got != 5 {
+ t.Fatalf("expected 5 private images past the cap to keep original src, got %d", got)
+ }
+ // Only the 20 inlinable private images are downloaded.
+ if log.count() != 20 {
+ t.Fatalf("downloaded %d images, want 20: %v", log.count(), log.all())
+ }
+}
+
+func TestAssetResolverIgnoresDataSrcAttribute(t *testing.T) {
+ log := newReqLog()
+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {
+ log.add(req)
+ return imageResp(http.StatusOK, "image/png", testPNG), nil
+ })
+ // data-src must not be mistaken for src; there is no real src here.
+ in := `<img data-src="/wiki/download/attachments/1/a.png">`
+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
+ if out != in {
+ t.Fatalf("data-src was treated as src:\n%s", out)
+ }
+ if log.count() != 0 {
+ t.Fatalf("data-src triggered a download: %v", log.all())
+ }
+}
+
+func TestAssetResolverToleratesGreaterThanInAttribute(t *testing.T) {
+ log := newReqLog()
+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {
+ log.add(req)
+ return imageResp(http.StatusOK, "image/png", testPNG), nil
+ })
+ in := `<img alt="a > b" src="/wiki/download/attachments/1/a.png">`
+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
+ if !strings.Contains(out, testDataURI(testPNG)) {
+ t.Fatalf("image with '>' in alt was not inlined:\n%s", out)
+ }
+ if !strings.Contains(out, `alt="a > b"`) {
+ t.Fatalf("alt attribute was corrupted:\n%s", out)
+ }
+}
+
+func TestAssetResolverUnescapesAmpersandInQuery(t *testing.T) {
+ var gotPath string
+ var gotQuery string
+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {
+ gotPath = req.URL.Path
+ gotQuery = req.URL.RawQuery
+ return imageResp(http.StatusOK, "image/png", testPNG), nil
+ })
+ in := `<img src="/wiki/download/attachments/1/a.png?api=v2&amp;v=3">`
+ out := newAssetResolver(newImageTestClient(rt)).Resolve(context.Background(), in)
+ if !strings.Contains(out, "data:image/png;base64,") {
+ t.Fatalf("image with &amp; query was not inlined:\n%s", out)
+ }
+ if gotPath != "/wiki/download/attachments/1/a.png" || gotQuery != "api=v2&v=3" {
+ t.Fatalf("query not unescaped: path=%q query=%q", gotPath, gotQuery)
+ }
+}
+
+func TestDownloadPrivateImageLimitsBodySize(t *testing.T) {
+ big := bytes.Repeat([]byte("a"), int(maxImageBytes)+1)
+ rt := roundTripper(func(_ *http.Request) (*http.Response, error) {
+ return imageResp(http.StatusOK, "image/png", big), nil
+ })
+ c := newImageTestClient(rt)
+ _, _, err := c.downloadImage(context.Background(), "https://confluence.test/wiki/download/attachments/1/big.png")
+ if !errors.Is(err, errImageTooLarge) {
+ t.Fatalf("downloadImage() error = %v, want errImageTooLarge", err)
+ }
+}
+
+func TestDownloadPrivateImageValidatesContentType(t *testing.T) {
+ rt := roundTripper(func(_ *http.Request) (*http.Response, error) {
+ return imageResp(http.StatusOK, "application/json", []byte(`{"error":"unauthorized"}`)), nil
+ })
+ c := newImageTestClient(rt)
+ _, _, err := c.downloadImage(context.Background(), "https://confluence.test/wiki/download/attachments/1/a.png")
+ if !errors.Is(err, errImageNonImageContent) {
+ t.Fatalf("downloadImage() error = %v, want errImageNonImageContent", err)
+ }
+}
+
+func TestDownloadPrivateImageSniffsWhenContentTypeMissing(t *testing.T) {
+ png := append([]byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}, bytes.Repeat([]byte{0}, 64)...)
+ rt := roundTripper(func(_ *http.Request) (*http.Response, error) {
+ return imageResp(http.StatusOK, "", png), nil
+ })
+ c := newImageTestClient(rt)
+ body, mime, err := c.downloadImage(
+ context.Background(),
+ "https://confluence.test/wiki/download/attachments/1/a.png",
+ )
+ if err != nil {
+ t.Fatalf("downloadImage() error = %v", err)
+ }
+ if mime != "image/png" {
+ t.Fatalf("sniffed mime = %q, want image/png", mime)
+ }
+ if !bytes.Equal(body, png) {
+ t.Fatalf("body mismatch: %d bytes", len(body))
+ }
+}
+
+func TestDownloadPrivateImageRejectsCrossOriginRedirect(t *testing.T) {
+ rt := roundTripper(func(req *http.Request) (*http.Response, error) {
+ if req.URL.Host == "confluence.test" {
+ return redirectResp("https://evil.example.com/a.png"), nil
+ }
+ return imageResp(http.StatusOK, "image/png", testPNG), nil
+ })
+ c := newImageTestClient(rt)
+ _, _, err := c.downloadImage(context.Background(), "https://confluence.test/wiki/download/attachments/1/a.png")
+ if !errors.Is(err, errImageRedirectOffOrigin) {
+ t.Fatalf("downloadImage() error = %v, want errImageRedirectOffOrigin", err)
+ }
+}
+
+// TestFetchStreamInlinesPrivateImageAsDataURI is the connector integration test:
+// a body.view containing a same-origin private image flows through FetchStream
+// and the emitted FetchedItem.Content carries an inline data URI (which the
+// downstream docparser pipeline later persists and rewrites to resource://).
+func TestFetchStreamInlinesPrivateImageAsDataURI(t *testing.T) {
+ api := &streamAPI{
+ pages: []streamPage{{id: "p1", title: "Page", version: 1}},
+ bodyHTML: map[string]string{"p1": `<p>hi</p><img src="/wiki/download/attachments/p1/a.png">`},
+ }
+ h := &captureHandler{}
+ if _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), nil, h); err != nil {
+ t.Fatal(err)
+ }
+ if len(h.items) != 1 {
+ t.Fatalf("emitted %d items, want 1", len(h.items))
+ }
+ content := string(h.items[0].Content)
+ if !strings.Contains(content, "data:image/png;base64,") {
+ t.Fatalf("private image was not inlined into Content:\n%s", content)
+ }
+ if strings.Contains(content, "/wiki/download/attachments") {
+ t.Fatalf("original private Confluence URL leaked into Content:\n%s", content)
+ }
+ if api.imageCalls != 1 {
+ t.Fatalf("image download calls = %d, want 1", api.imageCalls)
+ }
+}
diff --git a/internal/datasource/connector/confluence/client.go b/internal/datasource/connector/confluence/client.go
index ec156e382..66fa29ab4 100644
--- a/internal/datasource/connector/confluence/client.go
+++ b/internal/datasource/connector/confluence/client.go
@@ -17,19 +17,29 @@ import (
)
const (
- maxJSONResponseBytes int64 = 20 << 20
- requestAttempts = 4
- maxErrorResponseRunes = 1000
- maxRetryDelay = 60 * time.Second
- maxPaginationHops = 10000
+ maxJSONResponseBytes int64 = 20 << 20
+ maxImageBytes int64 = 10 << 20
+ maxImageRedirects = 10
+ requestAttempts = 4
+ maxErrorResponseRunes = 1000
+ maxRetryDelay = 60 * time.Second
+ maxPaginationHops = 10000
+ maxTraversalNodes = 100000
+ maxTransparentTraversalNodes = 10000
+ maxTransparentTraversalDepth = 32
// maxEmptyServerPages bounds how far a Server/Data Center page listing
- // follows _links.next through empty pages. Some builds keep returning
- // next past the end of a space (#3596); permission filtering can also
- // empty one page mid-listing, so a single empty page is not the end.
+ // follows _links.next through empty pages.
maxEmptyServerPages = 3
- // maxDeletionProbes caps how many unlisted pages one sync looks up
- // individually after a cut-short listing; the rest wait for the next run.
- maxDeletionProbes = 200
+ maxDeletionProbes = 200
+)
+
+var (
+ errImageTooLarge = errors.New("confluence image exceeds size limit")
+ errImageNonImageContent = errors.New("confluence image response is not an image")
+ // errImageRedirectOffOrigin fires whenever a redirect target leaves the base
+ // URL boundary resolveEndpoint enforces — a different origin, or the same host
+ // but a path outside the configured context path (e.g. /wiki → /other-app).
+ errImageRedirectOffOrigin = errors.New("confluence image redirect left the configured base URL")
)
type apiError struct {
@@ -232,6 +242,121 @@ func (c *client) resourceURL(link string) string {
return resolved
}
+// sameOrigin reports whether u shares scheme and host with the configured base
+// URL. Only same-origin URLs may carry Confluence credentials.
+func (c *client) sameOrigin(u *url.URL) bool {
+ if u == nil {
+ return false
+ }
+ base, err := url.Parse(c.cfg.baseURL)
+ if err != nil {
+ return false
+ }
+ return strings.EqualFold(base.Scheme, u.Scheme) && strings.EqualFold(base.Host, u.Host)
+}
+
+// withinImageScope reports whether u stays inside the boundary resolveEndpoint
+// enforces for the initial image URL: the same origin AND a path inside the
+// configured context path. Redirect targets are validated with this same rule,
+// not merely sameOrigin, because Go forwards Basic Auth on a same-host redirect —
+// so a hop like /wiki/download → /other-app/image must fail closed too.
+func (c *client) withinImageScope(u *url.URL) bool {
+ if u == nil {
+ return false
+ }
+ _, err := c.resolveEndpoint(u.String())
+ return err == nil
+}
+
+// downloadImage fetches a same-origin private image using the connector's
+// credentials. It streams with a hard size cap, fails closed if a redirect left
+// the Confluence origin, and validates the response really is an image. It is a
+// best-effort single attempt: a failed image degrades to keeping its original
+// URL rather than retrying or failing the page.
+func (c *client) downloadImage(ctx context.Context, absURL string) ([]byte, string, error) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, absURL, nil)
+ if err != nil {
+ return nil, "", err
+ }
+ req.SetBasicAuth(c.cfg.username, c.cfg.secret)
+ req.Header.Set("Accept", "image/*,*/*;q=0.8")
+ resp, err := c.imageHTTPClient().Do(req)
+ if err != nil {
+ return nil, "", err
+ }
+ defer func() { _ = resp.Body.Close() }()
+ // Backstop for the redirect policy: imageHTTPClient's CheckRedirect already
+ // refuses any hop that leaves the base before it is followed, so this only
+ // re-verifies the final URL when the transport populated resp.Request. A custom
+ // RoundTripper may leave it nil, in which case there is no followed redirect to
+ // re-check and the download is allowed to proceed.
+ if resp.Request != nil && !c.withinImageScope(resp.Request.URL) {
+ return nil, "", errImageRedirectOffOrigin
+ }
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ return nil, "", &apiError{endpoint: "image", status: resp.StatusCode}
+ }
+ body, err := io.ReadAll(io.LimitReader(resp.Body, maxImageBytes+1))
+ if err != nil {
+ return nil, "", err
+ }
+ if int64(len(body)) > maxImageBytes {
+ return nil, "", errImageTooLarge
+ }
+ mime, ok := imageContentType(resp.Header.Get("Content-Type"), body)
+ if !ok {
+ return nil, "", errImageNonImageContent
+ }
+ return body, mime, nil
+}
+
+// imageHTTPClient reuses the connector's SSRF-safe transport and timeout but
+// pins redirects to the configured Confluence base, so a private-image fetch can
+// never be bounced to a third party or to another app on the same host. The
+// redirect check fails closed: any hop that resolveEndpoint would reject — a
+// foreign origin, or a same-host path outside the context path (which Go would
+// still send Basic Auth to) — is refused outright rather than merely stripped of
+// credentials.
+func (c *client) imageHTTPClient() *http.Client {
+ return &http.Client{
+ Transport: c.http.Transport,
+ Timeout: c.http.Timeout,
+ CheckRedirect: func(req *http.Request, via []*http.Request) error {
+ if len(via) >= maxImageRedirects {
+ return errors.New("confluence image stopped after too many redirects")
+ }
+ if !c.withinImageScope(req.URL) {
+ return errImageRedirectOffOrigin
+ }
+ return nil
+ },
+ }
+}
+
+// imageContentType normalizes a response Content-Type to an image MIME type. An
+// empty or generic binary type falls back to content sniffing; any explicit
+// non-image type (for example the text/html login page Confluence returns when a
+// token expired) is rejected so it is never stored as an image.
+func imageContentType(header string, body []byte) (string, bool) {
+ ct := strings.ToLower(strings.TrimSpace(header))
+ if i := strings.Index(ct, ";"); i >= 0 {
+ ct = strings.TrimSpace(ct[:i])
+ }
+ switch {
+ case strings.HasPrefix(ct, "image/"):
+ return ct, true
+ case ct != "" && ct != "application/octet-stream":
+ return "", false
+ }
+ if sniff := strings.ToLower(http.DetectContentType(body)); strings.HasPrefix(sniff, "image/") {
+ if i := strings.Index(sniff, ";"); i >= 0 {
+ sniff = strings.TrimSpace(sniff[:i])
+ }
+ return sniff, true
+ }
+ return "", false
+}
+
func (c *client) ping(ctx context.Context) error {
if c.cfg.cloud() {
return c.get(ctx, "/api/v2/spaces?limit=1", nil)
@@ -308,10 +433,6 @@ func withCloudPageQuery(next string) string {
return parsed.String()
}
-// pages lists every current page in a space. complete is false when a Server
-// listing was cut short after maxEmptyServerPages empty pages that still
-// linked onward; the caller must not treat pages missing from such a listing
-// as deleted.
func (c *client) pages(ctx context.Context, s space) (pages []page, complete bool, err error) {
if c.cfg.cloud() {
var all []page
@@ -371,10 +492,6 @@ func (c *client) pages(ctx context.Context, s space) (pages []page, complete boo
return all, complete, err
}
-// pageInSpace reports whether a Server page is still a current page of the
-// space. Confluence answers 404 for a page that was deleted or moved to the
-// trash; a page that is no longer current or now belongs to another space is
-// gone from this one too, matching what a complete listing would omit.
func (c *client) pageInSpace(ctx context.Context, id, spaceKey string) (bool, error) {
var result struct {
Status string `json:"status"`
@@ -406,3 +523,349 @@ func (c *client) body(ctx context.Context, id string) (pageBody, error) {
err := c.get(ctx, "/rest/api/content/"+url.PathEscape(id)+"?expand=body.view,version,space", &result)
return result, err
}
+
+// pageDetail obtains identity, ownership, and version without requiring a
+// rendered body. Scope enumeration must remain complete when a body is
+// temporarily unavailable, otherwise a transient content error would be
+// indistinguishable from a missing page during deletion reconciliation.
+func (c *client) pageDetail(ctx context.Context, id string) (page, error) {
+ var result page
+ if c.cfg.cloud() {
+ err := c.get(ctx, "/api/v2/pages/"+url.PathEscape(id), &result)
+ return result, err
+ }
+ err := c.get(ctx, "/rest/api/content/"+url.PathEscape(id)+"?expand=version,space", &result)
+ return result, err
+}
+
+// validatePageOwnership performs the constant-cost check required before a
+// picker expansion. Ancestors are deliberately not fetched here: they are only
+// needed by ResolveResourceAncestors and buildSyncPlan.
+func (c *client) validatePageOwnership(ctx context.Context, s space, pageID string) error {
+ full, err := c.pageDetail(ctx, pageID)
+ if err != nil {
+ return err
+ }
+ if c.cfg.cloud() {
+ if full.SpaceID != s.ID {
+ return fmt.Errorf("page %s belongs to space %s, not %s", pageID, full.SpaceID, s.ID)
+ }
+ return nil
+ }
+ if full.Space.Key == "" || full.Space.Key != s.Key {
+ return fmt.Errorf("page %s does not belong to space %s", pageID, s.ID)
+ }
+ return nil
+}
+
+func serverChildPagesEndpoint(pageID string) string {
+ query := url.Values{"expand": []string{serverPageExpand}, "limit": []string{"100"}}
+ return "/rest/api/content/" + url.PathEscape(pageID) + "/child/page?" + query.Encode()
+}
+
+func serverTopLevelPagesEndpoint(spaceKey string) string {
+ query := url.Values{"depth": []string{"root"}, "expand": []string{serverPageExpand}, "limit": []string{"100"}}
+ return "/rest/api/space/" + url.PathEscape(spaceKey) + "/content/page?" + query.Encode()
+}
+
+func (c *client) listServerPages(ctx context.Context, endpoint string) ([]page, error) {
+ start, _ := url.Parse(endpoint)
+ startQuery := start.Query()
+ var all []page
+ err := c.paginate(ctx, endpoint, func(pageURL string) (string, error) {
+ var result pageList
+ if err := c.get(ctx, pageURL, &result); err != nil {
+ return "", err
+ }
+ all = append(all, result.Results...)
+ next := result.Links.Next
+ if next != "" {
+ next = withServerListQuery(next, startQuery)
+ }
+ return next, nil
+ })
+ return all, err
+}
+
+// Server pagination links commonly retain only start/limit. Restore the query
+// that defines the listing scope (notably depth=root) as well as expand.
+func withServerListQuery(next string, required url.Values) string {
+ parsed, err := url.Parse(next)
+ if err != nil {
+ return next
+ }
+ query := parsed.Query()
+ for key, values := range required {
+ if query.Get(key) == "" && len(values) > 0 {
+ query[key] = values
+ }
+ }
+ parsed.RawQuery = query.Encode()
+ return parsed.String()
+}
+
+func (c *client) listCloudHierarchy(ctx context.Context, endpoint string) ([]page, error) {
+ var all []page
+ err := c.paginate(ctx, endpoint, func(pageURL string) (string, error) {
+ var result cloudHierarchyList
+ if err := c.get(ctx, pageURL, &result); err != nil {
+ return "", err
+ }
+ for _, item := range result.Results {
+ all = append(
+ all,
+ page{
+ ID: item.ID,
+ Title: item.Title,
+ Status: item.Status,
+ Kind: item.Type,
+ SpaceID: item.SpaceID,
+ Links: item.Links,
+ },
+ )
+ }
+ return result.Links.Next, nil
+ })
+ return all, err
+}
+
+// topLevelPages retrieves one level only. The Server/DC depth=root endpoint
+// is paginated by the server; it deliberately never falls back to scanning all
+// pages because that would defeat lazy selection for large spaces.
+//
+// Cloud applies the same lazy contract: depth=0 returns only root-level pages,
+// and Cloud exposes no endpoint to enumerate a space's top-level folders
+// (CONFCLOUD-84275), so pages nested directly under those containers stay
+// unselectable in the picker for now. Recovering them would require a
+// whole-space scan per expand, which is exactly what lazy loading forbids;
+// selecting the whole space still syncs them. The connector flags this
+// limitation through resource metadata so the picker can surface it.
+func (c *client) topLevelPages(ctx context.Context, s space) ([]page, error) {
+ var pages []page
+ var err error
+ if c.cfg.cloud() {
+ pages, err = c.listCloudHierarchy(
+ ctx,
+ "/api/v2/spaces/"+url.PathEscape(s.ID)+"/pages?status=current&depth=0&limit=250",
+ )
+ if err == nil {
+ visible := pages[:0]
+ for _, page := range pages {
+ if page.Kind == "" || page.Kind == "page" {
+ visible = append(visible, page)
+ }
+ }
+ pages = visible
+ }
+ } else {
+ pages, err = c.listServerPages(ctx, serverTopLevelPagesEndpoint(s.Key))
+ }
+ if err == nil {
+ return pages, nil
+ }
+ // The depth=root navigation hint is Server/DC-specific; a Cloud 400/404 has
+ // different causes and must surface the raw API error for diagnosis.
+ if !c.cfg.cloud() {
+ var api *apiError
+ if errors.As(err, &api) && (api.status == http.StatusBadRequest || api.status == http.StatusNotFound) {
+ return nil, fmt.Errorf(
+ "confluence server does not support complete top-level page navigation for this space; "+
+ "select the Space to sync all pages: %w",
+ err,
+ )
+ }
+ }
+ return nil, err
+}
+
+func (c *client) directChildPages(ctx context.Context, _ space, pageID string) ([]page, error) {
+ if c.cfg.cloud() {
+ return c.visibleCloudPageChildren(ctx, pageID)
+ }
+ return c.listServerPages(ctx, serverChildPagesEndpoint(pageID))
+}
+
+func cloudDirectChildrenEndpoint(kind, id string) (string, error) {
+ plural := map[string]string{
+ "page": "pages", "folder": "folders", "database": "databases", "whiteboard": "whiteboards", "embed": "embeds",
+ }[kind]
+ if plural == "" {
+ return "", fmt.Errorf("unsupported Confluence Cloud hierarchy type %q", kind)
+ }
+ return "/api/v2/" + plural + "/" + url.PathEscape(id) + "/direct-children?limit=250", nil
+}
+
+// visibleCloudPageChildren projects Cloud's mixed-content hierarchy onto a
+// Page-only picker. Containers remain connector-internal and are traversed
+// until the next visible Page is found.
+func (c *client) visibleCloudPageChildren(ctx context.Context, pageID string) ([]page, error) {
+ return c.walkCloudPageHierarchy(ctx, pageID, false)
+}
+
+// walkCloudPageHierarchy traverses Cloud's mixed content tree through
+// direct-children endpoints. Picker expansion stops at visible pages, while a
+// sync walk continues through them to return the complete page subtree.
+func (c *client) walkCloudPageHierarchy(ctx context.Context, pageID string, descendPages bool) ([]page, error) {
+ type queuedNode struct {
+ id, kind string
+ depth int
+ }
+ queue := []queuedNode{{id: pageID, kind: "page"}}
+ visited := map[string]struct{}{"page:" + pageID: {}}
+ pages := make([]page, 0)
+ nodeLimit := maxTransparentTraversalNodes
+ if descendPages {
+ nodeLimit = maxTraversalNodes
+ }
+ for len(queue) > 0 {
+ current := queue[0]
+ queue = queue[1:]
+ endpoint, err := cloudDirectChildrenEndpoint(current.kind, current.id)
+ if err != nil {
+ return nil, err
+ }
+ children, err := c.listCloudHierarchy(ctx, endpoint)
+ if err != nil {
+ return nil, err
+ }
+ for _, child := range children {
+ kind := child.Kind
+ if kind == "" {
+ kind = "page"
+ }
+ key := kind + ":" + child.ID
+ if child.ID == "" || (kind == "page" && child.ID == pageID) {
+ continue
+ }
+ if _, exists := visited[key]; exists {
+ continue
+ }
+ if len(visited) >= nodeLimit {
+ return nil, fmt.Errorf(
+ "confluence Cloud hierarchy traversal exceeds %d nodes",
+ nodeLimit,
+ )
+ }
+ visited[key] = struct{}{}
+ if kind == "page" {
+ pages = append(pages, child)
+ if !descendPages {
+ continue
+ }
+ }
+ depth := 0
+ if kind != "page" {
+ depth = current.depth + 1
+ if current.kind == "page" {
+ depth = 1
+ }
+ }
+ if depth > maxTransparentTraversalDepth {
+ return nil, fmt.Errorf(
+ "confluence Cloud transparent traversal exceeds depth %d",
+ maxTransparentTraversalDepth,
+ )
+ }
+ if _, err := cloudDirectChildrenEndpoint(kind, child.ID); err != nil {
+ return nil, err
+ }
+ queue = append(queue, queuedNode{id: child.ID, kind: kind, depth: depth})
+ }
+ }
+ return pages, nil
+}
+
+// pageAncestors validates that pageID belongs to s and returns its path from
+// the space root. A selected page is fetched even if it is top-level, so a
+// forged page:{space}:{page} ID cannot cross a space boundary.
+func (c *client) pageAncestors(ctx context.Context, s space, pageID string) ([]page, error) {
+ if c.cfg.cloud() {
+ full, err := c.pageDetail(ctx, pageID)
+ if err != nil {
+ return nil, err
+ }
+ if full.SpaceID != s.ID {
+ return nil, fmt.Errorf("page %s belongs to space %s, not %s", pageID, full.SpaceID, s.ID)
+ }
+ ancestors, err := c.listCloudHierarchy(ctx, "/api/v2/pages/"+url.PathEscape(pageID)+"/ancestors?limit=250")
+ if err != nil {
+ return nil, err
+ }
+ return ancestors, nil
+ }
+ var full page
+ if err := c.get(ctx, "/rest/api/content/"+url.PathEscape(pageID)+"?expand=ancestors,space", &full); err != nil {
+ return nil, err
+ }
+ if full.Space.Key == "" || full.Space.Key != s.Key {
+ return nil, fmt.Errorf("page %s does not belong to space %s", pageID, s.ID)
+ }
+ return full.Ancestors, nil
+}
+
+// pageSubtree uses direct-child traversal for Server/DC because that endpoint
+// is stable across supported versions. Cloud uses the same complete
+// mixed-content hierarchy walk as the picker, but continues through visible
+// pages to include every page in the selected subtree.
+func (c *client) pageSubtree(ctx context.Context, s space, pageID string) ([]page, error) {
+ full, err := c.pageDetail(ctx, pageID)
+ if err != nil {
+ return nil, err
+ }
+ if c.cfg.cloud() {
+ if full.SpaceID != s.ID {
+ return nil, fmt.Errorf("page %s does not belong to space %s", pageID, s.ID)
+ }
+ } else if full.Space.Key == "" || full.Space.Key != s.Key {
+ return nil, fmt.Errorf("page %s does not belong to space %s", pageID, s.ID)
+ }
+ root := full
+ if c.cfg.cloud() {
+ descendants, err := c.walkCloudPageHierarchy(ctx, pageID, true)
+ if err != nil {
+ return nil, err
+ }
+ out := []page{root}
+ for _, descendant := range descendants {
+ // Direct-child responses are intentionally compact. Hydrate each page
+ // before version comparison; an unknown version must never be treated
+ // as an unchanged stable token.
+ detail, err := c.pageDetail(ctx, descendant.ID)
+ if err != nil {
+ return nil, err
+ }
+ if detail.SpaceID != s.ID {
+ return nil, fmt.Errorf("page %s does not belong to space %s", descendant.ID, s.ID)
+ }
+ out = append(out, detail)
+ }
+ return out, nil
+ }
+ queue, out := []page{root}, []page{root}
+ seen := map[string]bool{root.ID: true}
+ for len(queue) > 0 {
+ current := queue[0]
+ queue = queue[1:]
+ children, err := c.directChildPages(ctx, s, current.ID)
+ if err != nil {
+ return nil, err
+ }
+ for _, child := range children {
+ if child.ID == "" || seen[child.ID] {
+ continue
+ }
+ if len(seen) >= maxTraversalNodes {
+ return nil, fmt.Errorf("confluence page subtree exceeds %d nodes", maxTraversalNodes)
+ }
+ seen[child.ID] = true
+ if child.SpaceID != "" && child.SpaceID != s.ID {
+ return nil, fmt.Errorf("page %s does not belong to space %s", child.ID, s.ID)
+ }
+ if !c.cfg.cloud() && child.Space.Key != "" && child.Space.Key != s.Key {
+ return nil, fmt.Errorf("page %s does not belong to space %s", child.ID, s.ID)
+ }
+ queue, out = append(queue, child), append(out, child)
+ }
+ }
+ return out, nil
+}
diff --git a/internal/datasource/connector/confluence/client_pages_test.go b/internal/datasource/connector/confluence/client_pages_test.go
index 1a7df281c..1113a35d6 100644
--- a/internal/datasource/connector/confluence/client_pages_test.go
+++ b/internal/datasource/connector/confluence/client_pages_test.go
@@ -6,6 +6,7 @@ import (
"fmt"
"net/http"
"net/http/httptest"
+ "strings"
"testing"
)
@@ -145,6 +146,70 @@ func TestServerSpacePageListPrefersFlatResults(t *testing.T) {
}
}
+func TestTopLevelPagesServerRestoresScopeOnNext(t *testing.T) {
+ var depths, expands []string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/rest/api/space/ENG/content/page" {
+ http.NotFound(w, r)
+ return
+ }
+ depths = append(depths, r.URL.Query().Get("depth"))
+ expands = append(expands, r.URL.Query().Get("expand"))
+ w.Header().Set("Content-Type", "application/json")
+ if r.URL.Query().Get("start") == "100" {
+ _, _ = w.Write([]byte(`{
+ "results": [{"id": "2", "title": "Second", "version": {"number": 2}}]
+ }`))
+ return
+ }
+ _, _ = w.Write([]byte(`{
+ "_links": {"next": "/rest/api/space/ENG/content/page?limit=100&start=100"},
+ "results": [{"id": "1", "title": "Root", "version": {"number": 1}}]
+ }`))
+ }))
+ t.Cleanup(server.Close)
+
+ c := &client{cfg: config{baseURL: server.URL}, http: server.Client()}
+ pages, err := c.topLevelPages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(pages) != 2 || pages[0].ID != "1" || pages[1].ID != "2" {
+ t.Fatalf("pages = %#v", pages)
+ }
+ // The server's _links.next drops depth and expand; the connector must
+ // rebuild the listing scope or later pages would look versionless.
+ if len(depths) != 2 || depths[0] != "root" || depths[1] != "root" {
+ t.Fatalf("depth query = %#v", depths)
+ }
+ if len(expands) != 2 || expands[0] != serverPageExpand || expands[1] != serverPageExpand {
+ t.Fatalf("expand query = %#v", expands)
+ }
+}
+
+func TestTopLevelPagesCloudSurfacesRawAPIError(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusNotFound)
+ _, _ = w.Write([]byte(`{"message":"space not found"}`))
+ }))
+ t.Cleanup(server.Close)
+
+ cloud := &client{cfg: config{baseURL: server.URL + "/wiki", edition: editionCloud}, http: server.Client()}
+ _, err := cloud.topLevelPages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})
+ if err == nil {
+ t.Fatal("cloud topLevelPages unexpectedly succeeded")
+ }
+ if strings.Contains(err.Error(), "does not support") {
+ t.Fatalf("cloud 404 reused the Server/DC navigation hint: %v", err)
+ }
+
+ srv := &client{cfg: config{baseURL: server.URL + "/wiki"}, http: server.Client()}
+ _, err = srv.topLevelPages(context.Background(), space{ID: "1", Key: "ENG", Name: "Engineering"})
+ if err == nil || !strings.Contains(err.Error(), "confluence server does not support") {
+ t.Fatalf("server error = %v; want the Server/DC navigation hint", err)
+ }
+}
+
func TestPagesCloudFollowsNextAndKeepsDepth(t *testing.T) {
var depths []string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
diff --git a/internal/datasource/connector/confluence/connector.go b/internal/datasource/connector/confluence/connector.go
index fe2a6f5ca..083d5bf13 100644
--- a/internal/datasource/connector/confluence/connector.go
+++ b/internal/datasource/connector/confluence/connector.go
@@ -3,10 +3,14 @@ package confluence
import (
"context"
+ "crypto/sha256"
"errors"
"fmt"
"net/http"
+ "sort"
"strings"
+ "sync"
+ "time"
htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"
"github.com/Tencent/WeKnora/internal/datasource"
@@ -21,12 +25,64 @@ var (
// Connector implements datasource.StreamingConnector for Confluence.
type Connector struct {
- newClient func(config) (*client, error)
+ newClient func(config) (*client, error)
+ spaceCache spaceCache
+}
+
+const pickerSpaceCacheTTL = time.Minute
+
+type spaceCache struct {
+ mu sync.Mutex
+ entries map[string]cachedSpaces
+}
+
+type cachedSpaces struct {
+ values []space
+ expires time.Time
}
// NewConnector creates a Confluence connector.
func NewConnector() *Connector { return &Connector{newClient: newClient} }
+func (c *Connector) pickerSpaces(ctx context.Context, client *client, cfg config) ([]space, error) {
+ key := pickerSpaceCacheKey(cfg)
+ now := time.Now()
+ c.spaceCache.mu.Lock()
+ if cached, ok := c.spaceCache.entries[key]; ok && now.Before(cached.expires) {
+ values := append([]space(nil), cached.values...)
+ c.spaceCache.mu.Unlock()
+ return values, nil
+ }
+ c.spaceCache.mu.Unlock()
+ spaces, err := client.spaces(ctx)
+ if err != nil {
+ return nil, err
+ }
+ c.spaceCache.mu.Lock()
+ if c.spaceCache.entries == nil {
+ c.spaceCache.entries = make(map[string]cachedSpaces)
+ }
+ for key, cached := range c.spaceCache.entries {
+ if !now.Before(cached.expires) {
+ delete(c.spaceCache.entries, key)
+ }
+ }
+ c.spaceCache.entries[key] = cachedSpaces{
+ values: append([]space(nil), spaces...),
+ expires: now.Add(pickerSpaceCacheTTL),
+ }
+ c.spaceCache.mu.Unlock()
+ return spaces, nil
+}
+
+// pickerSpaceCacheKey partitions cached space visibility by every credential
+// component that can affect permissions. The secret is fingerprinted so it
+// never becomes a map key retained in process memory or appears in diagnostics.
+func pickerSpaceCacheKey(cfg config) string {
+ fingerprint := sha256.Sum256([]byte(cfg.secret))
+ return cfg.edition + "\x00" + cfg.baseURL + "\x00" + cfg.username + "\x00" + fmt.Sprintf("%x", fingerprint)
+}
+
// Type returns the connector type identifier.
func (*Connector) Type() string { return types.ConnectorTypeConfluence }
@@ -52,38 +108,142 @@ func (c *Connector) Validate(ctx context.Context, ds *types.DataSourceConfig) er
return client.ping(ctx)
}
-// ListResources returns the Confluence spaces the credentials can see.
+// ListResources exposes a lazy Space → Page tree. It never scans a whole
+// space merely to render the picker.
func (c *Connector) ListResources(
ctx context.Context, ds *types.DataSourceConfig, parentID string,
) ([]types.Resource, error) {
- if parentID != "" {
- return nil, nil
+ client, cfg, err := c.configured(ds)
+ if err != nil {
+ return nil, err
}
- client, _, err := c.configured(ds)
+ if parentID == "" {
+ spaces, err := c.pickerSpaces(ctx, client, cfg)
+ if err != nil {
+ return nil, err
+ }
+ out := make([]types.Resource, 0, len(spaces))
+ for _, s := range spaces {
+ metadata := map[string]interface{}{"space_key": s.Key}
+ if cfg.cloud() {
+ // Cloud cannot enumerate a space's top-level folders, so pages under
+ // them are unselectable in the picker; the frontend surfaces this
+ // limitation when the space is expanded.
+ metadata["hierarchy_limitation"] = "cloud_top_level_containers"
+ }
+ out = append(out, types.Resource{
+ ExternalID: s.ID, Name: s.Name, Type: "space",
+ URL: client.resourceURL(s.Links.WebUI), HasChildren: true,
+ Metadata: metadata,
+ })
+ }
+ return out, nil
+ }
+ ref, err := parseResourceID(parentID)
if err != nil {
return nil, err
}
- spaces, err := client.spaces(ctx)
+ spaces, err := c.pickerSpaces(ctx, client, cfg)
if err != nil {
return nil, err
}
- out := make([]types.Resource, 0, len(spaces))
+ var selected space
+ found := false
for _, s := range spaces {
- out = append(out, types.Resource{
- ExternalID: s.ID,
- Name: s.Name,
- Type: "space",
- URL: client.resourceURL(s.Links.WebUI),
- HasChildren: false,
- Metadata: map[string]interface{}{"space_key": s.Key},
- })
+ if s.ID == ref.SpaceID {
+ selected, found = s, true
+ break
+ }
+ }
+ if !found {
+ return nil, fmt.Errorf("confluence space %s is unavailable", ref.SpaceID)
+ }
+ var pages []page
+ if ref.Kind == resourceSpace {
+ pages, err = client.topLevelPages(ctx, selected)
+ } else {
+ if ref.Kind == resourcePage {
+ if err := client.validatePageOwnership(ctx, selected, ref.PageID); err != nil {
+ return nil, fmt.Errorf("validate Confluence page %s: %w", ref.PageID, err)
+ }
+ }
+ pages, err = client.directChildPages(ctx, selected, ref.PageID)
+ }
+ if err != nil {
+ return nil, fmt.Errorf("list Confluence resources under %s: %w", parentID, err)
+ }
+ out := make([]types.Resource, 0, len(pages))
+ for _, p := range pages {
+ out = append(out, pageResource(client, selected, parentID, p))
}
return out, nil
}
-// ResolveResourceAncestors is a no-op: Confluence spaces are a flat list.
-func (*Connector) ResolveResourceAncestors(context.Context, *types.DataSourceConfig, []string) ([]string, error) {
- return nil, nil
+func pageResource(client *client, s space, parentID string, p page) types.Resource {
+ return types.Resource{
+ ExternalID: makePageResourceID(s.ID, p.ID), ParentID: parentID, Name: p.Title, Type: "page",
+ URL: client.resourceURL(p.Links.WebUI), HasChildren: true,
+ Metadata: map[string]interface{}{"space_id": s.ID, "page_id": p.ID},
+ }
+}
+
+// ResolveResourceAncestors expands only the O(depth) path required to reveal
+// persisted page selections in the lazy picker. Resolution is best effort:
+// a deleted page, an unavailable space, or a malformed id skips just its own
+// reveal path instead of failing every saved selection; the sync path keeps
+// its strict validation in buildSyncPlan.
+func (c *Connector) ResolveResourceAncestors(
+ ctx context.Context,
+ ds *types.DataSourceConfig,
+ resourceIDs []string,
+) ([]string, error) {
+ client, _, err := c.configured(ds)
+ if err != nil {
+ return nil, err
+ }
+ spaces, err := client.spaces(ctx)
+ if err != nil {
+ return nil, err
+ }
+ byID := make(map[string]space, len(spaces))
+ for _, s := range spaces {
+ byID[s.ID] = s
+ }
+ seen, out := map[string]bool{}, []string{}
+ add := func(id string) {
+ if id != "" && !seen[id] {
+ seen[id] = true
+ out = append(out, id)
+ }
+ }
+ for _, id := range resourceIDs {
+ ref, err := parseResourceID(id)
+ if err != nil {
+ logger.Warnf(ctx, "skip malformed Confluence resource %q during ancestor resolution: %v", id, err)
+ continue
+ }
+ if ref.Kind == resourceSpace {
+ continue
+ }
+ s, ok := byID[ref.SpaceID]
+ if !ok {
+ logger.Warnf(ctx, "skip Confluence resource %s: space %s is unavailable", id, ref.SpaceID)
+ continue
+ }
+ ancestors, err := client.pageAncestors(ctx, s, ref.PageID)
+ if err != nil {
+ logger.Warnf(ctx, "skip Confluence resource %s during ancestor resolution: %v", id, err)
+ continue
+ }
+ add(s.ID)
+ for _, ancestor := range ancestors {
+ if ancestor.Kind != "" && ancestor.Kind != "page" {
+ continue
+ }
+ add(makePageResourceID(s.ID, ancestor.ID))
+ }
+ }
+ return out, nil
}
// FetchAll syncs every selected space. Deletion reconciliation needs FetchStream.
@@ -158,51 +318,90 @@ func (c *Connector) fetchStream(
if err != nil {
return nil, err
}
- spaces, err := client.spaces(ctx)
+ plan, err := c.buildSyncPlan(ctx, client, ds.ResourceIDs)
if err != nil {
- return nil, fmt.Errorf("list Confluence spaces: %w", err)
+ return nil, err
}
- byID := make(map[string]space, len(spaces))
- for _, s := range spaces {
- byID[s.ID] = s
+ currentRoots := make(map[string]bool, len(plan.Roots))
+ for _, root := range plan.Roots {
+ currentRoots[root.ResourceID] = true
}
baseline, next := prepareSyncCursors(old, forceFull)
- for _, resourceID := range ds.ResourceIDs {
- s, found := byID[resourceID]
- if !found {
- return nil, fmt.Errorf(
- "selected Confluence space %s is unavailable; refusing destructive reconciliation",
- resourceID,
- )
+ // A prior successful body is reusable when ownership moves to a different
+ // selected root and this run cannot fetch the new body. Keep that version in
+ // the new root rather than dropping it during ownership reconciliation.
+ priorVersionByPage := make(map[string]string)
+ for _, pages := range baseline.SpacePages {
+ for id, version := range pages {
+ if _, exists := priorVersionByPage[id]; !exists {
+ priorVersionByPage[id] = version
+ }
+ }
+ }
+ // A page may be returned by multiple overlapping roots (for example during
+ // a configuration change). Fetch it once, while retaining a cursor entry
+ // per scope root so existing cursor JSON remains compatible.
+ seenPages := make(map[string]struct{})
+ incompleteRoots := make(map[string]bool)
+ incompleteSpaces := make([]space, 0)
+ for _, root := range plan.Roots {
+ resourceID, s := root.ResourceID, root.Space
+ var pages []page
+ complete := true
+ if root.Kind == syncWholeSpace {
+ pages, complete, err = client.pages(ctx, s)
+ } else {
+ pages, err = client.pageSubtree(ctx, s, root.PageID)
}
- pages, complete, err := client.pages(ctx, s)
if err != nil {
- return nil, fmt.Errorf("list pages in Confluence space %s: %w", s.Key, err)
+ return nil, fmt.Errorf("list pages in Confluence scope %s: %w", resourceID, err)
}
+ priorPages := baseline.SpacePages[resourceID]
if !complete {
- logger.Warnf(ctx,
- "[Confluence] space %s: listing stopped after %d empty pages that still linked onward; "+
- "listed %d pages, checking unlisted ones one by one before deleting",
- s.Key, maxEmptyServerPages, len(pages))
+ logger.Warnf(
+ ctx,
+ "[Confluence] space %s listing was cut short; checking unlisted pages before deleting",
+ s.Key,
+ )
+ incompleteRoots[resourceID] = true
+ incompleteSpaces = append(incompleteSpaces, s)
}
- priorPages, hadBaseline := baseline.SpacePages[resourceID]
if next.SpacePages[resourceID] == nil {
next.SpacePages[resourceID] = map[string]string{}
}
- seen := make(map[string]struct{}, len(pages))
+ if complete && len(pages) == 0 && len(priorPages) > 0 {
+ return nil, fmt.Errorf(
+ "refusing Confluence mirror deletion in %s: listing returned 0 pages against a %d-page baseline",
+ s.Key, len(priorPages),
+ )
+ }
for _, summary := range pages {
if err := ctx.Err(); err != nil {
return nil, err
}
- seen[summary.ID] = struct{}{}
- version := pageVersion(summary)
- if forceFull {
- if next.SpacePages[resourceID][summary.ID] == version {
- continue
+ if _, duplicate := seenPages[summary.ID]; duplicate {
+ // The first scope already owns the emitted document; preserve this
+ // root's version record so a later selection change stays incremental.
+ if !forceFull {
+ if version, ok := priorPages[summary.ID]; ok {
+ next.SpacePages[resourceID][summary.ID] = version
+ }
}
- } else if priorPages[summary.ID] == version {
continue
}
+ seenPages[summary.ID] = struct{}{}
+ version, versionKnown := pageVersion(summary)
+ // An unknown version cannot prove a page unchanged, so it must
+ // always refresh the body instead of skipping on a stable "t:".
+ if versionKnown {
+ if forceFull {
+ if next.SpacePages[resourceID][summary.ID] == version {
+ continue
+ }
+ } else if priorPages[summary.ID] == version {
+ continue
+ }
+ }
full, err := client.body(ctx, summary.ID)
if err != nil {
if ctx.Err() != nil {
@@ -211,13 +410,23 @@ func (c *Connector) fetchStream(
if emitErr := h.Emit(ctx, failedPageItem(resourceID, summary, err)); emitErr != nil {
return nil, emitErr
}
+ if !forceFull {
+ if prior, ok := priorVersionByPage[summary.ID]; ok {
+ next.SpacePages[resourceID][summary.ID] = prior
+ }
+ }
continue
}
- item, err := markdownItem(client, resourceID, summary, full)
+ item, err := markdownItem(ctx, client, resourceID, summary, full)
if err != nil {
if emitErr := h.Emit(ctx, failedPageItem(resourceID, summary, err)); emitErr != nil {
return nil, emitErr
}
+ if !forceFull {
+ if prior, ok := priorVersionByPage[summary.ID]; ok {
+ next.SpacePages[resourceID][summary.ID] = prior
+ }
+ }
continue
}
if err := h.Emit(ctx, item); err != nil {
@@ -228,99 +437,117 @@ func (c *Connector) fetchStream(
return nil, err
}
}
- if !complete {
- if err := reconcileUnlisted(ctx, client, h, s, resourceID, priorPages, seen, &next); err != nil {
- return nil, err
+ }
+ // Reconcile previous owners only after every current scope has been enumerated.
+ // A page may move between roots, so a missing page must be checked against every
+ // scope whose listing was cut short before a tombstone can be emitted.
+ previousPages := make(map[string]string)
+ for owner, pages := range baseline.SpacePages {
+ for id := range pages {
+ if _, exists := previousPages[id]; !exists {
+ previousPages[id] = owner
}
- } else if hadBaseline {
- if len(pages) == 0 && len(priorPages) > 0 {
+ }
+ }
+ missing := make([]string, 0)
+ probeCount := 0
+ for id := range previousPages {
+ if _, present := seenPages[id]; present {
+ continue
+ }
+ owner := previousPages[id]
+ for _, s := range incompleteSpaces {
+ if probeCount >= maxDeletionProbes {
return nil, fmt.Errorf(
- "refusing Confluence mirror deletion in %s: listing returned 0 pages against a %d-page baseline",
- s.Key, len(priorPages),
+ "Confluence deletion reconciliation deferred: more than %d pages need verification",
+ maxDeletionProbes,
)
}
- missing := 0
- for id := range priorPages {
- if _, exists := seen[id]; !exists {
- missing++
+ probeCount++
+ present, err := client.pageInSpace(ctx, id, s.Key)
+ if err != nil {
+ if ctx.Err() != nil {
+ return nil, ctx.Err()
}
- }
- if missing >= 20 && missing*100 >= len(priorPages)*80 {
return nil, fmt.Errorf(
- "refusing Confluence mirror deletion in %s: %d/%d pages disappeared",
- s.Key, missing, len(priorPages),
+ "Confluence deletion reconciliation deferred: could not verify page %s in space %s: %w",
+ id, s.Key, err,
)
}
- for id := range priorPages {
- if _, exists := seen[id]; exists {
- continue
- }
- if err := emitDeleted(ctx, h, &next, resourceID, id); err != nil {
- return nil, err
+ if present {
+ version := baseline.SpacePages[owner][id]
+ if next.SpacePages[s.ID] == nil {
+ next.SpacePages[s.ID] = make(map[string]string)
}
+ next.SpacePages[s.ID][id] = version
+ seenPages[id] = struct{}{}
+ break
}
}
- }
- next.FullSync = false
- next.FullSyncBaseline = nil
- return next.syncCursor(), nil
-}
-
-// reconcileUnlisted handles the pages a cut-short listing did not return. The
-// listing proves nothing about them, so each one is looked up on its own and
-// deleted only when Confluence confirms it is gone from the space. Pages that
-// are still there, could not be checked, or exceed the per-run probe budget
-// stay tracked with their prior version and are checked again next run.
-func reconcileUnlisted(
- ctx context.Context,
- c *client,
- h datasource.StreamHandler,
- s space,
- resourceID string,
- priorPages map[string]string,
- seen map[string]struct{},
- next *cursor,
-) error {
- probed := 0
- for id, version := range priorPages {
- if _, exists := seen[id]; exists {
+ if _, present := seenPages[id]; present {
continue
}
- if probed >= maxDeletionProbes {
- next.SpacePages[resourceID][id] = version
+ missing = append(missing, id)
+ }
+ // Protect still-selected roots from a suspiciously large shrink. Removed
+ // roots are an explicit scope change, so their out-of-scope pages can be
+ // tombstoned once every current scope has been enumerated safely.
+ owners := make([]string, 0, len(baseline.SpacePages))
+ for owner := range baseline.SpacePages {
+ owners = append(owners, owner)
+ }
+ sort.Strings(owners)
+ for _, owner := range owners {
+ // A deliberate scope change may remove most of an old root. The guard is
+ // only for unexpected shrinkage of a still-selected, completely listed root.
+ if !currentRoots[owner] || incompleteRoots[owner] {
continue
}
- probed++
- present, err := c.pageInSpace(ctx, id, s.Key)
- if err != nil {
- if ctx.Err() != nil {
- return ctx.Err()
+ pages := baseline.SpacePages[owner]
+ missingInRoot := 0
+ for id := range pages {
+ if _, present := seenPages[id]; !present {
+ missingInRoot++
}
- logger.Warnf(ctx, "[Confluence] space %s: could not check unlisted page %s, keeping it: %v", s.Key, id, err)
- next.SpacePages[resourceID][id] = version
- continue
}
- if present {
- next.SpacePages[resourceID][id] = version
- continue
- }
- if err := emitDeleted(ctx, h, next, resourceID, id); err != nil {
- return err
+ if len(pages) > 0 && missingInRoot >= 20 && missingInRoot*100 >= len(pages)*80 {
+ return nil, fmt.Errorf(
+ "refusing Confluence mirror deletion: %d/%d previously synced pages would be removed from scope %s",
+ missingInRoot, len(pages), owner,
+ )
}
}
- return nil
-}
-
-func emitDeleted(ctx context.Context, h datasource.StreamHandler, next *cursor, resourceID, id string) error {
- deleted := types.FetchedItem{
- ExternalID: id, IsDeleted: true, SourceResourceID: resourceID,
- Metadata: map[string]string{"channel": types.ChannelConfluence},
+ sort.Strings(missing)
+ for _, id := range missing {
+ deleted := types.FetchedItem{
+ ExternalID: id, IsDeleted: true, SourceResourceID: previousPages[id],
+ Metadata: map[string]string{"channel": types.ChannelConfluence},
+ }
+ if err := h.Emit(ctx, deleted); err != nil {
+ return nil, err
+ }
+ // Drop the tombstoned page before checkpointing so the cursor records
+ // the deletion itself; an interrupted run resumes with the remaining
+ // candidates instead of re-emitting completed tombstones.
+ removePageOwnership(&next, id)
+ if err := h.Checkpoint(ctx, next.syncCursor()); err != nil {
+ return nil, err
+ }
}
- if err := h.Emit(ctx, deleted); err != nil {
- return err
+ // Commit exactly the current ownership roots only after reconciliation. This
+ // removes cancelled scopes while keeping the on-disk cursor field compatible.
+ committedRoots := make(map[string]map[string]string, len(plan.Roots))
+ for _, root := range plan.Roots {
+ if pages := next.SpacePages[root.ResourceID]; pages != nil {
+ committedRoots[root.ResourceID] = pages
+ } else {
+ committedRoots[root.ResourceID] = map[string]string{}
+ }
}
- delete(next.SpacePages[resourceID], id)
- return h.Checkpoint(ctx, next.syncCursor())
+ next.SpacePages = committedRoots
+ next.FullSync = false
+ next.FullSyncBaseline = nil
+ return next.syncCursor(), nil
}
func failedPageItem(resourceID string, summary page, err error) types.FetchedItem {
@@ -361,8 +588,18 @@ func classifyConfluenceError(err error) (code, reason string) {
return "confluence_sync_failed", "Confluence page could not be synced; see server logs"
}
-func markdownItem(client *client, resourceID string, summary page, full pageBody) (types.FetchedItem, error) {
+func markdownItem(
+ ctx context.Context,
+ client *client,
+ resourceID string,
+ summary page,
+ full pageBody,
+) (types.FetchedItem, error) {
html := full.Body.View.Value
+ // Inline same-origin private Confluence images as data URIs so the generic
+ // docparser image pipeline can persist them and rewrite to resource:// URLs.
+ // Best-effort: a failed image keeps its original src and never fails the page.
+ html = newAssetResolver(client).Resolve(ctx, html)
markdown, err := htmltomd.ConvertString(html)
if err != nil {
return types.FetchedItem{}, err
diff --git a/internal/datasource/connector/confluence/connector_stream_test.go b/internal/datasource/connector/confluence/connector_stream_test.go
index 2f38c787f..0197a55dd 100644
--- a/internal/datasource/connector/confluence/connector_stream_test.go
+++ b/internal/datasource/connector/confluence/connector_stream_test.go
@@ -10,6 +10,7 @@ import (
"net/http"
"strings"
"testing"
+ "time"
"github.com/Tencent/WeKnora/internal/datasource"
"github.com/Tencent/WeKnora/internal/types"
@@ -26,19 +27,18 @@ type streamPage struct {
}
type streamAPI struct {
- cloud bool
- pages []streamPage
- bodyCalls int
- bodyStatus map[string]int
- listCalls int
- // endlessNext mimics Server/Data Center builds that keep linking to a
- // next page after the last one, returning empty results forever (#3596).
- endlessNext bool
- // probe answers the per-page existence check (GET content/{id}?expand=space)
- // for pages that are not in the listing: an HTTP status, or 200 with the
- // space key the page now belongs to.
- probe map[string]probeReply
- probeCalls int
+ cloud bool
+ pages []streamPage
+ bodyCalls int
+ bodyStatus map[string]int
+ bodyHTML map[string]string
+ imageCalls int
+ listCalls int
+ ancestorCalls int
+ spaceCalls int
+ endlessNext bool
+ probe map[string]probeReply
+ probeCalls int
}
type probeReply struct {
@@ -71,7 +71,17 @@ func (a *streamAPI) pageMaps() []any {
func (a *streamAPI) response(req *http.Request) (*http.Response, error) {
path := req.URL.Path
switch {
+ case strings.HasPrefix(path, "/wiki/download/"):
+ a.imageCalls++
+ header := make(http.Header)
+ header.Set("Content-Type", "image/png")
+ return &http.Response{
+ StatusCode: http.StatusOK,
+ Header: header,
+ Body: io.NopCloser(bytes.NewReader([]byte("private-png-bytes"))),
+ }, nil
case path == "/wiki/rest/api/space":
+ a.spaceCalls++
return a.jsonResponse(map[string]any{
"results": []any{map[string]any{
"id": 1, "key": "ENG", "name": "Engineering",
@@ -80,19 +90,23 @@ func (a *streamAPI) response(req *http.Request) (*http.Response, error) {
})
case path == "/wiki/rest/api/space/ENG/content/page":
a.listCalls++
- if !a.endlessNext {
+ if req.URL.Query().Get("depth") == "root" {
+ return a.jsonResponse(map[string]any{"results": a.pageMaps()})
+ }
+ if a.endlessNext {
+ results := a.pageMaps()
+ if a.listCalls > 1 {
+ results = []any{}
+ }
return a.jsonResponse(map[string]any{
- "results": a.pageMaps(),
+ "results": results,
+ "_links": map[string]any{
+ "next": fmt.Sprintf("/rest/api/space/ENG/content/page?limit=100&start=%d", a.listCalls*100),
+ },
})
}
- results := a.pageMaps()
- if a.listCalls > 1 {
- results = []any{}
- }
- next := fmt.Sprintf("/rest/api/space/ENG/content/page?limit=100&start=%d", a.listCalls*100)
return a.jsonResponse(map[string]any{
- "results": results,
- "_links": map[string]any{"next": next},
+ "results": a.pageMaps(),
})
case strings.HasPrefix(path, "/wiki/rest/api/content/") && req.URL.Query().Get("expand") == "space":
a.probeCalls++
@@ -108,10 +122,36 @@ func (a *streamAPI) response(req *http.Request) (*http.Response, error) {
Body: io.NopCloser(strings.NewReader("{}")),
}, nil
}
- return a.jsonResponse(map[string]any{
- "id": id, "status": "current", "space": map[string]any{"key": reply.spaceKey},
- })
+ return a.jsonResponse(
+ map[string]any{"id": id, "status": "current", "space": map[string]any{"key": reply.spaceKey}},
+ )
+ case strings.HasPrefix(path, "/wiki/rest/api/content/") && strings.HasSuffix(path, "/child/page"):
+ return a.jsonResponse(map[string]any{"results": []any{}})
case strings.HasPrefix(path, "/wiki/rest/api/content/"):
+ if req.URL.Query().Get("expand") == "ancestors,space" {
+ a.ancestorCalls++
+ return a.jsonResponse(map[string]any{
+ "id": strings.TrimPrefix(path, "/wiki/rest/api/content/"),
+ "space": map[string]any{"key": "ENG", "name": "Engineering"},
+ "ancestors": []any{},
+ })
+ }
+ if req.URL.Query().Get("expand") == "version,space" {
+ id := strings.TrimPrefix(path, "/wiki/rest/api/content/")
+ for _, page := range a.pages {
+ if page.id == id {
+ return a.jsonResponse(
+ map[string]any{
+ "id": page.id,
+ "title": page.title,
+ "version": map[string]any{"number": page.version},
+ "space": map[string]any{"key": "ENG", "name": "Engineering"},
+ },
+ )
+ }
+ }
+ return nil, errors.New("missing page")
+ }
if req.URL.Query().Get("expand") != "body.view,version,space" {
return nil, errors.New("page body did not request rendered body.view")
}
@@ -137,8 +177,14 @@ func (a *streamAPI) response(req *http.Request) (*http.Response, error) {
})
}
return a.jsonResponse(map[string]any{"results": pages})
+ case strings.HasPrefix(path, "/wiki/api/v2/pages/") && strings.HasSuffix(path, "/ancestors"):
+ return a.jsonResponse(map[string]any{"results": []any{}})
+ case strings.HasPrefix(path, "/wiki/api/v2/pages/") && strings.HasSuffix(path, "/descendants"):
+ return a.jsonResponse(map[string]any{"results": []any{}})
+ case strings.HasPrefix(path, "/wiki/api/v2/pages/") && strings.HasSuffix(path, "/direct-children"):
+ return a.jsonResponse(map[string]any{"results": []any{}})
case strings.HasPrefix(path, "/wiki/api/v2/pages/"):
- if req.URL.Query().Get("body-format") != "view" {
+ if req.URL.Query().Get("body-format") != "" && req.URL.Query().Get("body-format") != "view" {
return nil, errors.New("cloud page body did not request body-format=view")
}
return a.pageBody(strings.TrimPrefix(path, "/wiki/api/v2/pages/"))
@@ -158,12 +204,17 @@ func (a *streamAPI) pageBody(id string) (*http.Response, error) {
}
for _, page := range a.pages {
if page.id == id {
+ value := "<p>" + page.title + "</p>"
+ if custom, ok := a.bodyHTML[id]; ok {
+ value = custom
+ }
return a.jsonResponse(map[string]any{
"id": page.id, "title": page.title,
+ "spaceId": "1",
"version": map[string]any{"number": page.version, "by": map[string]any{"displayName": "Ada"}},
"space": map[string]any{"key": "ENG", "name": "Engineering"},
"_links": map[string]any{"webui": "/wiki/pages/" + page.id},
- "body": map[string]any{"view": map[string]any{"value": "<p>" + page.title + "</p>"}},
+ "body": map[string]any{"view": map[string]any{"value": value}},
})
}
}
@@ -196,10 +247,11 @@ func cloudStreamConfig() *types.DataSourceConfig {
}
type captureHandler struct {
- items []types.FetchedItem
- checkpoints []*types.SyncCursor
- emitErr error
- succeedFirst int
+ items []types.FetchedItem
+ checkpoints []*types.SyncCursor
+ emitErr error
+ succeedFirst int
+ checkpointErrAt int
}
func (h *captureHandler) Emit(_ context.Context, item types.FetchedItem) error {
@@ -214,6 +266,9 @@ func (h *captureHandler) Emit(_ context.Context, item types.FetchedItem) error {
}
func (h *captureHandler) Checkpoint(_ context.Context, c *types.SyncCursor) error {
+ if h.checkpointErrAt > 0 && len(h.checkpoints)+1 == h.checkpointErrAt {
+ return errors.New("checkpoint failed")
+ }
raw, _ := json.Marshal(c)
var stored types.SyncCursor
_ = json.Unmarshal(raw, &stored)
@@ -248,16 +303,144 @@ func TestFetchStreamSkipsUnchangedPages(t *testing.T) {
}
}
+func TestFetchStreamUnknownVersionDoesNotSkipBody(t *testing.T) {
+ // version.number=0 with no when/createdAt leaves the version unknown; the
+ // legacy stable "t:" cursor value must not suppress the body refresh.
+ api := &streamAPI{pages: []streamPage{{id: "p1", title: "Page", version: 0}}}
+ h := &captureHandler{}
+ old := streamCursor(map[string]string{"p1": "t:"})
+ next, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if api.bodyCalls != 1 || len(h.items) != 1 || h.items[0].IsDeleted {
+ t.Fatalf("unknown-version page fetched=%d items=%#v", api.bodyCalls, h.items)
+ }
+ if decoded := decodeCursor(next); decoded.SpacePages["1"]["p1"] != "" {
+ t.Fatalf("cursor stored a stable unknown version: %#v", decoded)
+ }
+}
+
func TestFetchStreamDetectsDeletedPages(t *testing.T) {
api := &streamAPI{pages: []streamPage{{id: "p1", title: "Page", version: 1}}}
h := &captureHandler{}
old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"})
- if _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h); err != nil {
+ next, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(h.items) != 1 || !h.items[0].IsDeleted || h.items[0].ExternalID != "p2" {
+ t.Fatalf("deletion items = %#v", h.items)
+ }
+ if decoded := decodeCursor(next); decoded.SpacePages["1"]["p2"] != "" {
+ t.Fatalf("deleted page stayed in cursor as a ghost: %#v", decoded)
+ }
+}
+
+func TestFetchStreamRemovesDeletedPageFromCursor(t *testing.T) {
+ api := &streamAPI{pages: []streamPage{{id: "p1", title: "Page", version: 1}}}
+ h := &captureHandler{}
+ old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"})
+ next, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h)
+ if err != nil {
t.Fatal(err)
}
if len(h.items) != 1 || !h.items[0].IsDeleted || h.items[0].ExternalID != "p2" {
t.Fatalf("deletion items = %#v", h.items)
}
+ decoded := decodeCursor(next)
+ if decoded.SpacePages["1"]["p1"] != "v:1" {
+ t.Fatalf("surviving page missing from cursor: %#v", decoded)
+ }
+ for _, checkpoint := range h.checkpoints {
+ if stored := decodeCursor(checkpoint); stored.SpacePages["1"]["p2"] != "" {
+ t.Fatalf("checkpoint recorded a not-yet-deleted page as deleted: %#v", stored)
+ }
+ }
+ // A follow-up run must not re-emit the already tombstoned page.
+ followUp := &captureHandler{}
+ if _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), next, followUp); err != nil {
+ t.Fatal(err)
+ }
+ for _, item := range followUp.items {
+ if item.IsDeleted {
+ t.Fatalf("ghost page re-tombstoned on the next run: %#v", followUp.items)
+ }
+ }
+}
+
+func TestFetchStreamTombstoneResumeSkipsCompletedDeletions(t *testing.T) {
+ api := &streamAPI{pages: []streamPage{{id: "p1", title: "Page", version: 1}}}
+ old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1", "p3": "v:1"})
+ first := &captureHandler{succeedFirst: 1}
+ if _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, first); err == nil {
+ t.Fatal("interrupted tombstone run unexpectedly succeeded")
+ }
+ if len(first.items) != 1 || !first.items[0].IsDeleted || first.items[0].ExternalID != "p2" {
+ t.Fatalf("first run items = %#v", first.items)
+ }
+ if len(first.checkpoints) != 1 {
+ t.Fatalf("first run checkpoints = %d; want 1", len(first.checkpoints))
+ }
+ second := &captureHandler{}
+ next, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), first.checkpoints[0], second)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(second.items) != 1 || !second.items[0].IsDeleted || second.items[0].ExternalID != "p3" {
+ t.Fatalf("resume must tombstone only the remaining page: %#v", second.items)
+ }
+ if decoded := decodeCursor(next); decoded.SpacePages["1"]["p1"] != "v:1" || decoded.SpacePages["1"]["p3"] != "" {
+ t.Fatalf("resume cursor = %#v", decoded)
+ }
+}
+
+func TestFetchFullStreamTombstoneResumeSkipsCompletedDeletions(t *testing.T) {
+ api := &streamAPI{pages: []streamPage{{id: "p1", title: "Page", version: 1}}}
+ old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1", "p3": "v:1"})
+ // p1 re-fetch succeeds and the p2 tombstone is checkpointed before the
+ // worker dies on the p3 tombstone.
+ first := &captureHandler{succeedFirst: 2}
+ if _, err := newStreamConnector(api).FetchFullStream(context.Background(), streamConfig(), old, first); err == nil {
+ t.Fatal("interrupted full-sync tombstone run unexpectedly succeeded")
+ }
+ if len(first.items) != 2 || first.items[1].ExternalID != "p2" || !first.items[1].IsDeleted {
+ t.Fatalf("first run items = %#v", first.items)
+ }
+ if len(first.checkpoints) != 2 {
+ t.Fatalf("first run checkpoints = %d; want 2", len(first.checkpoints))
+ }
+ deletionCheckpoint := decodeCursor(first.checkpoints[1])
+ if !deletionCheckpoint.FullSync {
+ t.Fatalf("mid-full-sync checkpoint lost the resume flag: %#v", deletionCheckpoint)
+ }
+ if baseline := deletionCheckpoint.FullSyncBaseline["1"]; baseline["p1"] != "v:1" || baseline["p2"] != "" ||
+ baseline["p3"] != "v:1" {
+ t.Fatalf("tombstone checkpoint did not advance the deletion baseline: %#v", deletionCheckpoint)
+ }
+
+ resumeAPI := &streamAPI{pages: api.pages}
+ second := &captureHandler{}
+ next, err := newStreamConnector(
+ resumeAPI,
+ ).FetchFullStream(context.Background(), streamConfig(), first.checkpoints[1], second)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if resumeAPI.bodyCalls != 0 {
+ t.Fatalf("resume re-fetched %d already synced page bodies", resumeAPI.bodyCalls)
+ }
+ if len(second.items) != 1 || !second.items[0].IsDeleted || second.items[0].ExternalID != "p3" {
+ t.Fatalf("resume must tombstone only the remaining page: %#v", second.items)
+ }
+ decoded := decodeCursor(next)
+ if decoded.FullSync || decoded.FullSyncBaseline != nil {
+ t.Fatalf("completed full sync left resume fields set: %#v", decoded)
+ }
+ if decoded.SpacePages["1"]["p1"] != "v:1" || decoded.SpacePages["1"]["p2"] != "" ||
+ decoded.SpacePages["1"]["p3"] != "" {
+ t.Fatalf("completed resume cursor = %#v", decoded)
+ }
}
func TestFetchStreamRefusesMassDeletion(t *testing.T) {
@@ -275,6 +458,67 @@ func TestFetchStreamRefusesMassDeletion(t *testing.T) {
}
}
+func TestFetchStreamRefusesMassDeletionInSurvivingScope(t *testing.T) {
+ prior := make(map[string]string, 100)
+ for i := 0; i < 100; i++ {
+ prior[fmt.Sprintf("p%03d", i)] = "v:1"
+ }
+ // The selection is unchanged; a broken listing suddenly returns only five
+ // of the hundred previously synced pages.
+ listed := make([]streamPage, 0, 5)
+ for i := 0; i < 5; i++ {
+ listed = append(listed, streamPage{id: fmt.Sprintf("p%03d", i), title: "Page", version: 1})
+ }
+ api := &streamAPI{pages: listed}
+ h := &captureHandler{}
+ _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), streamCursor(prior), h)
+ if err == nil || len(h.items) != 0 {
+ t.Fatalf("surviving-scope mass deletion err=%v items=%#v", err, h.items)
+ }
+}
+
+func TestFetchStreamAllowsExplicitSpaceToSubtreeShrink(t *testing.T) {
+ prior := make(map[string]string, 100)
+ for i := 0; i < 100; i++ {
+ prior[fmt.Sprintf("p%03d", i)] = "v:1"
+ }
+ // The user deliberately narrows a whole-space selection down to a page
+ // subtree. The mass-deletion guard must only protect a surviving root.
+ api := &streamAPI{pages: []streamPage{{id: "erp", title: "ERP", version: 2}}}
+ ds := streamConfig()
+ ds.ResourceIDs = []string{"page:1:erp"}
+ h := &captureHandler{}
+ next, err := newStreamConnector(api).FetchStream(context.Background(), ds, streamCursor(prior), h)
+ if err != nil {
+ t.Fatalf("explicit scope shrink: %v", err)
+ }
+ if len(h.items) != len(prior)+1 || h.items[0].ExternalID != "erp" {
+ t.Fatalf("scope shrink items = %d, want selected page plus %d tombstones", len(h.items), len(prior))
+ }
+ deleted := 0
+ for _, item := range h.items[1:] {
+ if !item.IsDeleted {
+ t.Fatalf("scope shrink emitted a non-tombstone outside the new scope: %#v", item)
+ }
+ deleted++
+ }
+ if deleted != len(prior) {
+ t.Fatalf("scope shrink tombstones = %d, want %d", deleted, len(prior))
+ }
+ decoded := decodeCursor(next)
+ if len(decoded.SpacePages) != 1 || decoded.SpacePages["page:1:erp"]["erp"] != "v:2" {
+ t.Fatalf("scope shrink cursor retained cancelled roots: %#v", decoded.SpacePages)
+ }
+ if _, oldRootPresent := decoded.SpacePages["1"]; oldRootPresent {
+ t.Fatalf("cancelled space root returned to cursor: %#v", decoded.SpacePages)
+ }
+ for _, item := range h.items[1:] {
+ if item.SourceResourceID != "1" {
+ t.Fatalf("tombstone source = %q, want old root 1", item.SourceResourceID)
+ }
+ }
+}
+
func TestFetchStreamRefusesEmptyListingWhenBaselineExists(t *testing.T) {
prior := map[string]string{"p1": "v:1", "p2": "v:1"}
api := &streamAPI{}
@@ -349,6 +593,50 @@ func TestFetchFullStreamResumesFromCheckpoint(t *testing.T) {
}
}
+func TestFetchFullStreamRetryRefetchesFailedPage(t *testing.T) {
+ pages := []streamPage{
+ {id: "p1", title: "One", version: 1},
+ {id: "p2", title: "Broken then healthy", version: 2},
+ {id: "p3", title: "Three", version: 1},
+ }
+ api := &streamAPI{pages: pages, bodyStatus: map[string]int{"p2": http.StatusForbidden}}
+ first := &captureHandler{checkpointErrAt: 2}
+ old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1", "p3": "v:1"})
+ _, err := newStreamConnector(api).FetchFullStream(context.Background(), streamConfig(), old, first)
+ if err == nil {
+ t.Fatal("interrupted full sync unexpectedly succeeded")
+ }
+ if len(first.checkpoints) != 1 {
+ t.Fatalf("first full-sync checkpoints = %d; want one successful checkpoint", len(first.checkpoints))
+ }
+ checkpoint := decodeCursor(first.checkpoints[0])
+ if !checkpoint.FullSync || checkpoint.FullSyncBaseline["1"]["p2"] != "v:1" ||
+ checkpoint.SpacePages["1"]["p2"] != "" {
+ t.Fatalf("failed page was not left pending in the full-sync checkpoint: %#v", checkpoint)
+ }
+
+ resumeAPI := &streamAPI{pages: pages}
+ second := &captureHandler{}
+ next, err := newStreamConnector(resumeAPI).FetchFullStream(
+ context.Background(), streamConfig(), first.checkpoints[0], second,
+ )
+ if err != nil {
+ t.Fatal(err)
+ }
+ if resumeAPI.bodyCalls != 2 {
+ t.Fatalf("retry fetched %d bodies, want failed p2 and uncheckpointed p3", resumeAPI.bodyCalls)
+ }
+ if len(second.items) != 2 || second.items[0].ExternalID != "p2" || second.items[1].ExternalID != "p3" {
+ t.Fatalf("retry items = %#v; failed page was skipped", second.items)
+ }
+ decoded := decodeCursor(next)
+ if decoded.FullSync || decoded.FullSyncBaseline != nil || len(decoded.SpacePages) != 1 ||
+ decoded.SpacePages["1"]["p1"] != "v:1" || decoded.SpacePages["1"]["p2"] != "v:2" ||
+ decoded.SpacePages["1"]["p3"] != "v:1" {
+ t.Fatalf("completed full-sync cursor = %#v", decoded)
+ }
+}
+
func TestFetchStreamContinuesAfterPageBodyError(t *testing.T) {
api := &streamAPI{
pages: []streamPage{
@@ -430,57 +718,536 @@ func TestListResourcesKeepsInvalidWebUI(t *testing.T) {
}
}
-func TestFetchStreamChecksUnlistedPagesWhenServerListingNeverEnds(t *testing.T) {
- for _, full := range []bool{false, true} {
- t.Run(fmt.Sprintf("full=%v", full), func(t *testing.T) {
- api := &streamAPI{
- endlessNext: true,
- pages: []streamPage{{id: "p1", title: "Page", version: 2}},
- probe: map[string]probeReply{
- "gone": {status: http.StatusNotFound},
- "moved": {status: http.StatusOK, spaceKey: "OPS"},
- "kept": {status: http.StatusOK, spaceKey: "ENG"},
- "denied": {status: http.StatusForbidden},
- },
- }
- h := &captureHandler{}
- old := streamCursor(map[string]string{
- "p1": "v:1", "gone": "v:1", "moved": "v:1", "kept": "v:3", "denied": "v:1",
- })
- connector := newStreamConnector(api)
- fetch := connector.FetchStream
- if full {
- fetch = connector.FetchFullStream
- }
- got, err := fetch(context.Background(), streamConfig(), old, h)
- if err != nil {
- t.Fatalf("fetch: %v", err)
- }
- if api.listCalls != 1+maxEmptyServerPages {
- t.Fatalf("listCalls = %d, want %d", api.listCalls, 1+maxEmptyServerPages)
+func TestListResourcesLoadsConfluencePagesLazily(t *testing.T) {
+ api := &streamAPI{pages: []streamPage{{id: "p1", title: "Home", version: 1}}}
+ connector := newStreamConnector(api)
+ spaces, err := connector.ListResources(context.Background(), streamConfig(), "")
+ if err != nil || len(spaces) != 1 || !spaces[0].HasChildren {
+ t.Fatalf("spaces = %#v, %v", spaces, err)
+ }
+ pages, err := connector.ListResources(context.Background(), streamConfig(), "1")
+ if err != nil || len(pages) != 1 {
+ t.Fatalf("top pages = %#v, %v", pages, err)
+ }
+ if pages[0].ExternalID != "page:1:p1" || pages[0].ParentID != "1" || pages[0].Type != "page" ||
+ !pages[0].HasChildren {
+ t.Fatalf("page resource = %#v", pages[0])
+ }
+ children, err := connector.ListResources(context.Background(), streamConfig(), "page:1:p1")
+ if err != nil || len(children) != 0 {
+ t.Fatalf("children = %#v, %v", children, err)
+ }
+ if api.listCalls != 1 {
+ t.Fatalf("space expansion made %d whole-space list calls", api.listCalls)
+ }
+ if api.ancestorCalls != 0 {
+ t.Fatalf("page expansion made %d unnecessary ancestor calls", api.ancestorCalls)
+ }
+}
+
+func TestPickerSpaceCacheIsPartitionedByCredentialFingerprint(t *testing.T) {
+ api := &streamAPI{}
+ connector := newStreamConnector(api)
+ client, cfg, err := connector.configured(streamConfig())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := connector.pickerSpaces(context.Background(), client, cfg); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := connector.pickerSpaces(context.Background(), client, cfg); err != nil {
+ t.Fatal(err)
+ }
+ rotated := cfg
+ rotated.secret = "rotated-password"
+ if _, err := connector.pickerSpaces(context.Background(), client, rotated); err != nil {
+ t.Fatal(err)
+ }
+ if api.spaceCalls != 2 {
+ t.Fatalf("space list calls = %d; want 2 for distinct credentials", api.spaceCalls)
+ }
+}
+
+func TestPickerSpaceCachePrunesExpiredCredentialEntries(t *testing.T) {
+ api := &streamAPI{}
+ connector := newStreamConnector(api)
+ client, cfg, err := connector.configured(streamConfig())
+ if err != nil {
+ t.Fatal(err)
+ }
+ expired := cfg
+ expired.secret = "expired-password"
+ connector.spaceCache.entries = map[string]cachedSpaces{
+ pickerSpaceCacheKey(expired): {expires: time.Now().Add(-time.Second)},
+ }
+ if _, err := connector.pickerSpaces(context.Background(), client, cfg); err != nil {
+ t.Fatal(err)
+ }
+ if len(connector.spaceCache.entries) != 1 {
+ t.Fatalf("cache entries = %#v; expired credential entry was retained", connector.spaceCache.entries)
+ }
+}
+
+func TestFetchStreamSyncsSelectedPageSubtree(t *testing.T) {
+ api := &streamAPI{pages: []streamPage{{id: "p1", title: "Selected", version: 2}}}
+ ds := streamConfig()
+ ds.ResourceIDs = []string{"page:1:p1"}
+ h := &captureHandler{}
+ next, err := newStreamConnector(api).FetchStream(context.Background(), ds, nil, h)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(h.items) != 1 || h.items[0].SourceResourceID != "page:1:p1" || h.items[0].ExternalID != "p1" {
+ t.Fatalf("items = %#v", h.items)
+ }
+ if decoded := decodeCursor(next); decoded.SpacePages["page:1:p1"]["p1"] != "v:2" {
+ t.Fatalf("cursor = %#v", decoded)
+ }
+}
+
+func TestFetchStreamReconcilesCancelledSpaceScope(t *testing.T) {
+ api := &streamAPI{pages: []streamPage{{id: "p1", title: "Selected", version: 2}}}
+ ds := streamConfig()
+ ds.ResourceIDs = []string{"page:1:p1"}
+ old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"})
+ h := &captureHandler{}
+ next, err := newStreamConnector(api).FetchStream(context.Background(), ds, old, h)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(h.items) != 2 || !h.items[1].IsDeleted || h.items[1].ExternalID != "p2" {
+ t.Fatalf("items = %#v", h.items)
+ }
+ decoded := decodeCursor(next)
+ if _, oldScopePresent := decoded.SpacePages["1"]; oldScopePresent ||
+ decoded.SpacePages["page:1:p1"]["p1"] != "v:2" {
+ t.Fatalf("cursor = %#v", decoded)
+ }
+}
+
+func TestFetchStreamPreservesMovedPageWhenBodyFails(t *testing.T) {
+ api := &streamAPI{
+ pages: []streamPage{{id: "p1", title: "Selected", version: 2}},
+ bodyStatus: map[string]int{"p1": http.StatusNotFound},
+ }
+ ds := streamConfig()
+ ds.ResourceIDs = []string{"page:1:p1"}
+ old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"})
+ h := &captureHandler{}
+ next, err := newStreamConnector(api).FetchStream(context.Background(), ds, old, h)
+ if err != nil {
+ t.Fatal(err)
+ }
+ decoded := decodeCursor(next)
+ if decoded.SpacePages["page:1:p1"]["p1"] != "v:1" {
+ t.Fatalf("prior version was not preserved: %#v", decoded)
+ }
+ deletedP2 := false
+ for _, item := range h.items {
+ if item.IsDeleted && item.ExternalID == "p1" {
+ t.Fatalf("failed page was deleted: %#v", h.items)
+ }
+ deletedP2 = deletedP2 || (item.IsDeleted && item.ExternalID == "p2")
+ }
+ if !deletedP2 {
+ t.Fatalf("cancelled scope page was not deleted: %#v", h.items)
+ }
+}
+
+func TestFetchStreamDoesNotDeleteWhenNewScopeCannotEnumerate(t *testing.T) {
+ api := &streamAPI{pages: []streamPage{{id: "p1", title: "Selected", version: 2}}}
+ connector := newStreamConnector(api)
+ originalFactory := connector.newClient
+ connector.newClient = func(cfg config) (*client, error) {
+ c, err := originalFactory(cfg)
+ if err != nil {
+ return nil, err
+ }
+ originalTransport := c.http.Transport
+ c.http.Transport = roundTripper(func(req *http.Request) (*http.Response, error) {
+ if req.URL.Path == "/wiki/rest/api/content/p1" && req.URL.Query().Get("expand") == "version,space" {
+ return &http.Response{
+ StatusCode: http.StatusNotFound,
+ Header: make(http.Header),
+ Body: io.NopCloser(strings.NewReader("missing")),
+ }, nil
}
- if api.probeCalls != 4 {
- t.Fatalf("probeCalls = %d, want one per unlisted page", api.probeCalls)
+ return originalTransport.RoundTrip(req)
+ })
+ return c, nil
+ }
+ ds := streamConfig()
+ ds.ResourceIDs = []string{"page:1:p1"}
+ h := &captureHandler{}
+ _, err := connector.FetchStream(
+ context.Background(),
+ ds,
+ streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"}),
+ h,
+ )
+ if err == nil {
+ t.Fatal("scope enumeration failure unexpectedly succeeded")
+ }
+ for _, item := range h.items {
+ if item.IsDeleted {
+ t.Fatalf("incomplete scope emitted a tombstone: %#v", h.items)
+ }
+ }
+}
+
+func TestBuildSyncPlanNormalizesOverlappingScopes(t *testing.T) {
+ api := &streamAPI{}
+ client := &client{
+ cfg: config{baseURL: "https://confluence.test/wiki"},
+ http: &http.Client{Transport: roundTripper(func(req *http.Request) (*http.Response, error) {
+ switch req.URL.Path {
+ case "/wiki/rest/api/space":
+ return api.jsonResponse(
+ map[string]any{"results": []any{map[string]any{"id": 1, "key": "ENG", "name": "Engineering"}}},
+ )
+ case "/wiki/rest/api/content/parent":
+ return api.jsonResponse(
+ map[string]any{"id": "parent", "space": map[string]any{"key": "ENG"}, "ancestors": []any{}},
+ )
+ case "/wiki/rest/api/content/child":
+ return api.jsonResponse(
+ map[string]any{
+ "id": "child",
+ "space": map[string]any{"key": "ENG"},
+ "ancestors": []any{map[string]any{"id": "parent"}},
+ },
+ )
+ default:
+ return nil, errors.New("unexpected endpoint: " + req.URL.String())
}
- deleted := map[string]bool{}
- for _, item := range h.items {
- if item.IsDeleted {
- deleted[item.ExternalID] = true
+ })},
+ }
+ connector := NewConnector()
+ plan, err := connector.buildSyncPlan(context.Background(), client, []string{"page:1:child", "page:1:parent"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(plan.Roots) != 1 || plan.Roots[0].ResourceID != "page:1:parent" {
+ t.Fatalf("page overlap plan = %#v", plan)
+ }
+ plan, err = connector.buildSyncPlan(context.Background(), client, []string{"1", "page:1:child"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(plan.Roots) != 1 || plan.Roots[0].Kind != syncWholeSpace || plan.Roots[0].ResourceID != "1" {
+ t.Fatalf("space overlap plan = %#v", plan)
+ }
+}
+
+func TestFetchStreamCloudSyncsSelectedPageSubtree(t *testing.T) {
+ api := &streamAPI{cloud: true, pages: []streamPage{{id: "p1", title: "Cloud", version: 3}}}
+ ds := cloudStreamConfig()
+ ds.ResourceIDs = []string{"page:1:p1"}
+ h := &captureHandler{}
+ next, err := newStreamConnector(api).FetchStream(context.Background(), ds, nil, h)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(h.items) != 1 || h.items[0].ExternalID != "p1" || h.items[0].SourceResourceID != "page:1:p1" {
+ t.Fatalf("items = %#v", h.items)
+ }
+ if decoded := decodeCursor(next); decoded.SpacePages["page:1:p1"]["p1"] != "v:3" {
+ t.Fatalf("cursor = %#v", decoded)
+ }
+}
+
+func TestListResourcesCloudStaysLazyWithContainerLimitation(t *testing.T) {
+ api := &streamAPI{}
+ var listDepths []string
+ conn := &Connector{newClient: func(cfg config) (*client, error) {
+ return &client{
+ cfg: cfg,
+ http: &http.Client{Transport: roundTripper(func(req *http.Request) (*http.Response, error) {
+ switch req.URL.Path {
+ case "/wiki/api/v2/spaces":
+ return api.jsonResponse(
+ map[string]any{
+ "results": []any{map[string]any{"id": "1", "key": "ENG", "name": "Engineering"}},
+ },
+ )
+ case "/wiki/api/v2/spaces/1/pages":
+ listDepths = append(listDepths, req.URL.Query().Get("depth"))
+ // depth=0 surfaces root pages only; the folder and the page nested
+ // inside it are unreachable through this endpoint (CONFCLOUD-84275).
+ return api.jsonResponse(map[string]any{"results": []any{
+ map[string]any{"id": "top", "type": "page", "title": "Top"},
+ map[string]any{"id": "folder", "type": "folder", "title": "Folder"},
+ }})
+ default:
+ return nil, errors.New("unexpected endpoint: " + req.URL.String())
}
+ })},
+ }, nil
+ }}
+ // Cloud spaces carry the limitation marker so the picker can explain why
+ // pages under top-level containers are not selectable.
+ spaces, err := conn.ListResources(context.Background(), cloudStreamConfig(), "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(spaces) != 1 || spaces[0].Metadata["hierarchy_limitation"] != "cloud_top_level_containers" {
+ t.Fatalf("cloud space metadata = %#v", spaces)
+ }
+ resources, err := conn.ListResources(context.Background(), cloudStreamConfig(), "1")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(resources) != 1 || resources[0].ExternalID != "page:1:top" {
+ t.Fatalf("resources = %#v", resources)
+ }
+ // Expanding a space must stay a single shallow call, never a whole-space
+ // scan, even though top-level folders hide some pages.
+ if len(listDepths) != 1 || listDepths[0] != "0" {
+ t.Fatalf("space expansion depths = %#v; want a single depth=0 call", listDepths)
+ }
+}
+
+func TestResolveResourceAncestorsSkipsUnresolvableSelections(t *testing.T) {
+ api := &streamAPI{}
+ connector := &Connector{newClient: func(cfg config) (*client, error) {
+ return &client{
+ cfg: cfg,
+ http: &http.Client{Transport: roundTripper(func(req *http.Request) (*http.Response, error) {
+ switch req.URL.Path {
+ case "/wiki/rest/api/space":
+ return api.jsonResponse(map[string]any{"results": []any{map[string]any{
+ "id": 1, "key": "ENG", "name": "Engineering",
+ }}})
+ case "/wiki/rest/api/content/alive":
+ return api.jsonResponse(map[string]any{
+ "id": "alive", "space": map[string]any{"key": "ENG"},
+ "ancestors": []any{map[string]any{"id": "parent"}},
+ })
+ case "/wiki/rest/api/content/gone":
+ return &http.Response{
+ StatusCode: http.StatusNotFound,
+ Header: make(http.Header),
+ Body: io.NopCloser(strings.NewReader("missing page")),
+ }, nil
+ default:
+ return nil, errors.New("unexpected endpoint: " + req.URL.String())
+ }
+ })},
+ }, nil
+ }}
+ ancestors, err := connector.ResolveResourceAncestors(
+ context.Background(), streamConfig(),
+ []string{"page:1:alive", "page:1:gone", "page:9:elsewhere", "not-a-resource"},
+ )
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := []string{"1", "page:1:parent"}
+ if len(ancestors) != len(want) || ancestors[0] != want[0] || ancestors[1] != want[1] {
+ t.Fatalf("ancestors = %#v; want %#v", ancestors, want)
+ }
+}
+
+func TestVisibleCloudPageChildrenTraversesContainers(t *testing.T) {
+ api := &streamAPI{}
+ client := &client{
+ cfg: config{edition: editionCloud, baseURL: "https://confluence.test/wiki"},
+ http: &http.Client{Transport: roundTripper(func(req *http.Request) (*http.Response, error) {
+ switch req.URL.Path {
+ case "/wiki/api/v2/pages/a/direct-children":
+ return api.jsonResponse(map[string]any{"results": []any{
+ map[string]any{"id": "b", "type": "page", "title": "B"},
+ map[string]any{"id": "folder", "type": "folder", "title": "Folder"},
+ }})
+ case "/wiki/api/v2/folders/folder/direct-children":
+ return api.jsonResponse(map[string]any{"results": []any{
+ map[string]any{"id": "c", "type": "page", "title": "C"},
+ map[string]any{"id": "db", "type": "database", "title": "DB"},
+ }})
+ case "/wiki/api/v2/databases/db/direct-children":
+ return api.jsonResponse(
+ map[string]any{"results": []any{map[string]any{"id": "e", "type": "page", "title": "E"}}},
+ )
+ default:
+ return nil, errors.New("unexpected endpoint: " + req.URL.String())
}
- if len(deleted) != 2 || !deleted["gone"] || !deleted["moved"] {
- t.Fatalf("deleted = %v, want only the pages Confluence confirmed gone from the space", deleted)
- }
- pages := decodeCursor(got).SpacePages["1"]
- want := map[string]string{"p1": "v:2", "kept": "v:3", "denied": "v:1"}
- if len(pages) != len(want) {
- t.Fatalf("cursor pages = %#v, want %#v", pages, want)
+ })},
+ }
+ pages, err := client.visibleCloudPageChildren(context.Background(), "a")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(pages) != 3 || pages[0].ID != "b" || pages[1].ID != "c" || pages[2].ID != "e" {
+ t.Fatalf("projected pages = %#v", pages)
+ }
+}
+
+func TestCloudPageSubtreeWalksDeepMixedHierarchy(t *testing.T) {
+ api := &streamAPI{}
+ client := &client{
+ cfg: config{edition: editionCloud, baseURL: "https://confluence.test/wiki"},
+ http: &http.Client{Transport: roundTripper(func(req *http.Request) (*http.Response, error) {
+ if req.Method != http.MethodGet {
+ return nil, fmt.Errorf("unexpected method %s", req.Method)
}
- for id, version := range want {
- if pages[id] != version {
- t.Fatalf("cursor pages = %#v, want %#v", pages, want)
- }
+ switch req.URL.Path {
+ case "/wiki/api/v2/pages/root":
+ return api.jsonResponse(map[string]any{"id": "root", "spaceId": "1"})
+ case "/wiki/api/v2/pages/p1":
+ return api.jsonResponse(map[string]any{
+ "id": "p1", "spaceId": "1", "version": map[string]any{"number": 2},
+ })
+ case "/wiki/api/v2/pages/p2":
+ return api.jsonResponse(map[string]any{
+ "id": "p2", "spaceId": "1", "version": map[string]any{"number": 3},
+ })
+ case "/wiki/api/v2/pages/p3":
+ return api.jsonResponse(map[string]any{
+ "id": "p3", "spaceId": "1", "version": map[string]any{"number": 4},
+ })
+ case "/wiki/api/v2/pages/root/direct-children":
+ return api.jsonResponse(map[string]any{"results": []any{
+ map[string]any{"id": "p1", "type": "page", "title": "First page"},
+ map[string]any{"id": "folder", "type": "folder", "title": "Folder"},
+ }})
+ case "/wiki/api/v2/pages/p1/direct-children":
+ return api.jsonResponse(map[string]any{"results": []any{
+ map[string]any{"id": "db", "type": "database", "title": "Database"},
+ }})
+ case "/wiki/api/v2/folders/folder/direct-children":
+ return api.jsonResponse(map[string]any{"results": []any{
+ map[string]any{"id": "p2", "type": "page", "title": "Second page"},
+ }})
+ case "/wiki/api/v2/databases/db/direct-children":
+ return api.jsonResponse(map[string]any{"results": []any{
+ map[string]any{"id": "p3", "type": "page", "title": "Deep page"},
+ }})
+ case "/wiki/api/v2/pages/p2/direct-children", "/wiki/api/v2/pages/p3/direct-children":
+ return api.jsonResponse(map[string]any{"results": []any{}})
+ default:
+ return nil, fmt.Errorf("unexpected endpoint: %s", req.URL.String())
}
- })
+ })},
+ }
+
+ pages, err := client.pageSubtree(context.Background(), space{ID: "1"}, "root")
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := []string{"root", "p1", "p2", "p3"}
+ if len(pages) != len(want) {
+ t.Fatalf("subtree pages = %#v; want IDs %v", pages, want)
+ }
+ for i, id := range want {
+ if pages[i].ID != id || pages[i].SpaceID != "1" {
+ t.Fatalf("subtree page[%d] = %#v; want ID %q in space 1", i, pages[i], id)
+ }
+ }
+}
+
+func TestFetchStreamChecksUnlistedPagesWhenServerListingNeverEnds(t *testing.T) {
+ api := &streamAPI{
+ endlessNext: true,
+ pages: []streamPage{{id: "p1", title: "Page", version: 2}},
+ probe: map[string]probeReply{
+ "gone": {status: http.StatusNotFound},
+ "kept": {status: http.StatusOK, spaceKey: "ENG"},
+ },
+ }
+ h := &captureHandler{}
+ old := streamCursor(map[string]string{"p1": "v:1", "gone": "v:1", "kept": "v:1"})
+ got, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if api.listCalls != 1+maxEmptyServerPages || api.probeCalls != 2 {
+ t.Fatalf("list/probe calls = %d/%d", api.listCalls, api.probeCalls)
+ }
+ if len(h.items) != 2 || !h.items[1].IsDeleted || h.items[1].ExternalID != "gone" {
+ t.Fatalf("items = %#v", h.items)
+ }
+ pages := decodeCursor(got).SpacePages["1"]
+ if pages["p1"] != "v:2" || pages["kept"] != "v:1" || len(pages) != 2 {
+ t.Fatalf("cursor pages = %#v", pages)
+ }
+}
+
+func TestFetchStreamDoesNotRestoreCancelledRootForPageFoundByPartialProbe(t *testing.T) {
+ api := &streamAPI{
+ endlessNext: true,
+ probe: map[string]probeReply{
+ "kept": {status: http.StatusOK, spaceKey: "ENG"},
+ },
+ }
+ h := &captureHandler{}
+ old := (&cursor{SpacePages: map[string]map[string]string{
+ "page:1:old-root": {"kept": "v:1"},
+ }}).syncCursor()
+ next, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(h.items) != 0 {
+ t.Fatalf("confirmed-present page emitted items: %#v", h.items)
+ }
+ decoded := decodeCursor(next)
+ if len(decoded.SpacePages) != 1 || decoded.SpacePages["1"]["kept"] != "v:1" {
+ t.Fatalf("reconciled cursor = %#v; expected only the current root", decoded.SpacePages)
+ }
+ if _, oldRootPresent := decoded.SpacePages["page:1:old-root"]; oldRootPresent {
+ t.Fatalf("cancelled root was restored to cursor: %#v", decoded.SpacePages)
+ }
+}
+
+func TestFetchStreamDoesNotTombstoneWhenPartialListingProbeFails(t *testing.T) {
+ api := &streamAPI{
+ endlessNext: true,
+ probe: map[string]probeReply{
+ "gone": {status: http.StatusNotFound},
+ "unknown": {status: http.StatusForbidden},
+ },
+ }
+ h := &captureHandler{}
+ old := streamCursor(map[string]string{"gone": "v:1", "unknown": "v:1"})
+ _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), old, h)
+ if err == nil {
+ t.Fatal("unverified partial-listing candidate unexpectedly completed reconciliation")
+ }
+ for _, item := range h.items {
+ if item.IsDeleted {
+ t.Fatalf("probe failure allowed a tombstone: %#v", h.items)
+ }
+ }
+ if len(h.checkpoints) != 0 {
+ t.Fatalf("probe failure checkpointed reconciliation: %#v", h.checkpoints)
+ }
+ if api.probeCalls == 0 {
+ t.Fatal("empty incomplete listing was rejected before checking prior pages")
+ }
+}
+
+func TestFetchStreamDefersWhenPartialListingProbeBudgetIsExhausted(t *testing.T) {
+ prior := make(map[string]string, maxDeletionProbes+1)
+ probes := make(map[string]probeReply, maxDeletionProbes+1)
+ for i := 0; i <= maxDeletionProbes; i++ {
+ id := fmt.Sprintf("p%03d", i)
+ prior[id] = "v:1"
+ probes[id] = probeReply{status: http.StatusNotFound}
+ }
+ api := &streamAPI{endlessNext: true, probe: probes}
+ h := &captureHandler{}
+ _, err := newStreamConnector(api).FetchStream(context.Background(), streamConfig(), streamCursor(prior), h)
+ if err == nil {
+ t.Fatal("partial listing with unverified candidates unexpectedly completed")
+ }
+ if api.probeCalls != maxDeletionProbes {
+ t.Fatalf("existence probes = %d; want capped at %d", api.probeCalls, maxDeletionProbes)
+ }
+ if len(h.items) != 0 || len(h.checkpoints) != 0 {
+ t.Fatalf(
+ "probe budget exhaustion emitted or checkpointed reconciliation: items=%#v checkpoints=%#v",
+ h.items,
+ h.checkpoints,
+ )
}
}
diff --git a/internal/datasource/connector/confluence/resource.go b/internal/datasource/connector/confluence/resource.go
new file mode 100644
index 000000000..a34a16195
--- /dev/null
+++ b/internal/datasource/connector/confluence/resource.go
@@ -0,0 +1,43 @@
+package confluence
+
+import (
+ "fmt"
+ "strings"
+)
+
+type resourceKind uint8
+
+const (
+ resourceSpace resourceKind = iota
+ resourcePage
+)
+
+// resourceRef is the canonical, persisted representation of a selection.
+// Space IDs deliberately remain unprefixed for backwards compatibility.
+type resourceRef struct {
+ Kind resourceKind
+ SpaceID string
+ PageID string
+}
+
+func makePageResourceID(spaceID, pageID string) string {
+ return "page:" + spaceID + ":" + pageID
+}
+
+func parseResourceID(id string) (resourceRef, error) {
+ id = strings.TrimSpace(id)
+ if id == "" {
+ return resourceRef{}, fmt.Errorf("invalid empty Confluence resource ID")
+ }
+ if !strings.HasPrefix(id, "page:") {
+ if strings.Contains(id, ":") {
+ return resourceRef{}, fmt.Errorf("invalid Confluence space resource ID %q", id)
+ }
+ return resourceRef{Kind: resourceSpace, SpaceID: id}, nil
+ }
+ parts := strings.Split(id, ":")
+ if len(parts) != 3 || parts[0] != "page" || strings.TrimSpace(parts[1]) == "" || strings.TrimSpace(parts[2]) == "" {
+ return resourceRef{}, fmt.Errorf("invalid Confluence page resource ID %q", id)
+ }
+ return resourceRef{Kind: resourcePage, SpaceID: parts[1], PageID: parts[2]}, nil
+}
diff --git a/internal/datasource/connector/confluence/resource_test.go b/internal/datasource/connector/confluence/resource_test.go
new file mode 100644
index 000000000..b03e92f2d
--- /dev/null
+++ b/internal/datasource/connector/confluence/resource_test.go
@@ -0,0 +1,23 @@
+package confluence
+
+import "testing"
+
+func TestParseResourceID(t *testing.T) {
+ space, err := parseResourceID("86802433")
+ if err != nil || space.Kind != resourceSpace || space.SpaceID != "86802433" {
+ t.Fatalf("space = %#v, %v", space, err)
+ }
+ page, err := parseResourceID("page:86802433:123456789")
+ if err != nil || page.Kind != resourcePage || page.SpaceID != "86802433" || page.PageID != "123456789" {
+ t.Fatalf("page = %#v, %v", page, err)
+ }
+ invalid := []string{
+ "", "page:", "page:100", "page::200", "page:100:", "page:100:200:300",
+ "node:86802433:folder-1", "space:100",
+ }
+ for _, id := range invalid {
+ if _, err := parseResourceID(id); err == nil {
+ t.Errorf("parseResourceID(%q) succeeded", id)
+ }
+ }
+}
diff --git a/internal/datasource/connector/confluence/sync_plan.go b/internal/datasource/connector/confluence/sync_plan.go
new file mode 100644
index 000000000..c61484d92
--- /dev/null
+++ b/internal/datasource/connector/confluence/sync_plan.go
@@ -0,0 +1,112 @@
+package confluence
+
+import (
+ "context"
+ "fmt"
+ "sort"
+)
+
+type syncRootKind uint8
+
+const (
+ syncWholeSpace syncRootKind = iota
+ syncPageSubtree
+)
+
+type syncRoot struct {
+ ResourceID string
+ SpaceID string
+ Space space
+ PageID string
+ Kind syncRootKind
+}
+
+type syncPlan struct{ Roots []syncRoot }
+
+// buildSyncPlan validates selections before any destructive reconciliation and
+// removes scopes which are already covered by a selected ancestor.
+func (c *Connector) buildSyncPlan(ctx context.Context, client *client, resourceIDs []string) (*syncPlan, error) {
+ spaces, err := client.spaces(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("list Confluence spaces: %w", err)
+ }
+ byID := make(map[string]space, len(spaces))
+ for _, s := range spaces {
+ byID[s.ID] = s
+ }
+
+ plan := &syncPlan{}
+ wholeSpaces := make(map[string]bool)
+ pageRoots := make(map[string]syncRoot)
+ ancestorPaths := make(map[string][]page)
+ for _, resourceID := range resourceIDs {
+ ref, err := parseResourceID(resourceID)
+ if err != nil {
+ return nil, err
+ }
+ s, ok := byID[ref.SpaceID]
+ if !ok {
+ return nil, fmt.Errorf(
+ "selected Confluence space %s is unavailable; refusing destructive reconciliation",
+ ref.SpaceID,
+ )
+ }
+ if ref.Kind == resourceSpace {
+ wholeSpaces[ref.SpaceID] = true
+ continue
+ }
+ // pageAncestors fetches the page itself on Server/DC and Cloud; that is
+ // also the authoritative check that a client did not forge its space ID.
+ ancestors, err := client.pageAncestors(ctx, s, ref.PageID)
+ if err != nil {
+ return nil, fmt.Errorf("validate selected Confluence page %s: %w", ref.PageID, err)
+ }
+ pageRoots[resourceID] = syncRoot{
+ ResourceID: resourceID,
+ SpaceID: ref.SpaceID,
+ Space: s,
+ PageID: ref.PageID,
+ Kind: syncPageSubtree,
+ }
+ ancestorPaths[resourceID] = ancestors
+ for _, ancestor := range ancestors {
+ if ancestor.ID == ref.PageID {
+ return nil, fmt.Errorf("invalid cyclic Confluence page ancestry for %s", ref.PageID)
+ }
+ }
+ }
+ spaceIDs := make([]string, 0, len(wholeSpaces))
+ for id := range wholeSpaces {
+ spaceIDs = append(spaceIDs, id)
+ }
+ sort.Strings(spaceIDs)
+ for _, id := range spaceIDs {
+ plan.Roots = append(plan.Roots, syncRoot{ResourceID: id, SpaceID: id, Space: byID[id], Kind: syncWholeSpace})
+ }
+ pageIDs := make([]string, 0, len(pageRoots))
+ for id := range pageRoots {
+ pageIDs = append(pageIDs, id)
+ }
+ sort.Strings(pageIDs)
+ for _, id := range pageIDs {
+ root := pageRoots[id]
+ if wholeSpaces[root.SpaceID] {
+ continue
+ }
+ ancestors := ancestorPaths[id]
+ covered := false
+ for _, ancestor := range ancestors {
+ if ancestor.Kind != "" && ancestor.Kind != "page" {
+ continue
+ }
+ if _, ok := pageRoots[makePageResourceID(root.SpaceID, ancestor.ID)]; ok {
+ covered = true
+ break
+ }
+ }
+ if !covered {
+ plan.Roots = append(plan.Roots, pageRoots[id])
+ }
+ }
+ return plan, nil
+}
diff --git a/internal/datasource/connector/confluence/types.go b/internal/datasource/connector/confluence/types.go
index 7884f349d..498288083 100644
--- a/internal/datasource/connector/confluence/types.go
+++ b/internal/datasource/connector/confluence/types.go
@@ -95,10 +95,11 @@ func isAtlassianCloudHost(host string) bool {
}
type space struct {
- ID string `json:"id"`
- Key string `json:"key"`
- Name string `json:"name"`
- Links struct {
+ ID string `json:"id"`
+ Key string `json:"key"`
+ Name string `json:"name"`
+ HomepageID string `json:"homepageId"`
+ Links struct {
WebUI string `json:"webui"`
} `json:"_links"`
}
@@ -125,10 +126,13 @@ type serverSpaceList struct {
}
type page struct {
- ID string `json:"id"`
- Title string `json:"title"`
- Status string `json:"status"`
- Space struct {
+ ID string `json:"id"`
+ Title string `json:"title"`
+ Status string `json:"status"`
+ Kind string `json:"-"`
+ SpaceID string `json:"spaceId"`
+ Ancestors []page `json:"ancestors"`
+ Space struct {
Key string `json:"key"`
Name string `json:"name"`
} `json:"space"`
@@ -189,6 +193,27 @@ type cloudPage struct {
} `json:"_links"`
}
+// cloudHierarchyItem is shared by Cloud direct-children, descendants, and
+// ancestors. Non-page items are intentionally retained by the API model so we
+// can filter them explicitly at the navigation/import boundary.
+type cloudHierarchyItem struct {
+ ID string `json:"id"`
+ Type string `json:"type"`
+ Title string `json:"title"`
+ SpaceID string `json:"spaceId"`
+ Status string `json:"status"`
+ Links struct {
+ WebUI string `json:"webui"`
+ } `json:"_links"`
+}
+
+type cloudHierarchyList struct {
+ Results []cloudHierarchyItem `json:"results"`
+ Links struct {
+ Next string `json:"next"`
+ } `json:"_links"`
+}
+
type cloudPageList struct {
Results []cloudPage `json:"results"`
Links struct {
@@ -278,14 +303,37 @@ func prepareSyncCursors(old *types.SyncCursor, forceFull bool) (baseline, next c
return baseline, next
}
-func pageVersion(p page) string {
+// removePageOwnership drops a page from every root ownership in the cursor.
+// It runs after each tombstone emit so the checkpoint mirrors the deletion;
+// otherwise the page would linger as a ghost, be re-tombstoned on every later
+// run, and eventually trip the mass-deletion guard. During a full sync the
+// deletion baseline lives in FullSyncBaseline, not SpacePages (which is being
+// rebuilt from scratch), so both maps must record the completed tombstone for
+// a retry to resume with the remaining candidates only.
+func removePageOwnership(c *cursor, pageID string) {
+ removeFromPageMap(c.SpacePages, pageID)
+ if c.FullSync {
+ removeFromPageMap(c.FullSyncBaseline, pageID)
+ }
+}
+
+func removeFromPageMap(m map[string]map[string]string, pageID string) {
+ for _, pages := range m {
+ delete(pages, pageID)
+ }
+}
+
+func pageVersion(p page) (string, bool) {
if p.Version.Number > 0 {
- return fmt.Sprintf("v:%d", p.Version.Number)
+ return fmt.Sprintf("v:%d", p.Version.Number), true
}
if p.Version.When != "" {
- return "t:" + p.Version.When
+ return "t:" + p.Version.When, true
+ }
+ if p.Version.CreatedAt != "" {
+ return "t:" + p.Version.CreatedAt, true
}
- return "t:" + p.Version.CreatedAt
+ return "", false
}
func pageUpdatedAt(p page) time.Time {
diff --git a/internal/datasource/connector/confluence/types_test.go b/internal/datasource/connector/confluence/types_test.go
index 6a2247f03..bf62f7f34 100644
--- a/internal/datasource/connector/confluence/types_test.go
+++ b/internal/datasource/connector/confluence/types_test.go
@@ -75,6 +75,34 @@ func TestPageFileNameIncludesID(t *testing.T) {
}
}
+func TestPageVersionUnknown(t *testing.T) {
+ withNumber := page{}
+ withNumber.Version.Number = 7
+ withWhen := page{}
+ withWhen.Version.When = "2026-01-02T03:04:05Z"
+ withCreatedAt := page{}
+ withCreatedAt.Version.CreatedAt = "2026-01-02T03:04:05Z"
+
+ for _, tc := range []struct {
+ name string
+ in page
+ want string
+ wantKnown bool
+ }{
+ {"number", withNumber, "v:7", true},
+ {"when", withWhen, "t:2026-01-02T03:04:05Z", true},
+ {"createdAt", withCreatedAt, "t:2026-01-02T03:04:05Z", true},
+ {"unknown", page{}, "", false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ got, known := pageVersion(tc.in)
+ if got != tc.want || known != tc.wantKnown {
+ t.Fatalf("pageVersion() = (%q, %v); want (%q, %v)", got, known, tc.want, tc.wantKnown)
+ }
+ })
+ }
+}
+
func TestPrepareSyncCursorsPreservesFullSyncBaseline(t *testing.T) {
old := streamCursor(map[string]string{"p1": "v:1", "p2": "v:1"})
baseline, next := prepareSyncCursors(old, true)