RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat[syncer]: add data source Confluence (#18501)
Haruko38691ddbcf60f0f
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.goindex a4e6cf7c30..9150ee4925 100644--- a/internal/syncer/connector/builtin.go+++ b/internal/syncer/connector/builtin.go@@ -27,6 +27,7 @@ import (// source registers both the task-context factory (used by the syncer runtime)// and the raw-config factory (used by the test-connection endpoint).func RegisterBuiltIns(registry *Registry) {+ registerBuiltIn(registry, "confluence", NewConfluenceConnector)registerBuiltIn(registry, "rss", NewRSSConnector)registerBuiltIn(registry, "bitbucket", NewBitbucketConnector)registerBuiltIn(registry, "github", NewGitHubConnector)diff --git a/internal/syncer/connector/confluence.go b/internal/syncer/connector/confluence.gonew file mode 100644index 0000000000..a56aae514e--- /dev/null+++ b/internal/syncer/connector/confluence.go@@ -0,0 +1,996 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "bytes"+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "html"+ "io"+ "net/http"+ "net/url"+ "os"+ "path"+ "regexp"+ "sort"+ "strings"+ "time"++ xhtml "golang.org/x/net/html"+)++const (+ defaultConfluenceBatchSize = 32+ defaultConfluenceAttachmentThreshold = 10 * 1024 * 1024+ confluenceRequestTimeout = 60 * time.Second+ maxConfluenceResponseSize = 32 * 1024 * 1024+ maxConfluenceSearchPages = 1000+)++var (+ confluenceWhitespaceRE = regexp.MustCompile(`[ \t]+`)+ confluenceNewlineRE = regexp.MustCompile(`\n{3,}`)+)++// ConfluenceConnector reads pages, comments, and attachments from Confluence.+type ConfluenceConnector struct {+ wikiBase string+ apiBase string+ isCloud bool+ indexMode string+ space string+ pageID string+ indexRecursively bool+ cqlQuery string+ username string+ accessToken string+ batchSize int+ attachmentThreshold int64+ client *http.Client++ getJSON func(ctx context.Context, path string, out any) error+ download func(ctx context.Context, rawURL string) ([]byte, error)+ getCurrentUser func(ctx context.Context, userID string) (string, error)+}++// NewConfluenceConnector creates a Confluence connector from Python-compatible config.+func NewConfluenceConnector(config map[string]any) (*ConfluenceConnector, error) {+ credentials := configAnyMap(config["credentials"])+ wikiBase := strings.TrimRight(strings.TrimSpace(stringConfig(config["wiki_base"])), "/")+ isCloud := configBoolDefault(config["is_cloud"], true)+ indexMode := strings.ToLower(firstNonEmpty(stringConfig(config["index_mode"]), "everything"))+ batchSize := configInt(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])), defaultConfluenceBatchSize)+ c := &ConfluenceConnector{+ wikiBase: wikiBase,+ apiBase: confluenceAPIBase(wikiBase, isCloud),+ isCloud: isCloud,+ indexMode: indexMode,+ space: strings.TrimSpace(stringConfig(config["space"])),+ pageID: strings.TrimSpace(stringConfig(config["page_id"])),+ indexRecursively: configBoolDefault(config["index_recursively"], false),+ cqlQuery: strings.TrimSpace(stringConfig(config["cql_query"])),+ username: strings.TrimSpace(stringConfig(credentials["confluence_username"])),+ accessToken: stringConfig(credentials["confluence_access_token"]),+ batchSize: batchSize,+ attachmentThreshold: confluenceAttachmentThreshold(),+ client: &http.Client{Timeout: confluenceRequestTimeout},+ }+ c.getJSON = c.doJSON+ c.download = c.downloadURL+ c.getCurrentUser = c.currentUserName+ return c, nil+}++// Validate validates Confluence settings and credentials.+func (c *ConfluenceConnector) Validate(ctx context.Context) error {+ if c == nil {+ return fmt.Errorf("confluence connector is nil")+ }+ if c.wikiBase == "" {+ return fmt.Errorf("Confluence wiki_base is required")+ }+ parsed, err := url.Parse(c.wikiBase)+ if err != nil || parsed.Scheme == "" || parsed.Host == "" {+ return fmt.Errorf("invalid Confluence wiki_base URL")+ }+ if parsed.Scheme != "http" && parsed.Scheme != "https" {+ return fmt.Errorf("Confluence wiki_base must use HTTP or HTTPS")+ }+ if c.accessToken == "" {+ return fmt.Errorf("Confluence access token is required")+ }+ if c.isCloud && c.username == "" {+ return fmt.Errorf("Confluence Cloud username is required")+ }+ if c.batchSize <= 0 {+ return fmt.Errorf("batch_size must be a positive integer")+ }+ switch c.indexMode {+ case "everything", "space", "page", "":+ default:+ return fmt.Errorf("invalid Confluence index_mode %q", c.indexMode)+ }+ if c.indexMode == "space" && c.space == "" {+ return fmt.Errorf("Confluence space key is required when index_mode is space")+ }+ if c.indexMode == "page" && c.pageID == "" {+ return fmt.Errorf("Confluence page_id is required when index_mode is page")+ }++ var spaces confluenceSearchResponse+ if err := c.getJSON(ctx, "rest/api/space?limit=1", &spaces); err != nil {+ return confluenceValidationError(err)+ }+ if len(spaces.Results) == 0 {+ return fmt.Errorf("no Confluence spaces found")+ }+ if c.indexMode == "space" {+ var space confluenceSpace+ if err := c.getJSON(ctx, "rest/api/space/"+url.PathEscape(c.space), &space); err != nil {+ return fmt.Errorf("invalid Confluence space key provided: %w", confluenceValidationError(err))+ }+ }+ return nil+}++// ValidateConnectorSetting validates Confluence settings from an unsaved config.+func (c *ConfluenceConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)+ defer cancel()+ return c.Validate(ctx)+}++// OpenSync opens one Confluence sync session.+func (c *ConfluenceConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {+ if err := c.Validate(ctx); err != nil {+ return nil, err+ }+ end := request.WindowEnd+ if end.IsZero() {+ end = time.Now().UTC()+ }+ session := &confluenceSyncSession{+ connector: c,+ batchSize: c.batchSize,+ windowStart: request.WindowStart,+ windowEnd: end,+ fromBeginning: request.FromBeginning,+ pageCursor: newConfluenceSearchCursor(c, c.pageCQL(request.WindowStart, end, request.FromBeginning), strings.Join(confluencePageExpansionFields, ",")),+ nameCounts: map[string]int{},+ }+ session.applyResume(request.Resume)+ return session, nil+}++// OpenPrune opens one complete Confluence prune snapshot session.+func (c *ConfluenceConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {+ if err := c.Validate(ctx); err != nil {+ return nil, err+ }+ documents, err := c.loadSlimDocuments(ctx)+ if err != nil {+ return nil, err+ }+ sort.Slice(documents, func(i, j int) bool { return documents[i].SourceID < documents[j].SourceID })+ return &confluencePruneSession{documents: documents, batchSize: c.batchSize}, nil+}++func (c *ConfluenceConnector) loadSlimDocuments(ctx context.Context) ([]SlimDocument, error) {+ pages, err := c.searchAll(ctx, c.basePageCQL(), "")+ if err != nil {+ return nil, err+ }+ documents := make([]SlimDocument, 0, len(pages))+ for _, page := range pages {+ documents = append(documents, SlimDocument{SourceID: c.documentURL(page.Links.WebUI)})+ attachments, err := c.searchAll(ctx, c.attachmentCQL(page.ID.String(), nil, time.Time{}, true), strings.Join(confluenceAttachmentExpansionFields, ","))+ if err != nil {+ return nil, err+ }+ for _, attachment := range attachments {+ if !isConfluenceAttachmentAccepted(attachment) {+ continue+ }+ documents = append(documents, SlimDocument{SourceID: c.documentURL(attachment.Links.WebUI)})+ }+ }+ return documents, nil+}++func (c *ConfluenceConnector) pageDocument(ctx context.Context, page confluenceContent) (SourceDocument, error) {+ pageURL := c.documentURL(page.Links.WebUI)+ content := confluenceHTMLText(page.bodyHTML())+ comments, err := c.pageComments(ctx, page.ID.String())+ if err != nil {+ return SourceDocument{}, err+ }+ if comments != "" {+ content = strings.TrimSpace(content + "\n\nComments:\n" + comments)+ }+ blob := []byte(content)+ updatedAt := parseConfluenceTime(page.Version.When)+ if updatedAt.IsZero() {+ updatedAt = time.Now().UTC()+ }+ metadata := map[string]any{}+ if page.Space.Name != "" {+ metadata["space"] = page.Space.Name+ }+ if len(page.Metadata.Labels.Results) > 0 {+ labels := make([]string, 0, len(page.Metadata.Labels.Results))+ for _, label := range page.Metadata.Labels.Results {+ if label.Name != "" {+ labels = append(labels, label.Name)+ }+ }+ if len(labels) > 0 {+ metadata["labels"] = labels+ }+ }+ return SourceDocument{+ SourceID: pageURL,+ Extension: ".txt",+ Blob: blob,+ UpdatedAt: updatedAt,+ SizeBytes: int64(len(blob)),+ Metadata: metadataOrNil(metadata),+ Fingerprint: contentFingerprint(blob),+ }, nil+}++func (c *ConfluenceConnector) pageComments(ctx context.Context, pageID string) (string, error) {+ comments, err := c.searchAll(ctx, fmt.Sprintf("type=comment and container='%s'", confluenceCQLQuote(pageID)), "body.storage.value,body.view.value")+ if err != nil {+ return "", err+ }+ parts := make([]string, 0, len(comments))+ for _, comment := range comments {+ text := confluenceHTMLText(comment.bodyHTML())+ if text != "" {+ parts = append(parts, text)+ }+ }+ return strings.Join(parts, "\n"), nil+}++func (c *ConfluenceConnector) attachmentDocument(ctx context.Context, page confluenceContent, attachment confluenceContent) (SourceDocument, bool, error) {+ if !isConfluenceAttachmentAccepted(attachment) {+ return SourceDocument{}, false, nil+ }+ if attachment.Extensions.FileSize > c.attachmentThreshold {+ return SourceDocument{}, false, nil+ }+ rawURL := attachment.Links.Download+ if rawURL == "" {+ return SourceDocument{}, false, nil+ }+ blob, err := c.download(ctx, c.documentURL(rawURL))+ if err != nil {+ return SourceDocument{}, false, err+ }+ if len(blob) == 0 {+ return SourceDocument{}, false, nil+ }+ updatedAt := parseConfluenceTime(attachment.Version.When)+ if updatedAt.IsZero() {+ updatedAt = parseConfluenceTime(page.Version.When)+ }+ if updatedAt.IsZero() {+ updatedAt = time.Now().UTC()+ }+ metadata := map[string]any{"parent_page_id": c.documentURL(page.Links.WebUI)}+ if firstNonEmpty(attachment.Space.Name, page.Space.Name) != "" {+ metadata["space"] = firstNonEmpty(attachment.Space.Name, page.Space.Name)+ }+ if len(attachment.Metadata.Labels.Results) > 0 {+ labels := make([]string, 0, len(attachment.Metadata.Labels.Results))+ for _, label := range attachment.Metadata.Labels.Results {+ if label.Name != "" {+ labels = append(labels, label.Name)+ }+ }+ if len(labels) > 0 {+ metadata["labels"] = labels+ }+ }+ title := confluenceAttachmentTitle(attachment)+ return SourceDocument{+ SourceID: c.documentURL(attachment.Links.WebUI),+ Extension: confluenceAttachmentExtension(title),+ Blob: blob,+ UpdatedAt: updatedAt,+ SizeBytes: int64(len(blob)),+ Metadata: metadata,+ Fingerprint: contentFingerprint(blob),+ }, true, nil+}++func confluenceFileNameFromURL(rawURL string) string {+ parsed, err := url.Parse(rawURL)+ if err != nil {+ return ""+ }+ name := path.Base(parsed.Path)+ if name == "." || name == "/" {+ return ""+ }+ return name+}++func confluenceAttachmentTitle(attachment confluenceContent) string {+ return firstNonEmpty(attachment.Title, confluenceFileNameFromURL(attachment.Links.Download), "attachment")+}++func (c *ConfluenceConnector) searchAll(ctx context.Context, cql, expand string) ([]confluenceContent, error) {+ cursor := newConfluenceSearchCursor(c, cql, expand)+ var out []confluenceContent+ for {+ item, ok, err := cursor.next(ctx)+ if err != nil {+ return nil, err+ }+ if !ok {+ return out, nil+ }+ out = append(out, item)+ }+}++// confluenceSearchCursor iterates one Confluence content search, fetching pages+// on demand while guarding against repeated next links and unbounded pagination.+type confluenceSearchCursor struct {+ connector *ConfluenceConnector+ nextPath string+ seen map[string]struct{}+ pages int+ results []confluenceContent+ index int+}++func newConfluenceSearchCursor(connector *ConfluenceConnector, cql, expand string) *confluenceSearchCursor {+ return &confluenceSearchCursor{+ connector: connector,+ nextPath: confluenceCQLPath(cql, expand, connector.batchSize),+ seen: map[string]struct{}{},+ }+}++func (cur *confluenceSearchCursor) next(ctx context.Context) (confluenceContent, bool, error) {+ for cur.index >= len(cur.results) {+ if cur.nextPath == "" {+ return confluenceContent{}, false, nil+ }+ if _, ok := cur.seen[cur.nextPath]; ok {+ return confluenceContent{}, false, fmt.Errorf("confluence pagination repeated the same next link")+ }+ cur.seen[cur.nextPath] = struct{}{}+ cur.pages+++ if cur.pages > maxConfluenceSearchPages {+ return confluenceContent{}, false, fmt.Errorf("confluence search exceeded %d pages", maxConfluenceSearchPages)+ }+ var page confluenceSearchResponse+ if err := cur.connector.getJSON(ctx, cur.nextPath, &page); err != nil {+ return confluenceContent{}, false, err+ }+ cur.results = page.Results+ cur.index = 0+ cur.nextPath = page.Links.Next+ }+ item := cur.results[cur.index]+ cur.index+++ return item, true, nil+}++func (c *ConfluenceConnector) doJSON(ctx context.Context, path string, out any) error {+ data, err := c.do(ctx, http.MethodGet, path)+ if err != nil {+ return err+ }+ if err := json.Unmarshal(data, out); err != nil {+ return fmt.Errorf("decode Confluence response: %w", err)+ }+ return nil+}++func (c *ConfluenceConnector) downloadURL(ctx context.Context, rawURL string) ([]byte, error) {+ return c.do(ctx, http.MethodGet, rawURL)+}++func (c *ConfluenceConnector) do(ctx context.Context, method, rawURL string) ([]byte, error) {+ resolved, err := c.resolveURL(rawURL)+ if err != nil {+ return nil, err+ }+ req, err := http.NewRequestWithContext(ctx, method, resolved, nil)+ if err != nil {+ return nil, err+ }+ req.Header.Set("Accept", "application/json")+ if c.isCloud || c.username != "" {+ req.SetBasicAuth(c.username, c.accessToken)+ } else {+ req.Header.Set("Authorization", "Bearer "+c.accessToken)+ }+ res, err := c.client.Do(req)+ if err != nil {+ return nil, err+ }+ defer func() { _ = res.Body.Close() }()+ body, err := io.ReadAll(io.LimitReader(res.Body, maxConfluenceResponseSize+1))+ if err != nil {+ return nil, err+ }+ if int64(len(body)) > maxConfluenceResponseSize {+ return nil, fmt.Errorf("Confluence response exceeds %d bytes", maxConfluenceResponseSize)+ }+ if res.StatusCode < 200 || res.StatusCode >= 300 {+ return nil, &confluenceStatusError{status: res.StatusCode, body: string(body)}+ }+ return body, nil+}++func (c *ConfluenceConnector) resolveURL(rawURL string) (string, error) {+ if c.isCloud && strings.HasPrefix(rawURL, "/rest/") && strings.HasSuffix(c.apiBase, "/wiki") {+ rawURL = "/wiki" + rawURL+ }+ parsed, err := url.Parse(rawURL)+ if err != nil {+ return "", err+ }+ base, err := url.Parse(strings.TrimRight(c.apiBase, "/") + "/")+ if err != nil {+ return "", err+ }+ if parsed.IsAbs() {+ if !strings.EqualFold(parsed.Scheme, base.Scheme) || !strings.EqualFold(parsed.Host, base.Host) {+ return "", fmt.Errorf("confluence URL %q targets a different origin than the configured wiki base", rawURL)+ }+ return parsed.String(), nil+ }+ return base.ResolveReference(parsed).String(), nil+}++func (c *ConfluenceConnector) documentURL(contentURL string) string {+ return buildConfluenceDocumentID(c.wikiBase, contentURL, c.isCloud)+}++func (c *ConfluenceConnector) currentUserName(ctx context.Context, userID string) (string, error) {+ var user struct {+ DisplayName string `json:"displayName"`+ }+ field := "key"+ if c.isCloud {+ field = "accountId"+ }+ if err := c.getJSON(ctx, "rest/api/user?"+field+"="+url.QueryEscape(userID), &user); err != nil {+ return "", err+ }+ return user.DisplayName, nil+}++func (c *ConfluenceConnector) basePageCQL() string {+ if c.cqlQuery != "" {+ return c.cqlQuery+ }+ query := "type=page"+ switch c.indexMode {+ case "space":+ query += fmt.Sprintf(" and space='%s'", confluenceCQLQuote(c.space))+ case "page":+ if c.indexRecursively {+ query += fmt.Sprintf(" and (ancestor='%s' or id='%s')", confluenceCQLQuote(c.pageID), confluenceCQLQuote(c.pageID))+ } else {+ query += fmt.Sprintf(" and id='%s'", confluenceCQLQuote(c.pageID))+ }+ }+ return query+}++func (c *ConfluenceConnector) pageCQL(start *time.Time, end time.Time, fromBeginning bool) string {+ query := c.basePageCQL()+ if !fromBeginning && start != nil && !start.IsZero() {+ query += " and lastmodified >= '" + confluenceCQLTime(*start) + "'"+ }+ if !end.IsZero() {+ query += " and lastmodified <= '" + confluenceCQLTime(end) + "'"+ }+ return query + " order by lastmodified asc"+}++func (c *ConfluenceConnector) attachmentCQL(pageID string, start *time.Time, end time.Time, fromBeginning bool) string {+ query := fmt.Sprintf("type=attachment and container='%s'", confluenceCQLQuote(pageID))+ if !fromBeginning && start != nil && !start.IsZero() {+ query += " and lastmodified >= '" + confluenceCQLTime(*start) + "'"+ }+ if !end.IsZero() {+ query += " and lastmodified <= '" + confluenceCQLTime(end) + "'"+ }+ return query + " order by lastmodified asc"+}++type confluenceSyncSession struct {+ connector *ConfluenceConnector+ batchSize int+ windowStart *time.Time+ windowEnd time.Time+ fromBeginning bool++ pageCursor *confluenceSearchCursor+ currentPage confluenceContent+ hasCurrentPage bool+ pageDocPending bool+ attachCursor *confluenceSearchCursor++ resumeSourceID string+ resumeMatched bool+ resumeUpdatedAt *time.Time++ nameCounts map[string]int+}++// NextBatch returns the next Confluence document batch, fetching pages,+// comments, and attachments incrementally so session memory stays bounded by+// batchSize.+func (s *confluenceSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {+ documents := make([]SourceDocument, 0, s.batchSize)+ var checkpoint *SyncCheckpoint+ for len(documents) < s.batchSize {+ doc, err := s.nextDocument(ctx)+ if errors.Is(err, io.EOF) {+ if s.resumeSourceID != "" && !s.resumeMatched {+ return SyncBatch{}, fmt.Errorf("confluence sync resume checkpoint %q was not found in the source; refusing to discard unprocessed documents", s.resumeSourceID)+ }+ if len(documents) == 0 {+ return SyncBatch{}, io.EOF+ }+ break+ }+ if err != nil {+ return SyncBatch{}, err+ }+ if !s.includeResumed(*doc) {+ continue+ }+ documents = append(documents, *doc)+ checkpoint = confluenceSyncCheckpoint(*doc)+ }+ return SyncBatch{Documents: documents, Checkpoint: checkpoint}, nil+}++// Close closes the Confluence sync session.+func (s *confluenceSyncSession) Close() error {+ return nil+}++// nextDocument produces the next document in page order: each page followed by+// its accepted attachments, all filtered by the configured window.+func (s *confluenceSyncSession) nextDocument(ctx context.Context) (*SourceDocument, error) {+ for {+ if !s.hasCurrentPage {+ page, ok, err := s.pageCursor.next(ctx)+ if err != nil {+ return nil, err+ }+ if !ok {+ return nil, io.EOF+ }+ s.currentPage = page+ s.hasCurrentPage = true+ s.pageDocPending = true+ s.attachCursor = newConfluenceSearchCursor(s.connector, s.connector.attachmentCQL(page.ID.String(), s.windowStart, s.windowEnd, s.fromBeginning), strings.Join(confluenceAttachmentExpansionFields, ","))+ }++ if s.pageDocPending {+ s.pageDocPending = false+ doc, err := s.connector.pageDocument(ctx, s.currentPage)+ if err != nil {+ return nil, err+ }+ if s.fromBeginning || inConfluenceWindow(doc.UpdatedAt, s.windowStart, s.windowEnd) {+ doc.SemanticIdentifier = confluenceSemanticIdentifier(s.currentPage.Space.Name, s.currentPage.ancestorTitles(), s.currentPage.Title, s.nameCounts)+ return &doc, nil+ }+ continue+ }++ attachment, ok, err := s.attachCursor.next(ctx)+ if err != nil {+ return nil, err+ }+ if !ok {+ s.hasCurrentPage = false+ continue+ }+ doc, accepted, err := s.connector.attachmentDocument(ctx, s.currentPage, attachment)+ if err != nil {+ return nil, err+ }+ if !accepted {+ continue+ }+ if s.fromBeginning || inConfluenceWindow(doc.UpdatedAt, s.windowStart, s.windowEnd) {+ doc.SemanticIdentifier = confluenceSemanticIdentifier(firstNonEmpty(s.currentPage.Space.Name, attachment.Space.Name), nil, s.currentPage.Title+" / "+confluenceAttachmentTitle(attachment), s.nameCounts)+ return &doc, nil+ }+ }+}++func (s *confluenceSyncSession) includeResumed(doc SourceDocument) bool {+ if s.resumeMatched || s.resumeSourceID == "" {+ return true+ }+ if doc.SourceID == s.resumeSourceID {+ s.resumeMatched = true+ return false+ }+ if s.resumeUpdatedAt != nil && doc.UpdatedAt.After(*s.resumeUpdatedAt) {+ s.resumeMatched = true+ return true+ }+ return false+}++func (s *confluenceSyncSession) applyResume(checkpoint *SyncCheckpoint) {+ if checkpoint == nil {+ return+ }+ s.resumeSourceID = firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)+ s.resumeUpdatedAt = checkpoint.UpdatedAt+}++func confluenceSyncCheckpoint(doc SourceDocument) *SyncCheckpoint {+ updatedAt := doc.UpdatedAt+ return &SyncCheckpoint{Cursor: doc.SourceID, SourceID: doc.SourceID, UpdatedAt: &updatedAt}+}++type confluencePruneSession struct {+ documents []SlimDocument+ batchSize int+ index int+}++// NextBatch returns the next Confluence prune snapshot batch.+func (s *confluencePruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {+ if s.index >= len(s.documents) {+ return PruneBatch{}, io.EOF+ }+ end := s.index + s.batchSize+ if end > len(s.documents) {+ end = len(s.documents)+ }+ batch := PruneBatch{Documents: s.documents[s.index:end]}+ s.index = end+ return batch, nil+}++// Close closes the Confluence prune session.+func (s *confluencePruneSession) Close() error {+ return nil+}++type confluenceSearchResponse struct {+ Results []confluenceContent `json:"results"`+ Links struct {+ Next string `json:"next"`+ } `json:"_links"`+}++type confluenceSpace struct {+ Key string `json:"key"`+ Name string `json:"name"`+}++type confluenceContent struct {+ ID confluenceString `json:"id"`+ Title string `json:"title"`+ Type string `json:"type"`+ Body struct {+ Storage confluenceBody `json:"storage"`+ View confluenceBody `json:"view"`+ } `json:"body"`+ Space confluenceSpace `json:"space"`+ Ancestors []struct {+ Title string `json:"title"`+ } `json:"ancestors"`+ Version struct {+ When string `json:"when"`+ By struct {+ DisplayName string `json:"displayName"`+ Email string `json:"email"`+ } `json:"by"`+ } `json:"version"`+ Metadata struct {+ MediaType string `json:"mediaType"`+ Labels struct {+ Results []struct {+ Name string `json:"name"`+ } `json:"results"`+ } `json:"labels"`+ } `json:"metadata"`+ Extensions struct {+ FileSize int64 `json:"fileSize"`+ } `json:"extensions"`+ Links struct {+ WebUI string `json:"webui"`+ Download string `json:"download"`+ } `json:"_links"`+}++type confluenceBody struct {+ Value string `json:"value"`+}++type confluenceString string++func (s *confluenceString) UnmarshalJSON(data []byte) error {+ var text string+ if err := json.Unmarshal(data, &text); err == nil {+ *s = confluenceString(text)+ return nil+ }++ var number json.Number+ decoder := json.NewDecoder(bytes.NewReader(data))+ decoder.UseNumber()+ if err := decoder.Decode(&number); err == nil {+ *s = confluenceString(number.String())+ return nil+ }++ return fmt.Errorf("Confluence string field must be string or number")+}++func (s confluenceString) String() string {+ return string(s)+}++func (c confluenceContent) bodyHTML() string {+ return firstNonEmpty(c.Body.Storage.Value, c.Body.View.Value)+}++func (c confluenceContent) ancestorTitles() []string {+ out := make([]string, 0, len(c.Ancestors))+ for _, ancestor := range c.Ancestors {+ if ancestor.Title != "" {+ out = append(out, ancestor.Title)+ }+ }+ return out+}++func confluenceAPIBase(wikiBase string, isCloud bool) string {+ base := strings.TrimRight(wikiBase, "/")+ if isCloud && base != "" && !strings.HasSuffix(base, "/wiki") {+ base += "/wiki"+ }+ return base+}++func buildConfluenceDocumentID(baseURL, contentURL string, isCloud bool) string {+ finalBase := strings.TrimRight(baseURL, "/") + "/"+ if isCloud && !strings.HasSuffix(finalBase, "/wiki/") {+ finalBase += "wiki/"+ }+ base, err := url.Parse(finalBase)+ if err != nil {+ return strings.TrimRight(finalBase, "/") + "/" + strings.TrimLeft(contentURL, "/")+ }+ ref, err := url.Parse(strings.TrimLeft(contentURL, "/"))+ if err != nil {+ return strings.TrimRight(finalBase, "/") + "/" + strings.TrimLeft(contentURL, "/")+ }+ return base.ResolveReference(ref).String()+}++func confluenceCQLPath(cql, expand string, limit int) string {+ values := url.Values{}+ values.Set("cql", cql)+ if expand != "" {+ values.Set("expand", expand)+ }+ values.Set("limit", fmt.Sprint(limit))+ return "rest/api/content/search?" + values.Encode()+}++func confluenceCQLQuote(value string) string {+ value = strings.ReplaceAll(value, `\`, `\\`)+ return strings.ReplaceAll(value, "'", "\\'")+}++func confluenceCQLTime(value time.Time) string {+ return value.UTC().Format("2006-01-02 15:04")+}++func inConfluenceWindow(updatedAt time.Time, start *time.Time, end time.Time) bool {+ if !end.IsZero() && updatedAt.After(end) {+ return false+ }+ if start != nil && !updatedAt.After(*start) {+ return false+ }+ return true+}++func parseConfluenceTime(value string) time.Time {+ value = strings.TrimSpace(value)+ if value == "" {+ return time.Time{}+ }+ layouts := []string{+ time.RFC3339Nano,+ time.RFC3339,+ "2006-01-02T15:04:05.000-0700",+ "2006-01-02T15:04:05-0700",+ "2006-01-02 15:04:05",+ }+ for _, layout := range layouts {+ if parsed, err := time.Parse(layout, value); err == nil {+ return parsed.UTC()+ }+ }+ return time.Time{}+}++func confluenceSemanticIdentifier(space string, ancestors []string, title string, counts map[string]int) string {+ title = firstNonEmpty(title, "Untitled")+ parts := make([]string, 0, len(ancestors)+2)+ if space != "" {+ parts = append(parts, space)+ }+ parts = append(parts, ancestors...)+ parts = append(parts, title)+ fullPath := strings.Join(parts, " / ")+ counts[title]+++ if counts[title] > 1 && fullPath != "" {+ return fullPath+ }+ return title+}++func confluenceHTMLText(raw string) string {+ raw = strings.TrimSpace(raw)+ if raw == "" {+ return ""+ }+ root, err := xhtml.Parse(strings.NewReader(raw))+ if err != nil {+ return html.UnescapeString(raw)+ }+ var buffer bytes.Buffer+ var walk func(*xhtml.Node)+ walk = func(node *xhtml.Node) {+ if node.Type == xhtml.ElementNode {+ switch strings.ToLower(node.Data) {+ case "script", "style":+ return+ case "br", "p", "div", "tr", "li", "table", "h1", "h2", "h3", "h4", "h5", "h6":+ buffer.WriteByte('\n')+ }+ }+ if node.Type == xhtml.TextNode {+ text := strings.TrimSpace(html.UnescapeString(node.Data))+ if text != "" {+ if buffer.Len() > 0 {+ buffer.WriteByte(' ')+ }+ buffer.WriteString(text)+ }+ }+ for child := node.FirstChild; child != nil; child = child.NextSibling {+ walk(child)+ }+ if node.Type == xhtml.ElementNode {+ switch strings.ToLower(node.Data) {+ case "p", "div", "tr", "li", "table", "h1", "h2", "h3", "h4", "h5", "h6":+ buffer.WriteByte('\n')+ }+ }+ }+ walk(root)+ text := strings.ReplaceAll(buffer.String(), "\r\n", "\n")+ text = confluenceWhitespaceRE.ReplaceAllString(text, " ")+ text = strings.ReplaceAll(text, " \n", "\n")+ text = strings.ReplaceAll(text, "\n ", "\n")+ text = confluenceNewlineRE.ReplaceAllString(text, "\n\n")+ return strings.TrimSpace(text)+}++func metadataOrNil(metadata map[string]any) map[string]any {+ if len(metadata) == 0 {+ return nil+ }+ return metadata+}++func isConfluenceAttachmentAccepted(attachment confluenceContent) bool {+ mediaType := strings.ToLower(attachment.Metadata.MediaType)+ if strings.HasPrefix(mediaType, "image/") {+ switch mediaType {+ case "image/jpeg", "image/jpg", "image/png", "image/gif", "image/bmp", "image/tiff", "image/webp":+ return true+ default:+ return false+ }+ }+ ext := confluenceAttachmentExtension(attachment.Title)+ if _, ok := webdavTextExtensions[ext]; ok {+ return true+ }+ if _, ok := webdavDocumentExtensions[ext]; ok {+ return true+ }+ return false+}++func confluenceAttachmentExtension(title string) string {+ ext := strings.ToLower(path.Ext(title))+ if ext == "" {+ return ".unknown"+ }+ return ext+}++func confluenceAttachmentThreshold() int64 {+ if raw := strings.TrimSpace(os.Getenv("CONFLUENCE_CONNECTOR_ATTACHMENT_SIZE_THRESHOLD")); raw != "" {+ if parsed := configInt(raw, defaultConfluenceAttachmentThreshold); parsed > 0 {+ return int64(parsed)+ }+ }+ return defaultConfluenceAttachmentThreshold+}++func confluenceValidationError(err error) error {+ var statusErr *confluenceStatusError+ if errors.As(err, &statusErr) {+ switch statusErr.status {+ case http.StatusUnauthorized:+ return fmt.Errorf("invalid or expired Confluence credentials")+ case http.StatusForbidden:+ return fmt.Errorf("insufficient permissions to access Confluence resources")+ }+ }+ return err+}++type confluenceStatusError struct {+ status int+ body string+}++func (e *confluenceStatusError) Error() string {+ return fmt.Sprintf("Confluence request failed with status %d: %s", e.status, strings.TrimSpace(e.body))+}++var confluencePageExpansionFields = []string{+ "body.storage.value",+ "body.view.value",+ "space",+ "ancestors",+ "version",+ "metadata.labels",+}++var confluenceAttachmentExpansionFields = []string{+ "metadata",+ "metadata.labels",+ "extensions",+ "version",+ "space",+}diff --git a/internal/syncer/connector/confluence_test.go b/internal/syncer/connector/confluence_test.gonew file mode 100644index 0000000000..7cc887df42--- /dev/null+++ b/internal/syncer/connector/confluence_test.go@@ -0,0 +1,328 @@+package connector++import (+ "context"+ "encoding/base64"+ "encoding/json"+ "errors"+ "io"+ "net/http"+ "net/http/httptest"+ "net/url"+ "strings"+ "testing"+ "time"+)++func TestConfluenceConnectorSyncPagesCommentsAndAttachments(t *testing.T) {+ server := newConfluenceFixtureServer(t, http.StatusOK)+ defer server.Close()++ connector := newTestConfluenceConnector(t, server.URL, map[string]any{+ "index_mode": "page",+ "page_id": "123",+ "index_recursively": true,+ "batch_size": 10,+ })+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ WindowEnd: confluenceTestTime(t, "2026-01-03T00:00:00Z"),+ })+ if err != nil {+ t.Fatalf("OpenSync() error = %v", err)+ }+ defer func() { _ = session.Close() }()++ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch() error = %v", err)+ }+ if len(batch.Documents) != 2 {+ t.Fatalf("len(documents) = %d, want 2", len(batch.Documents))+ }++ page := batch.Documents[0]+ if page.SourceID != server.URL+"/wiki/display/SPACE/Page" {+ t.Fatalf("page SourceID = %q", page.SourceID)+ }+ if page.SemanticIdentifier != "Page" {+ t.Fatalf("page SemanticIdentifier = %q", page.SemanticIdentifier)+ }+ if !strings.Contains(string(page.Blob), "Hello Confluence") || !strings.Contains(string(page.Blob), "Comment text") {+ t.Fatalf("page blob did not include page and comment text: %q", string(page.Blob))+ }+ if page.Extension != ".txt" {+ t.Fatalf("page Extension = %q", page.Extension)+ }+ if page.Metadata["space"] != "Engineering" {+ t.Fatalf("page metadata = %#v", page.Metadata)+ }+ if page.Fingerprint != contentFingerprint(page.Blob) {+ t.Fatalf("page fingerprint mismatch")+ }++ attachment := batch.Documents[1]+ if attachment.SourceID != server.URL+"/wiki/download/attachments/file.txt?version=1" {+ t.Fatalf("attachment SourceID = %q", attachment.SourceID)+ }+ if attachment.Extension != ".txt" {+ t.Fatalf("attachment Extension = %q", attachment.Extension)+ }+ if string(attachment.Blob) != "attachment bytes" {+ t.Fatalf("attachment Blob = %q", string(attachment.Blob))+ }+ if attachment.Metadata["parent_page_id"] != page.SourceID {+ t.Fatalf("attachment metadata = %#v", attachment.Metadata)+ }+ if batch.Checkpoint == nil || batch.Checkpoint.SourceID != attachment.SourceID {+ t.Fatalf("checkpoint = %#v", batch.Checkpoint)+ }++ _, err = session.NextBatch(context.Background())+ if !errors.Is(err, io.EOF) {+ t.Fatalf("second NextBatch() error = %v, want EOF", err)+ }+}++func TestConfluenceConnectorResume(t *testing.T) {+ server := newConfluenceFixtureServer(t, http.StatusOK)+ defer server.Close()++ connector := newTestConfluenceConnector(t, server.URL, map[string]any{"batch_size": 1})+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ WindowEnd: confluenceTestTime(t, "2026-01-03T00:00:00Z"),+ })+ if err != nil {+ t.Fatalf("OpenSync() error = %v", err)+ }+ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("first NextBatch() error = %v", err)+ }++ resumed, err := connector.OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ WindowEnd: confluenceTestTime(t, "2026-01-03T00:00:00Z"),+ Resume: first.Checkpoint,+ })+ if err != nil {+ t.Fatalf("resumed OpenSync() error = %v", err)+ }+ batch, err := resumed.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("resumed NextBatch() error = %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].Extension != ".txt" || batch.Documents[0].SourceID == first.Documents[0].SourceID {+ t.Fatalf("resumed batch = %#v first = %#v", batch.Documents, first.Documents)+ }+}++func TestConfluenceConnectorPrune(t *testing.T) {+ server := newConfluenceFixtureServer(t, http.StatusOK)+ defer server.Close()++ connector := newTestConfluenceConnector(t, server.URL, nil)+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune() error = %v", err)+ }+ defer func() { _ = session.Close() }()++ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch() error = %v", err)+ }+ got := map[string]bool{}+ for _, doc := range batch.Documents {+ got[doc.SourceID] = true+ }+ for _, want := range []string{+ server.URL + "/wiki/display/SPACE/Page",+ server.URL + "/wiki/download/attachments/file.txt?version=1",+ } {+ if !got[want] {+ t.Fatalf("missing prune source id %q in %#v", want, batch.Documents)+ }+ }+}++func TestConfluenceConnectorValidateMapsUnauthorized(t *testing.T) {+ server := newConfluenceFixtureServer(t, http.StatusUnauthorized)+ defer server.Close()++ connector := newTestConfluenceConnector(t, server.URL, nil)+ err := connector.ValidateConnectorSetting(context.Background(), nil)+ if err == nil || !strings.Contains(err.Error(), "invalid or expired") {+ t.Fatalf("ValidateConnectorSetting() error = %v", err)+ }+}++func newTestConfluenceConnector(t *testing.T, wikiBase string, overrides map[string]any) *ConfluenceConnector {+ t.Helper()+ config := map[string]any{+ "wiki_base": wikiBase,+ "is_cloud": true,+ "batch_size": 10,+ "credentials": map[string]any{+ "confluence_username": "[email protected]",+ "confluence_access_token": "token",+ },+ }+ for key, value := range overrides {+ config[key] = value+ }+ connector, err := NewConfluenceConnector(config)+ if err != nil {+ t.Fatalf("NewConfluenceConnector() error = %v", err)+ }+ return connector+}++func newConfluenceFixtureServer(t *testing.T, validateStatus int) *httptest.Server {+ t.Helper()+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if validateStatus != http.StatusOK && strings.HasPrefix(r.URL.Path, "/wiki/rest/api/space") {+ http.Error(w, "unauthorized", validateStatus)+ return+ }+ if !confluenceHasBasicAuth(r, "[email protected]", "token") {+ http.Error(w, "bad auth", http.StatusUnauthorized)+ return+ }+ switch {+ case r.URL.Path == "/wiki/rest/api/space":+ writeConfluenceJSON(t, w, map[string]any{"results": []map[string]any{{"key": "SPACE", "name": "Engineering"}}})+ case r.URL.Path == "/wiki/rest/api/space/SPACE":+ writeConfluenceJSON(t, w, map[string]any{"key": "SPACE", "name": "Engineering"})+ case r.URL.Path == "/wiki/rest/api/content/search":+ cql := r.URL.Query().Get("cql")+ switch {+ case strings.Contains(cql, "type=page"):+ writeConfluenceJSON(t, w, map[string]any{"results": []map[string]any{confluenceFixturePage()}})+ case strings.Contains(cql, "type=comment"):+ writeConfluenceJSON(t, w, map[string]any{"results": []map[string]any{confluenceFixtureComment()}})+ case strings.Contains(cql, "type=attachment"):+ writeConfluenceJSON(t, w, map[string]any{"results": []map[string]any{confluenceFixtureAttachment()}})+ default:+ t.Errorf("Unexpected CQL: %q", cql)+ http.Error(w, "unexpected cql", http.StatusInternalServerError)+ }+ case r.URL.Path == "/wiki/download/attachments/file.txt":+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte("attachment bytes"))+ default:+ t.Errorf("unexpected request %s", r.URL.String())+ http.Error(w, "unexpected request", http.StatusInternalServerError)+ }+ }))+ return server+}++func confluenceHasBasicAuth(r *http.Request, username, password string) bool {+ const prefix = "Basic "+ raw := r.Header.Get("Authorization")+ if !strings.HasPrefix(raw, prefix) {+ return false+ }+ decoded, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(raw, prefix))+ if err != nil {+ return false+ }+ return string(decoded) == username+":"+password+}++func writeConfluenceJSON(t *testing.T, w http.ResponseWriter, payload any) {+ t.Helper()+ w.Header().Set("Content-Type", "application/json")+ if err := json.NewEncoder(w).Encode(payload); err != nil {+ t.Errorf("encode fixture: %v", err)+ }+}++func confluenceFixturePage() map[string]any {+ return map[string]any{+ "id": 123,+ "type": "page",+ "title": "Page",+ "body": map[string]any{"storage": map[string]any{"value": "<p>Hello <strong>Confluence</strong></p>"}},+ "space": map[string]any{"key": "SPACE", "name": "Engineering"},+ "ancestors": []map[string]any{+ {"title": "Root"},+ },+ "version": map[string]any{"when": "2026-01-01T10:00:00Z"},+ "metadata": map[string]any{"labels": map[string]any{"results": []map[string]any{{"name": "docs"}}}},+ "_links": map[string]any{"webui": "/display/SPACE/Page"},+ }+}++func confluenceFixtureComment() map[string]any {+ return map[string]any{+ "id": 456,+ "type": "comment",+ "title": "Comment",+ "body": map[string]any{"storage": map[string]any{"value": "<p>Comment text</p>"}},+ "version": map[string]any{"when": "2026-01-01T11:00:00Z"},+ "_links": map[string]any{"webui": "/display/SPACE/Page?focusedCommentId=c1"},+ }+}++func confluenceFixtureAttachment() map[string]any {+ return map[string]any{+ "id": 789,+ "type": "attachment",+ "title": "file.txt",+ "space": map[string]any{"key": "SPACE", "name": "Engineering"},+ "metadata": map[string]any{+ "mediaType": "text/plain",+ "labels": map[string]any{"results": []map[string]any{{"name": "file"}}},+ },+ "extensions": map[string]any{"fileSize": 16},+ "version": map[string]any{"when": "2026-01-01T12:00:00Z"},+ "_links": map[string]any{+ "webui": "/download/attachments/file.txt?version=1",+ "download": "/download/attachments/file.txt?version=1",+ },+ }+}++func confluenceTestTime(t *testing.T, value string) time.Time {+ t.Helper()+ parsed, err := time.Parse(time.RFC3339, value)+ if err != nil {+ t.Fatalf("parse test time %q: %v", value, err)+ }+ return parsed+}++func TestBuildConfluenceDocumentIDAddsWikiForCloud(t *testing.T) {+ got := buildConfluenceDocumentID("https://example.atlassian.net", "/display/SPACE/Page", true)+ want := "https://example.atlassian.net/wiki/display/SPACE/Page"+ if got != want {+ t.Fatalf("buildConfluenceDocumentID() = %q, want %q", got, want)+ }++ got = buildConfluenceDocumentID("https://example.atlassian.net/wiki", "/display/SPACE/Page", true)+ if got != want {+ t.Fatalf("buildConfluenceDocumentID() with wiki base = %q, want %q", got, want)+ }+}++func TestConfluenceCQLPathEscapesQuery(t *testing.T) {+ raw := confluenceCQLPath("type=page and space='A B'", "version", 10)+ parsed, err := url.Parse(raw)+ if err != nil {+ t.Fatalf("parse path: %v", err)+ }+ if parsed.Query().Get("cql") != "type=page and space='A B'" {+ t.Fatalf("cql = %q", parsed.Query().Get("cql"))+ }+}++func TestConfluenceCQLQuoteEscapesBackslashes(t *testing.T) {+ got := confluenceCQLQuote(`path\name's`)+ want := `path\\name\'s`+ if got != want {+ t.Fatalf("confluenceCQLQuote() = %q, want %q", got, want)+ }+}