Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(go-syncer): add Slack connector with checkpoint resume (#18603)

mkaaad05af33b21464
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.go
index 75a32a21fc..42ca0d17d9 100644
--- a/internal/syncer/connector/builtin.go
+++ b/internal/syncer/connector/builtin.go
@@ -45,6 +45,7 @@ func RegisterBuiltIns(registry *Registry) {
registerBuiltIn(registry, "moodle", NewMoodleConnector)
registerBuiltIn(registry, "mysql", NewMySQLConnector)
registerBuiltIn(registry, "postgresql", NewPostgreSQLConnector)
+ registerBuiltIn(registry, "slack", NewSlackConnector)
registerBuiltIn(registry, "discord", NewDiscordConnector)
registerBuiltIn(registry, "webdav", NewWebDAVConnector)
}
diff --git a/internal/syncer/connector/slack.go b/internal/syncer/connector/slack.go
new file mode 100644
index 0000000000..f0f8157b86
--- /dev/null
+++ b/internal/syncer/connector/slack.go
@@ -0,0 +1,1144 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "regexp"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+)
+
+// Slack connector constants.
+const (
+ slackDefaultBaseURL = "https://slack.com/api"
+ slackDefaultBatchSize = 2
+ slackRequestTimeout = 60 * time.Second
+ slackPageSize = 900
+ slackMaxRetryDelay = 60 * time.Second
+ slackSnippetLength = 50
+ slackMaxChannelsToLog = 20
+ slackUSLACKBOTUser = "USLACKBOT"
+)
+
+// Retry/backoff knobs for Slack API calls. They are package variables so tests
+// can shrink the delays.
+var (
+ slackRetryTries = 3
+ slackRetryBaseDelay = time.Second
+ slackRetryBackoff = 2
+)
+
+// slackMaxResponseSize caps Slack API JSON responses. Package variable so
+// tests can shrink it.
+var slackMaxResponseSize int64 = 32 * 1024 * 1024
+
+// SlackConnector reads public and private channel messages and threads from a
+// Slack workspace through the Slack Web API with a bot token.
+type SlackConnector struct {
+ token string
+ channels []string
+ channelRegexEnabled bool
+ batchSize int
+ baseURL string
+ client *http.Client
+ userCache map[string]string
+}
+
+// NewSlackConnector creates a Slack connector from connector config.
+func NewSlackConnector(config map[string]any) (*SlackConnector, error) {
+ credentials := configAnyMap(config["credentials"])
+ token := strings.TrimSpace(stringConfig(credentials["slack_bot_token"]))
+
+ batchSize := configInt(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])), slackDefaultBatchSize)
+ if batchSize <= 0 {
+ batchSize = slackDefaultBatchSize
+ }
+
+ return &SlackConnector{
+ token: token,
+ channels: normalizeSlackChannelNames(coerceDiscordStringList(config["channels"])),
+ channelRegexEnabled: configBoolDefault(config["channel_regex_enabled"], false),
+ batchSize: batchSize,
+ baseURL: strings.TrimRight(slackDefaultBaseURL, "/"),
+ client: &http.Client{Timeout: slackRequestTimeout},
+ userCache: map[string]string{},
+ }, nil
+}
+
+// Validate validates Slack connector settings, credentials, and connectivity.
+func (c *SlackConnector) Validate(ctx context.Context) error {
+ if c == nil {
+ return &ConnectorValidationError{Message: "Slack connector is nil"}
+ }
+ if c.token == "" {
+ return &ConnectorMissingCredentialError{Message: "Slack connector requires 'slack_bot_token' in credentials"}
+ }
+ if c.batchSize <= 0 {
+ return &ConnectorValidationError{Message: "Slack connector batch_size must be a positive integer"}
+ }
+
+ // 1) Validate workspace connection.
+ var auth slackAuthTestResponse
+ if err := c.callAPI(ctx, "auth.test", url.Values{}, &auth); err != nil {
+ return slackValidationError(err)
+ }
+
+ // 2) Confirm listing channels works.
+ var list slackListChannelsResponse
+ if err := c.callAPI(ctx, "conversations.list", url.Values{
+ "limit": {"1"},
+ "types": {"public_channel"},
+ }, &list); err != nil {
+ return slackValidationError(err)
+ }
+
+ // 3) Confirm users:read scope is available (required to resolve senders).
+ var user slackUserInfoResponse
+ if err := c.callAPI(ctx, "users.info", url.Values{"user": {slackUSLACKBOTUser}}, &user); err != nil {
+ if errors.As(err, new(*slackUserNotFoundError)) {
+ return nil
+ }
+ return slackValidationError(err)
+ }
+ return nil
+}
+
+// ValidateConnectorSetting validates Slack settings from an unsaved config.
+func (c *SlackConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
+ defer cancel()
+ return c.Validate(ctx)
+}
+
+// OpenSync opens one Slack sync session.
+func (c *SlackConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
+ if err := c.Validate(ctx); err != nil {
+ return nil, err
+ }
+ channels, err := c.resolvedChannels(ctx)
+ if err != nil {
+ return nil, err
+ }
+ session := &slackSyncSession{
+ connector: c,
+ request: request,
+ channels: channels,
+ batchSize: c.batchSize,
+ seenThreads: map[string]struct{}{},
+ }
+ if request.Resume != nil {
+ session.applyResume(request.Resume)
+ }
+ return session, nil
+}
+
+// OpenPrune opens one complete Slack prune snapshot session.
+func (c *SlackConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {
+ if err := c.Validate(ctx); err != nil {
+ return nil, err
+ }
+ channels, err := c.resolvedChannels(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &slackPruneSession{connector: c, channels: channels, batchSize: c.batchSize}, nil
+}
+
+// ---------------------------------------------------------------------------
+// Slack Web API client
+// ---------------------------------------------------------------------------
+
+// slackAPIResponse is embedded in every Slack API response payload.
+type slackAPIResponse struct {
+ OK bool `json:"ok"`
+ Error string `json:"error"`
+}
+
+// slackResponseMetadata carries the opaque pagination cursor.
+type slackResponseMetadata struct {
+ NextCursor string `json:"next_cursor"`
+}
+
+type slackAuthTestResponse struct {
+ slackAPIResponse
+ URL string `json:"url"`
+ Team string `json:"team"`
+ User string `json:"user"`
+ TeamID string `json:"team_id"`
+ UserID string `json:"user_id"`
+}
+
+type slackChannel struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+ IsArchived bool `json:"is_archived"`
+ IsMember bool `json:"is_member"`
+ IsPrivate bool `json:"is_private"`
+}
+
+type slackListChannelsResponse struct {
+ slackAPIResponse
+ Channels []slackChannel `json:"channels"`
+ ResponseMetadata slackResponseMetadata `json:"response_metadata"`
+}
+
+type slackMessage struct {
+ TS string `json:"ts"`
+ ThreadTS string `json:"thread_ts"`
+ User string `json:"user"`
+ Text string `json:"text"`
+ Subtype string `json:"subtype"`
+ BotID string `json:"bot_id"`
+ AppID string `json:"app_id"`
+ BotProfile struct {
+ Name string `json:"name"`
+ } `json:"bot_profile"`
+}
+
+type slackHistoryResponse struct {
+ slackAPIResponse
+ Messages []slackMessage `json:"messages"`
+ HasMore bool `json:"has_more"`
+ ResponseMetadata slackResponseMetadata `json:"response_metadata"`
+}
+
+type slackUserInfoResponse struct {
+ slackAPIResponse
+ User struct {
+ ID string `json:"id"`
+ RealName string `json:"real_name"`
+ Profile struct {
+ DisplayName string `json:"display_name"`
+ RealName string `json:"real_name"`
+ } `json:"profile"`
+ } `json:"user"`
+}
+
+// slackTransientError marks a retryable Slack API failure.
+type slackTransientError struct {
+ retryAfter time.Duration
+ rateLimited bool
+ message string
+}
+
+func (e *slackTransientError) Error() string { return e.message }
+
+// slackUserNotFoundError is returned when users.info cannot resolve a user.
+type slackUserNotFoundError struct{ message string }
+
+func (e *slackUserNotFoundError) Error() string { return e.message }
+
+// slackAPIError wraps a classified Slack API error with the raw error code so
+// callers can branch on the specific Slack failure.
+type slackAPIError struct {
+ code string
+ err error
+}
+
+func (e *slackAPIError) Error() string { return e.err.Error() }
+func (e *slackAPIError) Unwrap() error { return e.err }
+
+// callAPI sends one Slack Web API call, retrying transient failures.
+func (c *SlackConnector) callAPI(ctx context.Context, method string, params url.Values, out any) error {
+ return c.retrySlack(ctx, func() error {
+ return c.callAPINoRetry(ctx, method, params, out)
+ })
+}
+
+func (c *SlackConnector) callAPINoRetry(ctx context.Context, method string, params url.Values, out any) error {
+ endpoint := c.baseURL + "/" + method
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, strings.NewReader(params.Encode()))
+ if err != nil {
+ return err
+ }
+ req.Header.Set("Authorization", "Bearer "+c.token)
+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+ resp, err := c.client.Do(req)
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(io.LimitReader(resp.Body, slackMaxResponseSize+1))
+ if err != nil {
+ return err
+ }
+ if int64(len(body)) > slackMaxResponseSize {
+ return &ConnectorValidationError{Message: fmt.Sprintf("Slack API response from %s exceeds maximum size of %d bytes", method, slackMaxResponseSize)}
+ }
+ if resp.StatusCode == http.StatusTooManyRequests {
+ return &slackTransientError{
+ retryAfter: slackRetryAfterSeconds(resp.Header.Get("Retry-After"), body),
+ rateLimited: true,
+ message: "Slack API rate limited",
+ }
+ }
+
+ var probe slackAPIResponse
+ if err := json.Unmarshal(body, &probe); err != nil {
+ if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
+ return &ConnectorValidationError{Message: fmt.Sprintf("Slack API %s request failed with status %d", method, resp.StatusCode)}
+ }
+ return fmt.Errorf("Slack API %s returned invalid JSON: %w", method, err)
+ }
+ if !probe.OK {
+ return classifySlackAPIError(method, probe.Error, body)
+ }
+ if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
+ return &ConnectorValidationError{Message: fmt.Sprintf("Slack API %s request failed with status %d", method, resp.StatusCode)}
+ }
+ return json.Unmarshal(body, out)
+}
+
+func classifySlackAPIError(method, apiError string, body []byte) error {
+ var err error
+ switch apiError {
+ case "ratelimited":
+ err = &slackTransientError{
+ retryAfter: slackRetryAfterSeconds("", body),
+ rateLimited: true,
+ message: "Slack API rate limited",
+ }
+ case "invalid_auth":
+ err = &ConnectorValidationError{Message: "Invalid Slack bot token (invalid_auth)."}
+ case "not_authed":
+ err = &ConnectorMissingCredentialError{Message: "Invalid or expired Slack bot token (not_authed)."}
+ case "missing_scope":
+ err = &ConnectorValidationError{Message: slackMissingScopeMessage(method, body)}
+ case "not_in_channel":
+ err = &ConnectorValidationError{Message: "Slack bot is not a member of the channel (not_in_channel)."}
+ case "channel_not_found":
+ err = &ConnectorValidationError{Message: "Slack channel not found (channel_not_found)."}
+ case "user_not_found":
+ err = &slackUserNotFoundError{message: "Slack user not found (user_not_found)."}
+ case "method_not_supported_for_channel_type":
+ err = &ConnectorValidationError{Message: "Slack method is not supported for this channel type."}
+ default:
+ if apiError == "" {
+ err = &ConnectorValidationError{Message: fmt.Sprintf("Slack API %s request failed without an error code", method)}
+ } else {
+ err = &ConnectorValidationError{Message: fmt.Sprintf("Slack API %s request failed: %s", method, apiError)}
+ }
+ }
+ if apiError == "" {
+ return err
+ }
+ return &slackAPIError{code: apiError, err: err}
+}
+
+// slackMissingScopeMessage builds an actionable message for a missing_scope
+// Slack error, naming the scope the call needed. It falls back to a generic
+// message when Slack does not include the needed scope in the response.
+func slackMissingScopeMessage(method string, body []byte) string {
+ var payload struct {
+ Needed any `json:"needed"`
+ }
+ msg := fmt.Sprintf("Slack bot token lacks the necessary OAuth scope to access %s (missing_scope)", method)
+ if err := json.Unmarshal(body, &payload); err != nil {
+ return msg + "."
+ }
+ if needed := slackScopeNames(payload.Needed); needed != "" {
+ return fmt.Sprintf("%s. Needed: %s", msg, needed)
+ }
+ return msg + "."
+}
+
+// slackScopeNames flattens a Slack scope field, which is either a single
+// comma-separated string or a JSON array of scope names.
+func slackScopeNames(value any) string {
+ switch v := value.(type) {
+ case string:
+ return strings.TrimSpace(v)
+ case []any:
+ parts := make([]string, 0, len(v))
+ for _, item := range v {
+ if s, ok := item.(string); ok && strings.TrimSpace(s) != "" {
+ parts = append(parts, strings.TrimSpace(s))
+ }
+ }
+ return strings.Join(parts, ", ")
+ }
+ return ""
+}
+
+func slackRetryAfterSeconds(header string, body []byte) time.Duration {
+ if seconds, err := strconv.Atoi(strings.TrimSpace(header)); err == nil && seconds > 0 {
+ return time.Duration(seconds) * time.Second
+ }
+ var payload struct {
+ RetryAfter float64 `json:"retry_after"`
+ }
+ if err := json.Unmarshal(body, &payload); err == nil && payload.RetryAfter > 0 {
+ return time.Duration(payload.RetryAfter * float64(time.Second))
+ }
+ return 0
+}
+
+func (c *SlackConnector) retrySlack(ctx context.Context, fn func() error) error {
+ var lastErr error
+ for attempt := 0; attempt < slackRetryTries; attempt++ {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ err := fn()
+ if err == nil {
+ return nil
+ }
+ lastErr = err
+ if !isSlackTransient(err) || attempt+1 >= slackRetryTries {
+ break
+ }
+ delay := slackRetryBaseDelay
+ for i := 0; i < attempt; i++ {
+ delay *= time.Duration(slackRetryBackoff)
+ }
+ var transient *slackTransientError
+ if errors.As(err, &transient) && transient.retryAfter > delay {
+ delay = transient.retryAfter
+ }
+ if delay > slackMaxRetryDelay {
+ delay = slackMaxRetryDelay
+ }
+ timer := time.NewTimer(delay)
+ select {
+ case <-ctx.Done():
+ timer.Stop()
+ return ctx.Err()
+ case <-timer.C:
+ }
+ }
+ return lastErr
+}
+
+func isSlackTransient(err error) bool {
+ if err == nil {
+ return false
+ }
+ var transient *slackTransientError
+ if errors.As(err, &transient) {
+ return true
+ }
+ var urlErr *url.Error
+ return errors.As(err, &urlErr)
+}
+
+// slackValidationError treats a persistent rate limit as a warning so that
+// settings validation proceeds, matching the reference SDK behavior.
+func slackValidationError(err error) error {
+ var transient *slackTransientError
+ if errors.As(err, &transient) && transient.rateLimited {
+ return nil
+ }
+ return err
+}
+
+// ---------------------------------------------------------------------------
+// Channel and message enumeration
+// ---------------------------------------------------------------------------
+
+// resolvedChannels returns the configured channels, sorted by ID so checkpoints
+// advance deterministically.
+func (c *SlackConnector) resolvedChannels(ctx context.Context) ([]slackChannel, error) {
+ all, err := c.listChannels(ctx)
+ if err != nil {
+ return nil, err
+ }
+ filtered, err := filterSlackChannels(all, c.channels, c.channelRegexEnabled)
+ if err != nil {
+ return nil, err
+ }
+ sort.SliceStable(filtered, func(i, j int) bool {
+ return filtered[i].ID < filtered[j].ID
+ })
+ return filtered, nil
+}
+
+// listChannels lists public and private channels, falling back to public-only
+// when private channel enumeration fails.
+func (c *SlackConnector) listChannels(ctx context.Context) ([]slackChannel, error) {
+ channels, err := c.collectChannels(ctx, []string{"public_channel", "private_channel"})
+ if err != nil {
+ channels, err = c.collectChannels(ctx, []string{"public_channel"})
+ }
+ return channels, err
+}
+
+func (c *SlackConnector) collectChannels(ctx context.Context, channelTypes []string) ([]slackChannel, error) {
+ var channels []slackChannel
+ cursor := ""
+ for {
+ params := url.Values{
+ "limit": {strconv.Itoa(slackPageSize)},
+ "types": {strings.Join(channelTypes, ",")},
+ "exclude_archived": {"true"},
+ }
+ if cursor != "" {
+ params.Set("cursor", cursor)
+ }
+ var resp slackListChannelsResponse
+ if err := c.callAPI(ctx, "conversations.list", params, &resp); err != nil {
+ return nil, err
+ }
+ channels = append(channels, resp.Channels...)
+ if resp.ResponseMetadata.NextCursor == "" {
+ return channels, nil
+ }
+ cursor = resp.ResponseMetadata.NextCursor
+ }
+}
+
+// channelMessages returns every top-level message in a channel, optionally
+// bounded by oldest/latest Slack epoch-second strings (empty means unbounded).
+func (c *SlackConnector) channelMessages(ctx context.Context, channel slackChannel, oldest, latest string) ([]slackMessage, error) {
+ if err := c.joinChannel(ctx, channel); err != nil {
+ return nil, err
+ }
+ var messages []slackMessage
+ cursor := ""
+ for {
+ params := url.Values{
+ "channel": {channel.ID},
+ "limit": {strconv.Itoa(slackPageSize)},
+ }
+ if oldest != "" {
+ params.Set("oldest", oldest)
+ }
+ if latest != "" {
+ params.Set("latest", latest)
+ }
+ if cursor != "" {
+ params.Set("cursor", cursor)
+ }
+ var resp slackHistoryResponse
+ if err := c.callAPI(ctx, "conversations.history", params, &resp); err != nil {
+ return nil, err
+ }
+ messages = append(messages, resp.Messages...)
+ if resp.ResponseMetadata.NextCursor == "" {
+ return messages, nil
+ }
+ cursor = resp.ResponseMetadata.NextCursor
+ }
+}
+
+// getThread returns the full reply thread for a channel message timestamp.
+func (c *SlackConnector) getThread(ctx context.Context, channelID, threadTS string) ([]slackMessage, error) {
+ var messages []slackMessage
+ cursor := ""
+ for {
+ params := url.Values{
+ "channel": {channelID},
+ "ts": {threadTS},
+ "limit": {strconv.Itoa(slackPageSize)},
+ }
+ if cursor != "" {
+ params.Set("cursor", cursor)
+ }
+ var resp slackHistoryResponse
+ if err := c.callAPI(ctx, "conversations.replies", params, &resp); err != nil {
+ return nil, err
+ }
+ messages = append(messages, resp.Messages...)
+ if resp.ResponseMetadata.NextCursor == "" {
+ return messages, nil
+ }
+ cursor = resp.ResponseMetadata.NextCursor
+ }
+}
+
+// errSlackChannelUnavailable marks Slack channels that the bot cannot join so
+// callers can skip them instead of failing the whole sync/prune run.
+var errSlackChannelUnavailable = errors.New("slack channel unavailable")
+
+// slackUnjoinableErrors are conversations.join failure codes that mean the bot
+// cannot join the channel.
+var slackUnjoinableErrors = map[string]struct{}{
+ "is_archived": {},
+ "method_not_supported_for_channel_type": {},
+ "channel_not_found": {},
+ "restricted_action": {},
+}
+
+// joinChannel joins a channel so the bot can read its messages. Channels the
+// bot cannot join are wrapped in errSlackChannelUnavailable so callers can
+// skip them; unrelated API errors propagate unchanged. Automatic joins are
+// limited to the configured channels selection when one is present.
+func (c *SlackConnector) joinChannel(ctx context.Context, channel slackChannel) error {
+ if channel.IsMember {
+ return nil
+ }
+ if len(c.channels) > 0 && !slackChannelSelected(channel, c.channels, c.channelRegexEnabled) {
+ return nil
+ }
+ var resp struct {
+ slackAPIResponse
+ }
+ if err := c.callAPI(ctx, "conversations.join", url.Values{"channel": {channel.ID}}, &resp); err != nil {
+ var apiErr *slackAPIError
+ if errors.As(err, &apiErr) {
+ if _, ok := slackUnjoinableErrors[apiErr.code]; ok {
+ return fmt.Errorf("%w: %v", errSlackChannelUnavailable, err)
+ }
+ }
+ return err
+ }
+ return nil
+}
+
+// slackChannelSelected reports whether a channel is part of the configured
+// channels selection (exact names, or regex patterns when enabled). An empty
+// selection matches every channel.
+func slackChannelSelected(channel slackChannel, configured []string, regexEnabled bool) bool {
+ if len(configured) == 0 {
+ return true
+ }
+ for _, pattern := range configured {
+ if regexEnabled {
+ re, err := regexp.Compile("^" + pattern + "$")
+ if err != nil {
+ continue
+ }
+ if re.MatchString(channel.Name) {
+ return true
+ }
+ continue
+ }
+ if channel.Name == pattern {
+ return true
+ }
+ }
+ return false
+}
+
+// displayName resolves a Slack user ID to a display name, caching results.
+func (c *SlackConnector) displayName(ctx context.Context, userID string) (string, error) {
+ if userID == "" {
+ return "", nil
+ }
+ if name, ok := c.userCache[userID]; ok {
+ return name, nil
+ }
+ var resp slackUserInfoResponse
+ if err := c.callAPI(ctx, "users.info", url.Values{"user": {userID}}, &resp); err != nil {
+ var notFound *slackUserNotFoundError
+ if errors.As(err, &notFound) {
+ return "", nil
+ }
+ return "", err
+ }
+ name := resp.User.Profile.DisplayName
+ if name == "" {
+ name = resp.User.RealName
+ }
+ if name == "" {
+ name = resp.User.Profile.RealName
+ }
+ if name == "" {
+ return "", nil
+ }
+ c.userCache[userID] = name
+ return name, nil
+}
+
+// filterSlackChannels narrows the workspace channels to the configured set.
+func filterSlackChannels(all []slackChannel, channels []string, regexEnabled bool) ([]slackChannel, error) {
+ if len(channels) == 0 {
+ return all, nil
+ }
+ if regexEnabled {
+ var out []slackChannel
+ for _, pattern := range channels {
+ re, err := regexp.Compile("^" + pattern + "$")
+ if err != nil {
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Invalid Slack channel regex %q", pattern)}
+ }
+ for _, channel := range all {
+ if re.MatchString(channel.Name) {
+ out = append(out, channel)
+ }
+ }
+ }
+ return out, nil
+ }
+
+ available := map[string]slackChannel{}
+ for _, channel := range all {
+ available[channel.Name] = channel
+ }
+ var missing []string
+ for _, name := range channels {
+ if _, ok := available[name]; !ok {
+ missing = append(missing, name)
+ }
+ }
+ if len(missing) > 0 {
+ names := make([]string, 0, len(available))
+ for name := range available {
+ names = append(names, name)
+ }
+ sort.Strings(names)
+ if len(names) > slackMaxChannelsToLog {
+ names = names[:slackMaxChannelsToLog]
+ }
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf(
+ "Channel(s) %s not found in workspace. Available channels: %s",
+ strings.Join(missing, ", "), strings.Join(names, ", "))}
+ }
+ out := make([]slackChannel, 0, len(channels))
+ for _, name := range channels {
+ out = append(out, available[name])
+ }
+ return out, nil
+}
+
+// ---------------------------------------------------------------------------
+// Message filtering and text cleaning
+// ---------------------------------------------------------------------------
+
+// slackDisallowedSubtypes lists message subtypes that carry no informative
+// content.
+var slackDisallowedSubtypes = map[string]struct{}{
+ "channel_join": {},
+ "channel_leave": {},
+ "channel_archive": {},
+ "channel_unarchive": {},
+ "pinned_item": {},
+ "unpinned_item": {},
+ "ekm_access_denied": {},
+ "channel_posting_permissions": {},
+ "group_join": {},
+ "group_leave": {},
+ "group_archive": {},
+ "group_unarchive": {},
+ "channel_name": {},
+}
+
+// acceptSlackMessage reports whether a message should be indexed.
+func acceptSlackMessage(message slackMessage) bool {
+ if message.BotID != "" || message.AppID != "" {
+ if message.BotProfile.Name == "DanswerBot Testing" {
+ return true
+ }
+ return false
+ }
+ if message.Subtype != "" {
+ if _, ok := slackDisallowedSubtypes[message.Subtype]; ok {
+ return false
+ }
+ }
+ return true
+}
+
+var (
+ slackUserMentionRe = regexp.MustCompile(`<@(.*?)>`)
+ slackChannelMentionRe = regexp.MustCompile(`<#(.*?)\|(.*?)>`)
+ slackSpecialCatchallRe = regexp.MustCompile(`<!([^|]+)\|([^>]+)>`)
+)
+
+// indexClean normalizes Slack markdown links and mentions into readable text.
+func (c *SlackConnector) indexClean(ctx context.Context, text string) (string, error) {
+ cleaned, err := c.replaceUserIDsWithNames(ctx, text)
+ if err != nil {
+ return "", err
+ }
+ cleaned = slackReplaceChannelsBasic(cleaned)
+ cleaned = slackReplaceSpecialMentions(cleaned)
+ cleaned = slackReplaceSpecialCatchall(cleaned)
+ return cleaned, nil
+}
+
+func (c *SlackConnector) replaceUserIDsWithNames(ctx context.Context, text string) (string, error) {
+ matches := slackUserMentionRe.FindAllStringSubmatch(text, -1)
+ for _, match := range matches {
+ if len(match) < 2 {
+ continue
+ }
+ userID := match[1]
+ name, err := c.displayName(ctx, userID)
+ if err != nil {
+ return "", err
+ }
+ if name != "" {
+ text = strings.ReplaceAll(text, match[0], "@"+name)
+ }
+ }
+ return text, nil
+}
+
+func slackReplaceChannelsBasic(text string) string {
+ return slackChannelMentionRe.ReplaceAllString(text, "#$2")
+}
+
+func slackReplaceSpecialMentions(text string) string {
+ text = strings.ReplaceAll(text, "<!channel>", "@channel")
+ text = strings.ReplaceAll(text, "<!here>", "@here")
+ text = strings.ReplaceAll(text, "<!everyone>", "@everyone")
+ return text
+}
+
+func slackReplaceSpecialCatchall(text string) string {
+ return slackSpecialCatchallRe.ReplaceAllString(text, "$2")
+}
+
+// ---------------------------------------------------------------------------
+// Document building
+// ---------------------------------------------------------------------------
+
+func slackDocID(channelID, ts string) string {
+ return channelID + "__" + ts
+}
+
+// threadToDocument flattens a filtered thread into one text document. docTS is
+// the thread root timestamp used as the document identity.
+func (c *SlackConnector) threadToDocument(ctx context.Context, channel slackChannel, thread []slackMessage, docTS string) (*SourceDocument, error) {
+ senderName := "Unknown"
+ if len(thread) > 0 {
+ if name, err := c.displayName(ctx, thread[0].User); err != nil {
+ return nil, err
+ } else if name != "" {
+ senderName = name
+ }
+ }
+
+ cleaned := make([]string, 0, len(thread))
+ for _, message := range thread {
+ text, err := c.indexClean(ctx, message.Text)
+ if err != nil {
+ return nil, err
+ }
+ cleaned = append(cleaned, text)
+ }
+
+ first := ""
+ if len(cleaned) > 0 {
+ first = cleaned[0]
+ }
+ snippet := first
+ if runes := []rune(first); len(runes) > slackSnippetLength {
+ snippet = strings.TrimRight(string(runes[:slackSnippetLength]), " \t\n") + "..."
+ }
+
+ semantic := fmt.Sprintf("%s in #%s: %s", senderName, channel.Name, snippet)
+ semantic = strings.ReplaceAll(semantic, "\n", " ")
+
+ content := strings.Join(cleaned, "\n\n")
+ blob := []byte(content)
+
+ var updatedAt time.Time
+ for _, message := range thread {
+ if ts := slackMessageTime(message.TS); ts.After(updatedAt) {
+ updatedAt = ts
+ }
+ }
+
+ doc := &SourceDocument{
+ SourceID: slackDocID(channel.ID, docTS),
+ SemanticIdentifier: semantic,
+ Extension: ".txt",
+ Blob: blob,
+ UpdatedAt: updatedAt,
+ SizeBytes: int64(len(blob)),
+ Metadata: map[string]any{"Channel": channel.Name},
+ }
+ return doc, nil
+}
+
+func slackMessageTime(ts string) time.Time {
+ seconds, err := strconv.ParseFloat(ts, 64)
+ if err != nil {
+ return time.Time{}
+ }
+ whole := int64(seconds)
+ nanos := int64((seconds - float64(whole)) * 1e9)
+ return time.Unix(whole, nanos).UTC()
+}
+
+func normalizeSlackChannelNames(channels []string) []string {
+ out := make([]string, 0, len(channels))
+ for _, name := range channels {
+ name = strings.TrimSpace(strings.TrimPrefix(name, "#"))
+ if name != "" {
+ out = append(out, name)
+ }
+ }
+ return out
+}
+
+// ---------------------------------------------------------------------------
+// Sync session with checkpoint resume
+// ---------------------------------------------------------------------------
+
+type slackSyncSession struct {
+ connector *SlackConnector
+ request SyncRequest
+ channels []slackChannel
+ batchSize int
+ seenThreads map[string]struct{}
+
+ resumeAfterChannelID string
+ channelIndex int
+ pending []SourceDocument
+ pendingCheckpoint *SyncCheckpoint
+ done bool
+}
+
+// NextBatch returns the next Slack document batch.
+func (s *slackSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
+ for {
+ if s.done {
+ return SyncBatch{}, io.EOF
+ }
+ if len(s.pending) == 0 {
+ if s.channelIndex >= len(s.channels) {
+ s.done = true
+ return SyncBatch{}, io.EOF
+ }
+ channel := s.channels[s.channelIndex]
+ s.channelIndex++
+ if channel.ID <= s.resumeAfterChannelID {
+ continue
+ }
+ documents, err := s.channelDocuments(ctx, channel)
+ if err != nil {
+ return SyncBatch{}, err
+ }
+ if len(documents) == 0 {
+ continue
+ }
+ s.pending = documents
+ updatedAt := documents[len(documents)-1].UpdatedAt
+ checkpoint := &SyncCheckpoint{
+ Cursor: fmt.Sprintf("slack_channel_%s", channel.ID),
+ SourceID: fmt.Sprintf("slack_channel_%s", channel.ID),
+ UpdatedAt: &updatedAt,
+ }
+ // The checkpoint only advances once the whole channel is flushed so
+ // an interrupted run re-processes a partially committed channel.
+ s.pendingCheckpoint = checkpoint
+ }
+ n := s.batchSize
+ if n > len(s.pending) {
+ n = len(s.pending)
+ }
+ chunk := s.pending[:n]
+ s.pending = s.pending[n:]
+ var checkpoint *SyncCheckpoint
+ if len(s.pending) == 0 && s.pendingCheckpoint != nil {
+ checkpoint = s.pendingCheckpoint
+ s.pendingCheckpoint = nil
+ }
+ return SyncBatch{Documents: chunk, Checkpoint: checkpoint}, nil
+ }
+}
+
+// Close closes the Slack sync session.
+func (s *slackSyncSession) Close() error {
+ return nil
+}
+
+func (s *slackSyncSession) applyResume(checkpoint *SyncCheckpoint) {
+ sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
+ const prefix = "slack_channel_"
+ if !strings.HasPrefix(sourceID, prefix) {
+ return
+ }
+ channelID := strings.TrimPrefix(sourceID, prefix)
+ if channelID == "" {
+ return
+ }
+ s.resumeAfterChannelID = channelID
+}
+
+// channelDocuments builds the documents for one channel.
+func (s *slackSyncSession) channelDocuments(ctx context.Context, channel slackChannel) ([]SourceDocument, error) {
+ s.seenThreads = map[string]struct{}{}
+ oldest, latest := "", ""
+ if !s.request.FromBeginning {
+ if s.request.WindowStart != nil {
+ oldest = strconv.FormatInt(s.request.WindowStart.Unix(), 10)
+ }
+ if !s.request.WindowEnd.IsZero() {
+ latest = strconv.FormatInt(s.request.WindowEnd.Unix(), 10)
+ }
+ }
+ messages, err := s.connector.channelMessages(ctx, channel, oldest, latest)
+ if err != nil {
+ if errors.Is(err, errSlackChannelUnavailable) {
+ return nil, nil
+ }
+ return nil, err
+ }
+
+ var documents []SourceDocument
+ for _, message := range messages {
+ doc, err := s.processMessage(ctx, channel, message)
+ if err != nil {
+ return nil, err
+ }
+ if doc != nil {
+ documents = append(documents, *doc)
+ }
+ }
+ return documents, nil
+}
+
+// processMessage converts one top-level message into a document, fetching and
+// merging its reply thread when present.
+func (s *slackSyncSession) processMessage(ctx context.Context, channel slackChannel, message slackMessage) (*SourceDocument, error) {
+ threadOrMessageTS := message.ThreadTS
+ if threadOrMessageTS == "" {
+ threadOrMessageTS = message.TS
+ }
+ if _, seen := s.seenThreads[threadOrMessageTS]; seen {
+ return nil, nil
+ }
+ s.seenThreads[threadOrMessageTS] = struct{}{}
+
+ if message.ThreadTS != "" {
+ thread, err := s.connector.getThread(ctx, channel.ID, message.ThreadTS)
+ if err != nil {
+ return nil, err
+ }
+ filtered := filterSlackMessages(thread)
+ if len(filtered) == 0 {
+ return nil, nil
+ }
+ return s.connector.threadToDocument(ctx, channel, filtered, threadOrMessageTS)
+ }
+ if !acceptSlackMessage(message) {
+ return nil, nil
+ }
+ return s.connector.threadToDocument(ctx, channel, []slackMessage{message}, threadOrMessageTS)
+}
+
+func filterSlackMessages(messages []slackMessage) []slackMessage {
+ filtered := make([]slackMessage, 0, len(messages))
+ for _, message := range messages {
+ if acceptSlackMessage(message) {
+ filtered = append(filtered, message)
+ }
+ }
+ return filtered
+}
+
+type slackPruneSession struct {
+ connector *SlackConnector
+ channels []slackChannel
+ batchSize int
+
+ channelIndex int
+ pending []SlimDocument
+ done bool
+}
+
+// NextBatch returns the next Slack prune snapshot batch, paging channels
+// lazily so the whole workspace is never collected in memory at once.
+func (s *slackPruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {
+ for {
+ if s.done {
+ return PruneBatch{}, io.EOF
+ }
+ if len(s.pending) == 0 {
+ if s.channelIndex >= len(s.channels) {
+ s.done = true
+ return PruneBatch{}, io.EOF
+ }
+ channel := s.channels[s.channelIndex]
+ s.channelIndex++
+ documents, err := s.connector.pruneChannelDocuments(ctx, channel)
+ if err != nil {
+ return PruneBatch{}, err
+ }
+ if len(documents) == 0 {
+ continue
+ }
+ s.pending = documents
+ }
+ n := s.batchSize
+ if n > len(s.pending) {
+ n = len(s.pending)
+ }
+ chunk := s.pending[:n]
+ s.pending = s.pending[n:]
+ return PruneBatch{Documents: chunk}, nil
+ }
+}
+
+// Close closes the Slack prune session.
+func (s *slackPruneSession) Close() error {
+ return nil
+}
+
+// pruneChannelDocuments returns the slim snapshot documents for one channel.
+func (c *SlackConnector) pruneChannelDocuments(ctx context.Context, channel slackChannel) ([]SlimDocument, error) {
+ messages, err := c.channelMessages(ctx, channel, "", "")
+ if err != nil {
+ if errors.Is(err, errSlackChannelUnavailable) {
+ return nil, nil
+ }
+ return nil, err
+ }
+ return c.slackPruneDocuments(ctx, channel, messages)
+}
+
+// slackPruneDocuments maps channel history to slim snapshot documents. Each
+// thread is resolved with the same full-thread view and acceptance logic as
+// sync (conversations.replies), so prune identities match sync documents.
+func (c *SlackConnector) slackPruneDocuments(ctx context.Context, channel slackChannel, messages []slackMessage) ([]SlimDocument, error) {
+ seen := map[string]struct{}{}
+ var documents []SlimDocument
+ for _, message := range messages {
+ rootTS := slackThreadRootTS(message)
+ if _, ok := seen[rootTS]; ok {
+ continue
+ }
+ seen[rootTS] = struct{}{}
+ accepted := false
+ if message.ThreadTS != "" {
+ thread, err := c.getThread(ctx, channel.ID, message.ThreadTS)
+ if err != nil {
+ return nil, err
+ }
+ accepted = len(filterSlackMessages(thread)) > 0
+ } else {
+ accepted = acceptSlackMessage(message)
+ }
+ if accepted {
+ documents = append(documents, SlimDocument{SourceID: slackDocID(channel.ID, rootTS)})
+ }
+ }
+ return documents, nil
+}
+
+// slackThreadRootTS returns the root timestamp of a message's thread, falling
+// back to the message's own timestamp for top-level messages.
+func slackThreadRootTS(message slackMessage) string {
+ if message.ThreadTS != "" {
+ return message.ThreadTS
+ }
+ return message.TS
+}
diff --git a/internal/syncer/connector/slack_test.go b/internal/syncer/connector/slack_test.go
new file mode 100644
index 0000000000..d0a1be22c1
--- /dev/null
+++ b/internal/syncer/connector/slack_test.go
@@ -0,0 +1,1180 @@
+package connector
+
+import (
+ "context"
+ "errors"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+func withSlackTestHooks(t *testing.T) {
+ t.Helper()
+ origTries := slackRetryTries
+ origBaseDelay := slackRetryBaseDelay
+ origBackoff := slackRetryBackoff
+ slackRetryTries = 2
+ slackRetryBaseDelay = time.Millisecond
+ slackRetryBackoff = 2
+ t.Cleanup(func() {
+ slackRetryTries = origTries
+ slackRetryBaseDelay = origBaseDelay
+ slackRetryBackoff = origBackoff
+ })
+}
+
+func mustSlackConnector(t *testing.T, serverURL string, extra map[string]any) *SlackConnector {
+ t.Helper()
+ config := map[string]any{
+ "batch_size": 2,
+ "credentials": map[string]any{
+ "slack_bot_token": "xoxb-test",
+ },
+ }
+ for key, value := range extra {
+ config[key] = value
+ }
+ connector, err := NewSlackConnector(config)
+ if err != nil {
+ t.Fatalf("NewSlackConnector: %v", err)
+ }
+ if serverURL != "" {
+ connector.baseURL = strings.TrimRight(serverURL, "/") + "/api"
+ }
+ return connector
+}
+
+type slackTestFixtures struct {
+ authError string
+ channelsJSON string
+ historyByChannel map[string]string
+ repliesByKey map[string]string
+ usersByID map[string]string
+
+ listCalls *atomic.Int64
+ historyCalls *atomic.Int64
+ joinCalls *atomic.Int64
+ usersCalls *atomic.Int64
+ historyParams []url.Values
+}
+
+func newTestSlackServer(t *testing.T, fixtures *slackTestFixtures) *httptest.Server {
+ t.Helper()
+ mux := http.NewServeMux()
+ server := httptest.NewServer(mux)
+ t.Cleanup(server.Close)
+
+ mux.HandleFunc("/api/auth.test", func(w http.ResponseWriter, r *http.Request) {
+ if fixtures.authError != "" {
+ writeSlackError(w, fixtures.authError)
+ return
+ }
+ w.Write([]byte(`{"ok":true,"url":"https://example.slack.com/","team":"Test","user":"bot","team_id":"T1","user_id":"U1"}`))
+ })
+ mux.HandleFunc("/api/conversations.list", func(w http.ResponseWriter, r *http.Request) {
+ if fixtures.listCalls != nil {
+ fixtures.listCalls.Add(1)
+ }
+ body := fixtures.channelsJSON
+ if body == "" {
+ body = `{"ok":true,"channels":[],"response_metadata":{"next_cursor":""}}`
+ }
+ w.Write([]byte(body))
+ })
+ mux.HandleFunc("/api/conversations.history", func(w http.ResponseWriter, r *http.Request) {
+ if fixtures.historyCalls != nil {
+ fixtures.historyCalls.Add(1)
+ }
+ if err := r.ParseForm(); err != nil {
+ http.Error(w, "bad form", http.StatusBadRequest)
+ return
+ }
+ if fixtures.historyParams != nil {
+ params := url.Values{}
+ for key, values := range r.Form {
+ params[key] = append([]string{}, values...)
+ }
+ fixtures.historyParams = append(fixtures.historyParams, params)
+ }
+ body := fixtures.historyByChannel[r.Form.Get("channel")]
+ if body == "" {
+ body = `{"ok":true,"messages":[],"has_more":false,"response_metadata":{"next_cursor":""}}`
+ }
+ w.Write([]byte(body))
+ })
+ mux.HandleFunc("/api/conversations.replies", func(w http.ResponseWriter, r *http.Request) {
+ if err := r.ParseForm(); err != nil {
+ http.Error(w, "bad form", http.StatusBadRequest)
+ return
+ }
+ key := r.Form.Get("channel") + ":" + r.Form.Get("ts")
+ body := fixtures.repliesByKey[key]
+ if body == "" {
+ body = `{"ok":true,"messages":[],"has_more":false,"response_metadata":{"next_cursor":""}}`
+ }
+ w.Write([]byte(body))
+ })
+ mux.HandleFunc("/api/conversations.join", func(w http.ResponseWriter, r *http.Request) {
+ if fixtures.joinCalls != nil {
+ fixtures.joinCalls.Add(1)
+ }
+ w.Write([]byte(`{"ok":true,"channel":{"id":"C1"}}`))
+ })
+ mux.HandleFunc("/api/users.info", func(w http.ResponseWriter, r *http.Request) {
+ if fixtures.usersCalls != nil {
+ fixtures.usersCalls.Add(1)
+ }
+ if err := r.ParseForm(); err != nil {
+ http.Error(w, "bad form", http.StatusBadRequest)
+ return
+ }
+ body := fixtures.usersByID[r.Form.Get("user")]
+ if body == "" {
+ body = `{"ok":false,"error":"user_not_found"}`
+ }
+ w.Write([]byte(body))
+ })
+ return server
+}
+
+func writeSlackError(w http.ResponseWriter, apiError string) {
+ w.Write([]byte(`{"ok":false,"error":"` + apiError + `"}`))
+}
+
+func slackHistoryBody(messages ...string) string {
+ return `{"ok":true,"messages":[` + strings.Join(messages, ",") + `],"has_more":false,"response_metadata":{"next_cursor":""}}`
+}
+
+const (
+ slackChannelsTwo = `{"ok":true,"channels":[
+ {"id":"C1","name":"general","is_archived":false,"is_member":true,"is_private":false},
+ {"id":"C2","name":"random","is_archived":false,"is_member":false,"is_private":false}
+ ],"response_metadata":{"next_cursor":""}}`
+
+ slackMsgPlain = `{"type":"message","ts":"1700000000.000001","user":"U123","text":"Hello <!channel> from <@U456>"}`
+ slackMsgThreadParent = `{"type":"message","ts":"1700000001.000001","user":"U123","text":"Thread root","thread_ts":"1700000001.000001"}`
+ slackMsgBot = `{"type":"message","ts":"1700000003.000001","user":"U123","text":"bot noise","bot_id":"B1","bot_profile":{"name":"Some Bot"}}`
+ slackMsgJoin = `{"type":"message","subtype":"channel_join","ts":"1700000004.000001","user":"U123","text":"<@U123> has joined the channel"}`
+ slackMsgDanswerBot = `{"type":"message","ts":"1700000005.000001","user":"U123","text":"danswer note","bot_id":"B2","bot_profile":{"name":"DanswerBot Testing"}}`
+ slackMsgNormal = `{"type":"message","ts":"1700000006.000001","user":"U123","text":"A normal message"}`
+)
+
+func slackThreadBody() string {
+ return `{"ok":true,"messages":[
+ {"type":"message","ts":"1700000001.000001","user":"U123","text":"Thread root","thread_ts":"1700000001.000001"},
+ {"type":"message","ts":"1700000002.000002","user":"U456","text":"A reply"}
+ ],"has_more":false,"response_metadata":{"next_cursor":""}}`
+}
+
+func slackUserAlice() string {
+ return `{"ok":true,"user":{"id":"U123","real_name":"Alice","profile":{"display_name":"Alice","real_name":"Alice"}}}`
+}
+
+func slackUserBob() string {
+ return `{"ok":true,"user":{"id":"U456","real_name":"Bob","profile":{"display_name":"","real_name":"Bob"}}}`
+}
+
+func collectSlackDocuments(t *testing.T, session SyncSession) []SourceDocument {
+ t.Helper()
+ var documents []SourceDocument
+ for {
+ batch, err := session.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ return documents
+ }
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ documents = append(documents, batch.Documents...)
+ }
+}
+
+func TestSlackConnectorValidate(t *testing.T) {
+ withSlackTestHooks(t)
+ fixtures := &slackTestFixtures{
+ channelsJSON: slackChannelsTwo,
+ usersByID: map[string]string{"USLACKBOT": slackUserAlice()},
+ listCalls: &atomic.Int64{},
+ }
+ server := newTestSlackServer(t, fixtures)
+ connector := mustSlackConnector(t, server.URL, nil)
+
+ if err := connector.Validate(context.Background()); err != nil {
+ t.Fatalf("Validate: %v", err)
+ }
+ if fixtures.listCalls == nil || fixtures.listCalls.Load() == 0 {
+ t.Fatalf("conversations.list was not called during validation")
+ }
+}
+
+func TestSlackConnectorValidateMissingToken(t *testing.T) {
+ connector := mustSlackConnector(t, "", map[string]any{"credentials": map[string]any{}})
+ err := connector.Validate(context.Background())
+ if err == nil {
+ t.Fatalf("expected missing credential error")
+ }
+ var missing *ConnectorMissingCredentialError
+ if !errors.As(err, &missing) {
+ t.Fatalf("error = %T, want *ConnectorMissingCredentialError", err)
+ }
+}
+
+func TestSlackConnectorValidateNil(t *testing.T) {
+ var connector *SlackConnector
+ if err := connector.Validate(context.Background()); err == nil {
+ t.Fatalf("expected error for nil connector")
+ }
+}
+
+func TestSlackConnectorValidateInvalidAuth(t *testing.T) {
+ withSlackTestHooks(t)
+ fixtures := &slackTestFixtures{authError: "invalid_auth"}
+ server := newTestSlackServer(t, fixtures)
+ connector := mustSlackConnector(t, server.URL, nil)
+
+ err := connector.Validate(context.Background())
+ if err == nil || !strings.Contains(err.Error(), "Invalid Slack bot token") {
+ t.Fatalf("Validate error = %v, want invalid bot token error", err)
+ }
+}
+
+func TestSlackConnectorValidateMissingScope(t *testing.T) {
+ withSlackTestHooks(t)
+ fixtures := &slackTestFixtures{
+ channelsJSON: slackChannelsTwo,
+ usersByID: map[string]string{"USLACKBOT": `{"ok":false,"error":"missing_scope"}`},
+ }
+ server := newTestSlackServer(t, fixtures)
+ connector := mustSlackConnector(t, server.URL, nil)
+
+ err := connector.Validate(context.Background())
+ if err == nil || !strings.Contains(strings.ToLower(err.Error()), "scope") {
+ t.Fatalf("Validate error = %v, want missing scope error", err)
+ }
+}
+
+func TestSlackConnectorMissingScopeMessageNamesNeededScope(t *testing.T) {
+ withSlackTestHooks(t)
+ body := []byte(`{"ok":false,"error":"missing_scope","needed":"channels:read","provided":"channels:write"}`)
+ err := classifySlackAPIError("conversations.list", "missing_scope", body)
+ var valErr *ConnectorValidationError
+ if !errors.As(err, &valErr) {
+ t.Fatalf("error = %T, want *ConnectorValidationError", err)
+ }
+ if !strings.Contains(valErr.Message, "channels:read") {
+ t.Fatalf("message = %q, want it to contain %q", valErr.Message, "channels:read")
+ }
+ if strings.Contains(valErr.Message, "channels:write") {
+ t.Fatalf("message = %q, want it not to mention provided scopes", valErr.Message)
+ }
+}
+
+func TestSlackConnectorMissingScopeMessageFallsBackWithoutNeeded(t *testing.T) {
+ withSlackTestHooks(t)
+ body := []byte(`{"ok":false,"error":"missing_scope"}`)
+ err := classifySlackAPIError("users.info", "missing_scope", body)
+ var valErr *ConnectorValidationError
+ if !errors.As(err, &valErr) {
+ t.Fatalf("error = %T, want *ConnectorValidationError", err)
+ }
+ if !strings.Contains(valErr.Message, "users.info") || !strings.Contains(valErr.Message, "missing_scope") {
+ t.Fatalf("message = %q, want generic missing scope message", valErr.Message)
+ }
+}
+
+func TestSlackConnectorValidateRateLimitedProceeds(t *testing.T) {
+ withSlackTestHooks(t)
+ fixtures := &slackTestFixtures{authError: "ratelimited"}
+ server := newTestSlackServer(t, fixtures)
+ connector := mustSlackConnector(t, server.URL, nil)
+
+ if err := connector.Validate(context.Background()); err != nil {
+ t.Fatalf("Validate should proceed after rate limit, got %v", err)
+ }
+}
+
+func TestSlackConnectorOpenSyncBuildsThreadAndMessageDocs(t *testing.T) {
+ withSlackTestHooks(t)
+ joinCalls := &atomic.Int64{}
+ fixtures := &slackTestFixtures{
+ channelsJSON: slackChannelsTwo,
+ historyByChannel: map[string]string{
+ "C1": slackHistoryBody(slackMsgThreadParent, slackMsgPlain),
+ },
+ repliesByKey: map[string]string{"C1:1700000001.000001": slackThreadBody()},
+ usersByID: map[string]string{"U123": slackUserAlice(), "U456": slackUserBob()},
+ joinCalls: joinCalls,
+ }
+ server := newTestSlackServer(t, fixtures)
+ connector := mustSlackConnector(t, server.URL, nil)
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ defer session.Close()
+ documents := collectSlackDocuments(t, session)
+ if len(documents) != 2 {
+ t.Fatalf("documents = %d, want 2", len(documents))
+ }
+
+ threadDoc := documents[0]
+ if threadDoc.SourceID != "C1__1700000001.000001" {
+ t.Fatalf("thread SourceID = %q", threadDoc.SourceID)
+ }
+ if threadDoc.SemanticIdentifier != "Alice in #general: Thread root" {
+ t.Fatalf("thread semantic = %q", threadDoc.SemanticIdentifier)
+ }
+ if threadDoc.Extension != ".txt" {
+ t.Fatalf("thread extension = %q", threadDoc.Extension)
+ }
+ if string(threadDoc.Blob) != "Thread root\n\nA reply" {
+ t.Fatalf("thread blob = %q", string(threadDoc.Blob))
+ }
+ if threadDoc.UpdatedAt.Unix() != 1700000002 {
+ t.Fatalf("thread updated_at = %v", threadDoc.UpdatedAt)
+ }
+ if threadDoc.Metadata["Channel"] != "general" {
+ t.Fatalf("thread metadata = %v", threadDoc.Metadata)
+ }
+
+ plainDoc := documents[1]
+ if plainDoc.SourceID != "C1__1700000000.000001" {
+ t.Fatalf("plain SourceID = %q", plainDoc.SourceID)
+ }
+ if plainDoc.SemanticIdentifier != "Alice in #general: Hello @channel from @Bob" {
+ t.Fatalf("plain semantic = %q", plainDoc.SemanticIdentifier)
+ }
+ if string(plainDoc.Blob) != "Hello @channel from @Bob" {
+ t.Fatalf("plain blob = %q", string(plainDoc.Blob))
+ }
+
+ if joinCalls.Load() != 1 {
+ t.Fatalf("conversations.join calls = %d, want 1 (C2 is not a member)", joinCalls.Load())
+ }
+}
+
+func TestSlackConnectorOpenSyncWindowBounds(t *testing.T) {
+ withSlackTestHooks(t)
+ fixtures := &slackTestFixtures{
+ channelsJSON: slackChannelsTwo,
+ historyByChannel: map[string]string{"C1": slackHistoryBody(slackMsgPlain)},
+ historyParams: []url.Values{},
+ }
+ server := newTestSlackServer(t, fixtures)
+ connector := mustSlackConnector(t, server.URL, nil)
+
+ windowStart := time.Unix(1700000000, 0).UTC()
+ windowEnd := time.Unix(1700000100, 0).UTC()
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ WindowStart: &windowStart,
+ WindowEnd: windowEnd,
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ defer session.Close()
+ collectSlackDocuments(t, session)
+
+ // Both C1 and C2 are enumerated; C1 is processed first.
+ if len(fixtures.historyParams) != 2 {
+ t.Fatalf("history calls = %d, want 2", len(fixtures.historyParams))
+ }
+ if got := fixtures.historyParams[0].Get("oldest"); got != "1700000000" {
+ t.Fatalf("oldest = %q, want 1700000000", got)
+ }
+ if got := fixtures.historyParams[0].Get("latest"); got != "1700000100" {
+ t.Fatalf("latest = %q, want 1700000100", got)
+ }
+}
+
+func TestSlackConnectorOpenSyncFromBeginningHasNoWindow(t *testing.T) {
+ withSlackTestHooks(t)
+ fixtures := &slackTestFixtures{
+ channelsJSON: slackChannelsTwo,
+ historyByChannel: map[string]string{"C1": slackHistoryBody(slackMsgPlain)},
+ historyParams: []url.Values{},
+ }
+ server := newTestSlackServer(t, fixtures)
+ connector := mustSlackConnector(t, server.URL, nil)
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ defer session.Close()
+ collectSlackDocuments(t, session)
+
+ if got := fixtures.historyParams[0].Get("oldest"); got != "" {
+ t.Fatalf("oldest = %q, want empty for full sync", got)
+ }
+ if got := fixtures.historyParams[0].Get("latest"); got != "" {
+ t.Fatalf("latest = %q, want empty for full sync", got)
+ }
+}
+
+func TestSlackConnectorOpenSyncFiltersMessages(t *testing.T) {
+ withSlackTestHooks(t)
+ fixtures := &slackTestFixtures{
+ channelsJSON: slackChannelsTwo,
+ historyByChannel: map[string]string{"C1": slackHistoryBody(slackMsgBot, slackMsgJoin, slackMsgDanswerBot, slackMsgNormal)},
+ usersByID: map[string]string{"U123": slackUserAlice()},
+ }
+ server := newTestSlackServer(t, fixtures)
+ connector := mustSlackConnector(t, server.URL, nil)
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ defer session.Close()
+ documents := collectSlackDocuments(t, session)
+ if len(documents) != 2 {
+ t.Fatalf("documents = %d, want 2 (Danswer bot and normal message)", len(documents))
+ }
+ if documents[0].SourceID != "C1__1700000005.000001" {
+ t.Fatalf("first doc SourceID = %q", documents[0].SourceID)
+ }
+ if documents[1].SourceID != "C1__1700000006.000001" {
+ t.Fatalf("second doc SourceID = %q", documents[1].SourceID)
+ }
+}
+
+func TestSlackConnectorOpenSyncChannelSelection(t *testing.T) {
+ withSlackTestHooks(t)
+ historyCalls := &atomic.Int64{}
+ fixtures := &slackTestFixtures{
+ channelsJSON: slackChannelsTwo,
+ historyByChannel: map[string]string{"C1": slackHistoryBody(slackMsgPlain)},
+ usersByID: map[string]string{"U123": slackUserAlice()},
+ historyCalls: historyCalls,
+ }
+ server := newTestSlackServer(t, fixtures)
+ connector := mustSlackConnector(t, server.URL, map[string]any{"channels": "general"})
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ defer session.Close()
+ documents := collectSlackDocuments(t, session)
+ if len(documents) != 1 || documents[0].SourceID != "C1__1700000000.000001" {
+ t.Fatalf("documents = %+v", documents)
+ }
+ if historyCalls.Load() != 1 {
+ t.Fatalf("history calls = %d, want 1", historyCalls.Load())
+ }
+}
+
+func TestSlackConnectorOpenSyncChannelNotFound(t *testing.T) {
+ withSlackTestHooks(t)
+ fixtures := &slackTestFixtures{channelsJSON: slackChannelsTwo}
+ server := newTestSlackServer(t, fixtures)
+ connector := mustSlackConnector(t, server.URL, map[string]any{"channels": "missing"})
+
+ _, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err == nil || !strings.Contains(err.Error(), "not found") {
+ t.Fatalf("OpenSync error = %v, want channel not found", err)
+ }
+}
+
+func TestSlackConnectorOpenSyncRegexChannels(t *testing.T) {
+ withSlackTestHooks(t)
+ historyCalls := &atomic.Int64{}
+ fixtures := &slackTestFixtures{
+ channelsJSON: slackChannelsTwo,
+ historyByChannel: map[string]string{"C1": slackHistoryBody(slackMsgPlain)},
+ usersByID: map[string]string{"U123": slackUserAlice()},
+ historyCalls: historyCalls,
+ }
+ server := newTestSlackServer(t, fixtures)
+ connector := mustSlackConnector(t, server.URL, map[string]any{
+ "channels": "gen.*",
+ "channel_regex_enabled": true,
+ })
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ defer session.Close()
+ documents := collectSlackDocuments(t, session)
+ if len(documents) != 1 || documents[0].SourceID != "C1__1700000000.000001" {
+ t.Fatalf("documents = %+v", documents)
+ }
+}
+
+func TestSlackConnectorOpenSyncResumesFromCheckpoint(t *testing.T) {
+ withSlackTestHooks(t)
+ historyCalls := &atomic.Int64{}
+ fixtures := &slackTestFixtures{
+ channelsJSON: `{"ok":true,"channels":[
+ {"id":"C1","name":"general","is_member":true},
+ {"id":"C2","name":"random","is_member":true},
+ {"id":"C3","name":"dev","is_member":true}
+ ],"response_metadata":{"next_cursor":""}}`,
+ historyByChannel: map[string]string{
+ "C1": slackHistoryBody(slackMsgPlain),
+ "C3": slackHistoryBody(slackMsgNormal),
+ },
+ usersByID: map[string]string{"U123": slackUserAlice()},
+ historyCalls: historyCalls,
+ }
+ server := newTestSlackServer(t, fixtures)
+ connector := mustSlackConnector(t, server.URL, nil)
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{
+ Cursor: "slack_channel_C2",
+ SourceID: "slack_channel_C2",
+ },
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ defer session.Close()
+ documents := collectSlackDocuments(t, session)
+ if len(documents) != 1 || documents[0].SourceID != "C3__1700000006.000001" {
+ t.Fatalf("documents = %+v, want only channel C3", documents)
+ }
+ if historyCalls.Load() != 1 {
+ t.Fatalf("history calls = %d, want 1 (only C3)", historyCalls.Load())
+ }
+}
+
+func TestSlackConnectorOpenSyncCheckpointAdvancesPerChannel(t *testing.T) {
+ withSlackTestHooks(t)
+ fixtures := &slackTestFixtures{
+ channelsJSON: slackChannelsTwo,
+ historyByChannel: map[string]string{
+ "C1": slackHistoryBody(slackMsgPlain, slackMsgNormal),
+ "C2": slackHistoryBody(slackMsgThreadParent),
+ },
+ repliesByKey: map[string]string{"C2:1700000001.000001": slackThreadBody()},
+ usersByID: map[string]string{"U123": slackUserAlice()},
+ }
+ server := newTestSlackServer(t, fixtures)
+ connector := mustSlackConnector(t, server.URL, map[string]any{"batch_size": 1})
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ defer session.Close()
+
+ var checkpoints []string
+ for {
+ batch, err := session.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) == 0 {
+ t.Fatalf("empty batch")
+ }
+ if batch.Checkpoint != nil {
+ checkpoints = append(checkpoints, batch.Checkpoint.Cursor)
+ }
+ }
+ // batch_size=1: C1 emits two batches (no checkpoint on the first), C2 emits
+ // one; the checkpoint advances only on the last batch of each channel.
+ if len(checkpoints) != 2 || checkpoints[0] != "slack_channel_C1" || checkpoints[1] != "slack_channel_C2" {
+ t.Fatalf("checkpoints = %v, want [slack_channel_C1 slack_channel_C2]", checkpoints)
+ }
+}
+
+func TestSlackConnectorOpenPruneSlimSnapshot(t *testing.T) {
+ withSlackTestHooks(t)
+ fixtures := &slackTestFixtures{
+ channelsJSON: slackChannelsTwo,
+ historyByChannel: map[string]string{"C1": slackHistoryBody(slackMsgPlain, slackMsgBot)},
+ }
+ server := newTestSlackServer(t, fixtures)
+ connector := mustSlackConnector(t, server.URL, nil)
+
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{TaskID: "t1"})
+ if err != nil {
+ t.Fatalf("OpenPrune: %v", err)
+ }
+ defer session.Close()
+
+ var slim []SlimDocument
+ for {
+ batch, err := session.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ slim = append(slim, batch.Documents...)
+ }
+ if len(slim) != 1 || slim[0].SourceID != "C1__1700000000.000001" {
+ t.Fatalf("slim documents = %+v", slim)
+ }
+}
+
+func TestSlackConnectorOpenPruneThreadRootIdentity(t *testing.T) {
+ withSlackTestHooks(t)
+ history := slackHistoryBody(
+ slackMsgThreadParent, // root: ts == thread_ts == 1700000001.000001
+ `{"type":"message","ts":"1700000002.000002","user":"U456","text":"A reply","thread_ts":"1700000001.000001"}`,
+ slackMsgPlain,
+ )
+ fixtures := &slackTestFixtures{
+ channelsJSON: slackChannelsTwo,
+ historyByChannel: map[string]string{"C1": history},
+ repliesByKey: map[string]string{"C1:1700000001.000001": slackThreadBody()},
+ }
+ server := newTestSlackServer(t, fixtures)
+ connector := mustSlackConnector(t, server.URL, nil)
+
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{TaskID: "t1"})
+ if err != nil {
+ t.Fatalf("OpenPrune: %v", err)
+ }
+ defer session.Close()
+
+ var slim []SlimDocument
+ for {
+ batch, err := session.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ slim = append(slim, batch.Documents...)
+ }
+ // The thread is emitted once under its root timestamp; the reply and plain
+ // message must not leak their own timestamps.
+ want := []string{"C1__1700000001.000001", "C1__1700000000.000001"}
+ if len(slim) != len(want) {
+ t.Fatalf("slim documents = %+v, want %v", slim, want)
+ }
+ for i, doc := range slim {
+ if doc.SourceID != want[i] {
+ t.Fatalf("slim[%d].SourceID = %q, want %q", i, doc.SourceID, want[i])
+ }
+ }
+}
+
+func TestSlackConnectorOpenPruneBotRootThread(t *testing.T) {
+ withSlackTestHooks(t)
+ history := slackHistoryBody(
+ `{"type":"message","ts":"1700000001.000001","user":"U123","text":"bot root","thread_ts":"1700000001.000001","bot_id":"B1","bot_profile":{"name":"Some Bot"}}`,
+ `{"type":"message","ts":"1700000002.000002","user":"U456","text":"A reply","thread_ts":"1700000001.000001"}`,
+ )
+ fixtures := &slackTestFixtures{
+ channelsJSON: slackChannelsTwo,
+ historyByChannel: map[string]string{"C1": history},
+ repliesByKey: map[string]string{
+ "C1:1700000001.000001": `{"ok":true,"messages":[
+ {"type":"message","ts":"1700000001.000001","user":"U123","text":"bot root","thread_ts":"1700000001.000001","bot_id":"B1","bot_profile":{"name":"Some Bot"}},
+ {"type":"message","ts":"1700000002.000002","user":"U456","text":"A reply","thread_ts":"1700000001.000001"}
+ ],"has_more":false,"response_metadata":{"next_cursor":""}}`,
+ },
+ }
+ server := newTestSlackServer(t, fixtures)
+ connector := mustSlackConnector(t, server.URL, nil)
+
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{TaskID: "t1"})
+ if err != nil {
+ t.Fatalf("OpenPrune: %v", err)
+ }
+ defer session.Close()
+
+ var slim []SlimDocument
+ for {
+ batch, err := session.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ slim = append(slim, batch.Documents...)
+ }
+ // A thread with an accepted reply is pruned under the root timestamp even
+ // when the root message itself is filtered out.
+ if len(slim) != 1 || slim[0].SourceID != "C1__1700000001.000001" {
+ t.Fatalf("slim documents = %+v, want [C1__1700000001.000001]", slim)
+ }
+}
+
+func TestSlackConnectorOpenPrunePagesChannelsLazily(t *testing.T) {
+ withSlackTestHooks(t)
+ historyCalls := &atomic.Int64{}
+ fixtures := &slackTestFixtures{
+ channelsJSON: slackChannelsTwo,
+ historyByChannel: map[string]string{"C1": slackHistoryBody(slackMsgPlain)},
+ historyCalls: historyCalls,
+ }
+ server := newTestSlackServer(t, fixtures)
+ connector := mustSlackConnector(t, server.URL, nil)
+
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{TaskID: "t1"})
+ if err != nil {
+ t.Fatalf("OpenPrune: %v", err)
+ }
+ defer session.Close()
+ if historyCalls.Load() != 0 {
+ t.Fatalf("history calls after OpenPrune = %d, want 0 (lazy paging)", historyCalls.Load())
+ }
+
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if historyCalls.Load() == 0 {
+ t.Fatalf("history calls after NextBatch = 0, want lazy channel fetch")
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "C1__1700000000.000001" {
+ t.Fatalf("batch documents = %+v", batch.Documents)
+ }
+}
+
+func TestSlackConnectorOpenPruneResolvesRepliesFromThreadAPI(t *testing.T) {
+ withSlackTestHooks(t)
+ // The bot root is not accepted, and history carries no replies. Only
+ // conversations.replies reveals the accepted reply, so prune must resolve
+ // the thread like sync does instead of trusting history alone.
+ history := slackHistoryBody(
+ `{"type":"message","ts":"1700000001.000001","user":"U123","text":"bot root","thread_ts":"1700000001.000001","bot_id":"B1","bot_profile":{"name":"Some Bot"}}`,
+ )
+ fixtures := &slackTestFixtures{
+ channelsJSON: slackChannelsTwo,
+ historyByChannel: map[string]string{"C1": history},
+ repliesByKey: map[string]string{
+ "C1:1700000001.000001": `{"ok":true,"messages":[
+ {"type":"message","ts":"1700000001.000001","user":"U123","text":"bot root","thread_ts":"1700000001.000001","bot_id":"B1","bot_profile":{"name":"Some Bot"}},
+ {"type":"message","ts":"1700000002.000002","user":"U456","text":"A reply","thread_ts":"1700000001.000001"}
+ ],"has_more":false,"response_metadata":{"next_cursor":""}}`,
+ },
+ }
+ server := newTestSlackServer(t, fixtures)
+ connector := mustSlackConnector(t, server.URL, nil)
+
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{TaskID: "t1"})
+ if err != nil {
+ t.Fatalf("OpenPrune: %v", err)
+ }
+ defer session.Close()
+
+ var slim []SlimDocument
+ for {
+ batch, err := session.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ slim = append(slim, batch.Documents...)
+ }
+ if len(slim) != 1 || slim[0].SourceID != "C1__1700000001.000001" {
+ t.Fatalf("slim documents = %+v, want [C1__1700000001.000001]", slim)
+ }
+}
+
+func TestSlackConnectorOpenPrunePropagatesThreadResolutionError(t *testing.T) {
+ withSlackTestHooks(t)
+ mux := http.NewServeMux()
+ mux.HandleFunc("/api/auth.test", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{"ok":true}`))
+ })
+ mux.HandleFunc("/api/conversations.list", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(slackChannelsTwo))
+ })
+ mux.HandleFunc("/api/conversations.join", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{"ok":true}`))
+ })
+ mux.HandleFunc("/api/conversations.history", func(w http.ResponseWriter, r *http.Request) {
+ if r.FormValue("channel") != "C1" {
+ w.Write([]byte(`{"ok":true,"messages":[],"has_more":false,"response_metadata":{"next_cursor":""}}`))
+ return
+ }
+ w.Write([]byte(slackHistoryBody(slackMsgThreadParent)))
+ })
+ mux.HandleFunc("/api/conversations.replies", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{"ok":false,"error":"invalid_auth"}`))
+ })
+ mux.HandleFunc("/api/users.info", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(slackUserAlice()))
+ })
+ server := httptest.NewServer(mux)
+ t.Cleanup(server.Close)
+
+ connector := mustSlackConnector(t, server.URL, nil)
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{TaskID: "t1"})
+ if err != nil {
+ t.Fatalf("OpenPrune: %v", err)
+ }
+ defer session.Close()
+
+ _, err = session.NextBatch(context.Background())
+ if err == nil {
+ t.Fatalf("NextBatch: want thread resolution error, got nil")
+ }
+}
+
+func TestSlackConnectorTextCleaning(t *testing.T) {
+ withSlackTestHooks(t)
+ usersCalls := &atomic.Int64{}
+ fixtures := &slackTestFixtures{usersByID: map[string]string{"U123": slackUserAlice()}, usersCalls: usersCalls}
+ server := newTestSlackServer(t, fixtures)
+ connector := mustSlackConnector(t, server.URL, nil)
+
+ text := "Hi <@U123> in <#C1|general> <!channel> <!here> <!everyone> <!date|123|formatted> and <@U123> again"
+ got, err := connector.indexClean(context.Background(), text)
+ if err != nil {
+ t.Fatalf("indexClean: %v", err)
+ }
+ want := "Hi @Alice in #general @channel @here @everyone 123|formatted and @Alice again"
+ if got != want {
+ t.Fatalf("indexClean = %q, want %q", got, want)
+ }
+ if usersCalls.Load() != 1 {
+ t.Fatalf("users.info calls = %d, want 1 (cached)", usersCalls.Load())
+ }
+}
+
+func TestSlackConnectorValidateConnectorSetting(t *testing.T) {
+ withSlackTestHooks(t)
+ fixtures := &slackTestFixtures{channelsJSON: slackChannelsTwo}
+ server := newTestSlackServer(t, fixtures)
+ connector := mustSlackConnector(t, server.URL, nil)
+
+ if err := connector.ValidateConnectorSetting(context.Background(), map[string]any{}); err != nil {
+ t.Fatalf("ValidateConnectorSetting: %v", err)
+ }
+}
+
+func TestSlackConnectorHistoryPagination(t *testing.T) {
+ withSlackTestHooks(t)
+ var calls atomic.Int64
+ mux := http.NewServeMux()
+ mux.HandleFunc("/api/auth.test", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{"ok":true}`))
+ })
+ mux.HandleFunc("/api/conversations.list", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(slackChannelsTwo))
+ })
+ mux.HandleFunc("/api/conversations.history", func(w http.ResponseWriter, r *http.Request) {
+ if r.FormValue("channel") != "C1" {
+ w.Write([]byte(`{"ok":true,"messages":[],"has_more":false,"response_metadata":{"next_cursor":""}}`))
+ return
+ }
+ if calls.Add(1) == 1 {
+ w.Write([]byte(`{"ok":true,"messages":[` + slackMsgPlain + `],"has_more":true,"response_metadata":{"next_cursor":"nextpage"}}`))
+ return
+ }
+ w.Write([]byte(`{"ok":true,"messages":[` + slackMsgNormal + `],"has_more":false,"response_metadata":{"next_cursor":""}}`))
+ })
+ mux.HandleFunc("/api/conversations.join", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{"ok":true}`))
+ })
+ mux.HandleFunc("/api/users.info", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(slackUserAlice()))
+ })
+ server := httptest.NewServer(mux)
+ t.Cleanup(server.Close)
+
+ connector := mustSlackConnector(t, server.URL, nil)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ defer session.Close()
+ documents := collectSlackDocuments(t, session)
+ if len(documents) != 2 {
+ t.Fatalf("documents = %d, want 2 across two history pages", len(documents))
+ }
+ if calls.Load() != 2 {
+ t.Fatalf("C1 history calls = %d, want 2", calls.Load())
+ }
+}
+
+func TestSlackConnectorChannelListPagination(t *testing.T) {
+ withSlackTestHooks(t)
+ var calls atomic.Int64
+ mux := http.NewServeMux()
+ mux.HandleFunc("/api/auth.test", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{"ok":true}`))
+ })
+ mux.HandleFunc("/api/conversations.list", func(w http.ResponseWriter, r *http.Request) {
+ if calls.Add(1) == 1 {
+ w.Write([]byte(`{"ok":true,"channels":[{"id":"C1","name":"general","is_member":true}],"response_metadata":{"next_cursor":"nextpage"}}`))
+ return
+ }
+ w.Write([]byte(`{"ok":true,"channels":[{"id":"C2","name":"random","is_member":true}],"response_metadata":{"next_cursor":""}}`))
+ })
+ mux.HandleFunc("/api/conversations.history", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{"ok":true,"messages":[],"has_more":false,"response_metadata":{"next_cursor":""}}`))
+ })
+ mux.HandleFunc("/api/conversations.join", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{"ok":true}`))
+ })
+ mux.HandleFunc("/api/users.info", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(slackUserAlice()))
+ })
+ server := httptest.NewServer(mux)
+ t.Cleanup(server.Close)
+
+ connector := mustSlackConnector(t, server.URL, nil)
+ channels, err := connector.resolvedChannels(context.Background())
+ if err != nil {
+ t.Fatalf("resolvedChannels: %v", err)
+ }
+ if len(channels) != 2 || channels[0].ID != "C1" || channels[1].ID != "C2" {
+ t.Fatalf("channels = %+v", channels)
+ }
+}
+
+func TestSlackConnectorRetriesTransientHistory(t *testing.T) {
+ withSlackTestHooks(t)
+ var calls atomic.Int64
+ mux := http.NewServeMux()
+ mux.HandleFunc("/api/auth.test", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{"ok":true}`))
+ })
+ mux.HandleFunc("/api/conversations.list", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(slackChannelsTwo))
+ })
+ mux.HandleFunc("/api/conversations.history", func(w http.ResponseWriter, r *http.Request) {
+ if r.FormValue("channel") != "C1" {
+ w.Write([]byte(`{"ok":true,"messages":[],"has_more":false,"response_metadata":{"next_cursor":""}}`))
+ return
+ }
+ if calls.Add(1) == 1 {
+ w.WriteHeader(http.StatusTooManyRequests)
+ w.Write([]byte(`{"ok":false,"error":"ratelimited"}`))
+ return
+ }
+ w.Write([]byte(`{"ok":true,"messages":[` + slackMsgPlain + `],"has_more":false,"response_metadata":{"next_cursor":""}}`))
+ })
+ mux.HandleFunc("/api/conversations.join", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{"ok":true}`))
+ })
+ mux.HandleFunc("/api/users.info", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(slackUserAlice()))
+ })
+ server := httptest.NewServer(mux)
+ t.Cleanup(server.Close)
+
+ connector := mustSlackConnector(t, server.URL, nil)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ defer session.Close()
+ documents := collectSlackDocuments(t, session)
+ if len(documents) != 1 {
+ t.Fatalf("documents = %d, want 1 after transient retry", len(documents))
+ }
+ if calls.Load() != 2 {
+ t.Fatalf("C1 history calls = %d, want 2 (429 then success)", calls.Load())
+ }
+}
+
+func TestSlackConnectorOpenSyncSkipsUnjoinableChannel(t *testing.T) {
+ withSlackTestHooks(t)
+ joinCalls := &atomic.Int64{}
+ mux := http.NewServeMux()
+ mux.HandleFunc("/api/auth.test", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{"ok":true}`))
+ })
+ mux.HandleFunc("/api/conversations.list", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(slackChannelsTwo))
+ })
+ mux.HandleFunc("/api/conversations.join", func(w http.ResponseWriter, r *http.Request) {
+ joinCalls.Add(1)
+ if r.FormValue("channel") == "C2" {
+ w.Write([]byte(`{"ok":false,"error":"is_archived"}`))
+ return
+ }
+ w.Write([]byte(`{"ok":true}`))
+ })
+ mux.HandleFunc("/api/conversations.history", func(w http.ResponseWriter, r *http.Request) {
+ if r.FormValue("channel") != "C1" {
+ w.Write([]byte(`{"ok":true,"messages":[],"has_more":false,"response_metadata":{"next_cursor":""}}`))
+ return
+ }
+ w.Write([]byte(slackHistoryBody(slackMsgPlain)))
+ })
+ mux.HandleFunc("/api/users.info", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(slackUserAlice()))
+ })
+ server := httptest.NewServer(mux)
+ t.Cleanup(server.Close)
+
+ connector := mustSlackConnector(t, server.URL, nil)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ defer session.Close()
+ documents := collectSlackDocuments(t, session)
+ if len(documents) != 1 || documents[0].SourceID != "C1__1700000000.000001" {
+ t.Fatalf("documents = %+v, want only C1", documents)
+ }
+ if joinCalls.Load() != 1 {
+ t.Fatalf("join calls = %d, want 1 (only C2)", joinCalls.Load())
+ }
+}
+
+func TestSlackConnectorOpenPruneSkipsUnjoinableChannel(t *testing.T) {
+ withSlackTestHooks(t)
+ joinCalls := &atomic.Int64{}
+ mux := http.NewServeMux()
+ mux.HandleFunc("/api/auth.test", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{"ok":true}`))
+ })
+ mux.HandleFunc("/api/conversations.list", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(slackChannelsTwo))
+ })
+ mux.HandleFunc("/api/conversations.join", func(w http.ResponseWriter, r *http.Request) {
+ joinCalls.Add(1)
+ if r.FormValue("channel") == "C2" {
+ w.Write([]byte(`{"ok":false,"error":"method_not_supported_for_channel_type"}`))
+ return
+ }
+ w.Write([]byte(`{"ok":true}`))
+ })
+ mux.HandleFunc("/api/conversations.history", func(w http.ResponseWriter, r *http.Request) {
+ if r.FormValue("channel") != "C1" {
+ w.Write([]byte(`{"ok":true,"messages":[],"has_more":false,"response_metadata":{"next_cursor":""}}`))
+ return
+ }
+ w.Write([]byte(slackHistoryBody(slackMsgPlain)))
+ })
+ mux.HandleFunc("/api/users.info", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(slackUserAlice()))
+ })
+ server := httptest.NewServer(mux)
+ t.Cleanup(server.Close)
+
+ connector := mustSlackConnector(t, server.URL, nil)
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{TaskID: "t1"})
+ if err != nil {
+ t.Fatalf("OpenPrune: %v", err)
+ }
+ defer session.Close()
+ var slim []SlimDocument
+ for {
+ batch, err := session.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ slim = append(slim, batch.Documents...)
+ }
+ if len(slim) != 1 || slim[0].SourceID != "C1__1700000000.000001" {
+ t.Fatalf("slim documents = %+v, want only C1", slim)
+ }
+}
+
+func TestSlackConnectorJoinPropagatesUnrelatedErrors(t *testing.T) {
+ withSlackTestHooks(t)
+ mux := http.NewServeMux()
+ mux.HandleFunc("/api/auth.test", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{"ok":true}`))
+ })
+ mux.HandleFunc("/api/conversations.list", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(slackChannelsTwo))
+ })
+ mux.HandleFunc("/api/conversations.join", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{"ok":false,"error":"invalid_auth"}`))
+ })
+ mux.HandleFunc("/api/conversations.history", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(slackHistoryBody(slackMsgPlain)))
+ })
+ mux.HandleFunc("/api/users.info", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(slackUserAlice()))
+ })
+ server := httptest.NewServer(mux)
+ t.Cleanup(server.Close)
+
+ connector := mustSlackConnector(t, server.URL, nil)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ defer session.Close()
+ // C2 is not a member, so join fails with invalid_auth, which is unrelated
+ // to channel availability and must abort the run.
+ var gotErr error
+ for {
+ _, nextErr := session.NextBatch(context.Background())
+ if errors.Is(nextErr, io.EOF) {
+ break
+ }
+ if nextErr != nil {
+ gotErr = nextErr
+ break
+ }
+ }
+ if gotErr == nil {
+ t.Fatalf("NextBatch: want error, got nil")
+ }
+ if errors.Is(gotErr, errSlackChannelUnavailable) {
+ t.Fatalf("NextBatch error = %v, want unrelated error to propagate unwrapped", gotErr)
+ }
+}
+
+func TestSlackConnectorJoinRestrictedToSelectedChannels(t *testing.T) {
+ withSlackTestHooks(t)
+ joinCalls := &atomic.Int64{}
+ mux := http.NewServeMux()
+ mux.HandleFunc("/api/conversations.join", func(w http.ResponseWriter, r *http.Request) {
+ joinCalls.Add(1)
+ w.Write([]byte(`{"ok":true}`))
+ })
+ server := httptest.NewServer(mux)
+ t.Cleanup(server.Close)
+
+ connector := mustSlackConnector(t, server.URL, map[string]any{"channels": "general"})
+ ctx := context.Background()
+ selected := slackChannel{ID: "C1", Name: "general", IsMember: false}
+ unselected := slackChannel{ID: "C9", Name: "random", IsMember: false}
+ if err := connector.joinChannel(ctx, unselected); err != nil {
+ t.Fatalf("joinChannel unselected: %v", err)
+ }
+ if joinCalls.Load() != 0 {
+ t.Fatalf("join calls after unselected channel = %d, want 0", joinCalls.Load())
+ }
+ if err := connector.joinChannel(ctx, selected); err != nil {
+ t.Fatalf("joinChannel selected: %v", err)
+ }
+ if joinCalls.Load() != 1 {
+ t.Fatalf("join calls after selected channel = %d, want 1", joinCalls.Load())
+ }
+
+ // Regex selection restricts joins the same way.
+ regexConnector := mustSlackConnector(t, server.URL, map[string]any{
+ "channels": "gen.*",
+ "channel_regex_enabled": true,
+ })
+ if err := regexConnector.joinChannel(ctx, unselected); err != nil {
+ t.Fatalf("regex joinChannel unselected: %v", err)
+ }
+ if joinCalls.Load() != 1 {
+ t.Fatalf("join calls after regex unselected channel = %d, want 1", joinCalls.Load())
+ }
+ if err := regexConnector.joinChannel(ctx, selected); err != nil {
+ t.Fatalf("regex joinChannel selected: %v", err)
+ }
+ if joinCalls.Load() != 2 {
+ t.Fatalf("join calls after regex selected channel = %d, want 2", joinCalls.Load())
+ }
+}