RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(go-syncer): add Slack connector with checkpoint resume (#18603)
mkaaad05af33b21464
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.goindex 75a32a21fc..42ca0d17d9 100644--- a/internal/syncer/connector/builtin.go+++ b/internal/syncer/connector/builtin.go@@ -45,6 +45,7 @@ func RegisterBuiltIns(registry *Registry) {registerBuiltIn(registry, "moodle", NewMoodleConnector)registerBuiltIn(registry, "mysql", NewMySQLConnector)registerBuiltIn(registry, "postgresql", NewPostgreSQLConnector)+ registerBuiltIn(registry, "slack", NewSlackConnector)registerBuiltIn(registry, "discord", NewDiscordConnector)registerBuiltIn(registry, "webdav", NewWebDAVConnector)}diff --git a/internal/syncer/connector/slack.go b/internal/syncer/connector/slack.gonew file mode 100644index 0000000000..f0f8157b86--- /dev/null+++ b/internal/syncer/connector/slack.go@@ -0,0 +1,1144 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "net/http"+ "net/url"+ "regexp"+ "sort"+ "strconv"+ "strings"+ "time"+)++// Slack connector constants.+const (+ slackDefaultBaseURL = "https://slack.com/api"+ slackDefaultBatchSize = 2+ slackRequestTimeout = 60 * time.Second+ slackPageSize = 900+ slackMaxRetryDelay = 60 * time.Second+ slackSnippetLength = 50+ slackMaxChannelsToLog = 20+ slackUSLACKBOTUser = "USLACKBOT"+)++// Retry/backoff knobs for Slack API calls. They are package variables so tests+// can shrink the delays.+var (+ slackRetryTries = 3+ slackRetryBaseDelay = time.Second+ slackRetryBackoff = 2+)++// slackMaxResponseSize caps Slack API JSON responses. Package variable so+// tests can shrink it.+var slackMaxResponseSize int64 = 32 * 1024 * 1024++// SlackConnector reads public and private channel messages and threads from a+// Slack workspace through the Slack Web API with a bot token.+type SlackConnector struct {+ token string+ channels []string+ channelRegexEnabled bool+ batchSize int+ baseURL string+ client *http.Client+ userCache map[string]string+}++// NewSlackConnector creates a Slack connector from connector config.+func NewSlackConnector(config map[string]any) (*SlackConnector, error) {+ credentials := configAnyMap(config["credentials"])+ token := strings.TrimSpace(stringConfig(credentials["slack_bot_token"]))++ batchSize := configInt(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])), slackDefaultBatchSize)+ if batchSize <= 0 {+ batchSize = slackDefaultBatchSize+ }++ return &SlackConnector{+ token: token,+ channels: normalizeSlackChannelNames(coerceDiscordStringList(config["channels"])),+ channelRegexEnabled: configBoolDefault(config["channel_regex_enabled"], false),+ batchSize: batchSize,+ baseURL: strings.TrimRight(slackDefaultBaseURL, "/"),+ client: &http.Client{Timeout: slackRequestTimeout},+ userCache: map[string]string{},+ }, nil+}++// Validate validates Slack connector settings, credentials, and connectivity.+func (c *SlackConnector) Validate(ctx context.Context) error {+ if c == nil {+ return &ConnectorValidationError{Message: "Slack connector is nil"}+ }+ if c.token == "" {+ return &ConnectorMissingCredentialError{Message: "Slack connector requires 'slack_bot_token' in credentials"}+ }+ if c.batchSize <= 0 {+ return &ConnectorValidationError{Message: "Slack connector batch_size must be a positive integer"}+ }++ // 1) Validate workspace connection.+ var auth slackAuthTestResponse+ if err := c.callAPI(ctx, "auth.test", url.Values{}, &auth); err != nil {+ return slackValidationError(err)+ }++ // 2) Confirm listing channels works.+ var list slackListChannelsResponse+ if err := c.callAPI(ctx, "conversations.list", url.Values{+ "limit": {"1"},+ "types": {"public_channel"},+ }, &list); err != nil {+ return slackValidationError(err)+ }++ // 3) Confirm users:read scope is available (required to resolve senders).+ var user slackUserInfoResponse+ if err := c.callAPI(ctx, "users.info", url.Values{"user": {slackUSLACKBOTUser}}, &user); err != nil {+ if errors.As(err, new(*slackUserNotFoundError)) {+ return nil+ }+ return slackValidationError(err)+ }+ return nil+}++// ValidateConnectorSetting validates Slack settings from an unsaved config.+func (c *SlackConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)+ defer cancel()+ return c.Validate(ctx)+}++// OpenSync opens one Slack sync session.+func (c *SlackConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {+ if err := c.Validate(ctx); err != nil {+ return nil, err+ }+ channels, err := c.resolvedChannels(ctx)+ if err != nil {+ return nil, err+ }+ session := &slackSyncSession{+ connector: c,+ request: request,+ channels: channels,+ batchSize: c.batchSize,+ seenThreads: map[string]struct{}{},+ }+ if request.Resume != nil {+ session.applyResume(request.Resume)+ }+ return session, nil+}++// OpenPrune opens one complete Slack prune snapshot session.+func (c *SlackConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {+ if err := c.Validate(ctx); err != nil {+ return nil, err+ }+ channels, err := c.resolvedChannels(ctx)+ if err != nil {+ return nil, err+ }+ return &slackPruneSession{connector: c, channels: channels, batchSize: c.batchSize}, nil+}++// ---------------------------------------------------------------------------+// Slack Web API client+// ---------------------------------------------------------------------------++// slackAPIResponse is embedded in every Slack API response payload.+type slackAPIResponse struct {+ OK bool `json:"ok"`+ Error string `json:"error"`+}++// slackResponseMetadata carries the opaque pagination cursor.+type slackResponseMetadata struct {+ NextCursor string `json:"next_cursor"`+}++type slackAuthTestResponse struct {+ slackAPIResponse+ URL string `json:"url"`+ Team string `json:"team"`+ User string `json:"user"`+ TeamID string `json:"team_id"`+ UserID string `json:"user_id"`+}++type slackChannel struct {+ ID string `json:"id"`+ Name string `json:"name"`+ IsArchived bool `json:"is_archived"`+ IsMember bool `json:"is_member"`+ IsPrivate bool `json:"is_private"`+}++type slackListChannelsResponse struct {+ slackAPIResponse+ Channels []slackChannel `json:"channels"`+ ResponseMetadata slackResponseMetadata `json:"response_metadata"`+}++type slackMessage struct {+ TS string `json:"ts"`+ ThreadTS string `json:"thread_ts"`+ User string `json:"user"`+ Text string `json:"text"`+ Subtype string `json:"subtype"`+ BotID string `json:"bot_id"`+ AppID string `json:"app_id"`+ BotProfile struct {+ Name string `json:"name"`+ } `json:"bot_profile"`+}++type slackHistoryResponse struct {+ slackAPIResponse+ Messages []slackMessage `json:"messages"`+ HasMore bool `json:"has_more"`+ ResponseMetadata slackResponseMetadata `json:"response_metadata"`+}++type slackUserInfoResponse struct {+ slackAPIResponse+ User struct {+ ID string `json:"id"`+ RealName string `json:"real_name"`+ Profile struct {+ DisplayName string `json:"display_name"`+ RealName string `json:"real_name"`+ } `json:"profile"`+ } `json:"user"`+}++// slackTransientError marks a retryable Slack API failure.+type slackTransientError struct {+ retryAfter time.Duration+ rateLimited bool+ message string+}++func (e *slackTransientError) Error() string { return e.message }++// slackUserNotFoundError is returned when users.info cannot resolve a user.+type slackUserNotFoundError struct{ message string }++func (e *slackUserNotFoundError) Error() string { return e.message }++// slackAPIError wraps a classified Slack API error with the raw error code so+// callers can branch on the specific Slack failure.+type slackAPIError struct {+ code string+ err error+}++func (e *slackAPIError) Error() string { return e.err.Error() }+func (e *slackAPIError) Unwrap() error { return e.err }++// callAPI sends one Slack Web API call, retrying transient failures.+func (c *SlackConnector) callAPI(ctx context.Context, method string, params url.Values, out any) error {+ return c.retrySlack(ctx, func() error {+ return c.callAPINoRetry(ctx, method, params, out)+ })+}++func (c *SlackConnector) callAPINoRetry(ctx context.Context, method string, params url.Values, out any) error {+ endpoint := c.baseURL + "/" + method+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, strings.NewReader(params.Encode()))+ if err != nil {+ return err+ }+ req.Header.Set("Authorization", "Bearer "+c.token)+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded")++ resp, err := c.client.Do(req)+ if err != nil {+ return err+ }+ defer resp.Body.Close()++ body, err := io.ReadAll(io.LimitReader(resp.Body, slackMaxResponseSize+1))+ if err != nil {+ return err+ }+ if int64(len(body)) > slackMaxResponseSize {+ return &ConnectorValidationError{Message: fmt.Sprintf("Slack API response from %s exceeds maximum size of %d bytes", method, slackMaxResponseSize)}+ }+ if resp.StatusCode == http.StatusTooManyRequests {+ return &slackTransientError{+ retryAfter: slackRetryAfterSeconds(resp.Header.Get("Retry-After"), body),+ rateLimited: true,+ message: "Slack API rate limited",+ }+ }++ var probe slackAPIResponse+ if err := json.Unmarshal(body, &probe); err != nil {+ if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {+ return &ConnectorValidationError{Message: fmt.Sprintf("Slack API %s request failed with status %d", method, resp.StatusCode)}+ }+ return fmt.Errorf("Slack API %s returned invalid JSON: %w", method, err)+ }+ if !probe.OK {+ return classifySlackAPIError(method, probe.Error, body)+ }+ if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {+ return &ConnectorValidationError{Message: fmt.Sprintf("Slack API %s request failed with status %d", method, resp.StatusCode)}+ }+ return json.Unmarshal(body, out)+}++func classifySlackAPIError(method, apiError string, body []byte) error {+ var err error+ switch apiError {+ case "ratelimited":+ err = &slackTransientError{+ retryAfter: slackRetryAfterSeconds("", body),+ rateLimited: true,+ message: "Slack API rate limited",+ }+ case "invalid_auth":+ err = &ConnectorValidationError{Message: "Invalid Slack bot token (invalid_auth)."}+ case "not_authed":+ err = &ConnectorMissingCredentialError{Message: "Invalid or expired Slack bot token (not_authed)."}+ case "missing_scope":+ err = &ConnectorValidationError{Message: slackMissingScopeMessage(method, body)}+ case "not_in_channel":+ err = &ConnectorValidationError{Message: "Slack bot is not a member of the channel (not_in_channel)."}+ case "channel_not_found":+ err = &ConnectorValidationError{Message: "Slack channel not found (channel_not_found)."}+ case "user_not_found":+ err = &slackUserNotFoundError{message: "Slack user not found (user_not_found)."}+ case "method_not_supported_for_channel_type":+ err = &ConnectorValidationError{Message: "Slack method is not supported for this channel type."}+ default:+ if apiError == "" {+ err = &ConnectorValidationError{Message: fmt.Sprintf("Slack API %s request failed without an error code", method)}+ } else {+ err = &ConnectorValidationError{Message: fmt.Sprintf("Slack API %s request failed: %s", method, apiError)}+ }+ }+ if apiError == "" {+ return err+ }+ return &slackAPIError{code: apiError, err: err}+}++// slackMissingScopeMessage builds an actionable message for a missing_scope+// Slack error, naming the scope the call needed. It falls back to a generic+// message when Slack does not include the needed scope in the response.+func slackMissingScopeMessage(method string, body []byte) string {+ var payload struct {+ Needed any `json:"needed"`+ }+ msg := fmt.Sprintf("Slack bot token lacks the necessary OAuth scope to access %s (missing_scope)", method)+ if err := json.Unmarshal(body, &payload); err != nil {+ return msg + "."+ }+ if needed := slackScopeNames(payload.Needed); needed != "" {+ return fmt.Sprintf("%s. Needed: %s", msg, needed)+ }+ return msg + "."+}++// slackScopeNames flattens a Slack scope field, which is either a single+// comma-separated string or a JSON array of scope names.+func slackScopeNames(value any) string {+ switch v := value.(type) {+ case string:+ return strings.TrimSpace(v)+ case []any:+ parts := make([]string, 0, len(v))+ for _, item := range v {+ if s, ok := item.(string); ok && strings.TrimSpace(s) != "" {+ parts = append(parts, strings.TrimSpace(s))+ }+ }+ return strings.Join(parts, ", ")+ }+ return ""+}++func slackRetryAfterSeconds(header string, body []byte) time.Duration {+ if seconds, err := strconv.Atoi(strings.TrimSpace(header)); err == nil && seconds > 0 {+ return time.Duration(seconds) * time.Second+ }+ var payload struct {+ RetryAfter float64 `json:"retry_after"`+ }+ if err := json.Unmarshal(body, &payload); err == nil && payload.RetryAfter > 0 {+ return time.Duration(payload.RetryAfter * float64(time.Second))+ }+ return 0+}++func (c *SlackConnector) retrySlack(ctx context.Context, fn func() error) error {+ var lastErr error+ for attempt := 0; attempt < slackRetryTries; attempt++ {+ if err := ctx.Err(); err != nil {+ return err+ }+ err := fn()+ if err == nil {+ return nil+ }+ lastErr = err+ if !isSlackTransient(err) || attempt+1 >= slackRetryTries {+ break+ }+ delay := slackRetryBaseDelay+ for i := 0; i < attempt; i++ {+ delay *= time.Duration(slackRetryBackoff)+ }+ var transient *slackTransientError+ if errors.As(err, &transient) && transient.retryAfter > delay {+ delay = transient.retryAfter+ }+ if delay > slackMaxRetryDelay {+ delay = slackMaxRetryDelay+ }+ timer := time.NewTimer(delay)+ select {+ case <-ctx.Done():+ timer.Stop()+ return ctx.Err()+ case <-timer.C:+ }+ }+ return lastErr+}++func isSlackTransient(err error) bool {+ if err == nil {+ return false+ }+ var transient *slackTransientError+ if errors.As(err, &transient) {+ return true+ }+ var urlErr *url.Error+ return errors.As(err, &urlErr)+}++// slackValidationError treats a persistent rate limit as a warning so that+// settings validation proceeds, matching the reference SDK behavior.+func slackValidationError(err error) error {+ var transient *slackTransientError+ if errors.As(err, &transient) && transient.rateLimited {+ return nil+ }+ return err+}++// ---------------------------------------------------------------------------+// Channel and message enumeration+// ---------------------------------------------------------------------------++// resolvedChannels returns the configured channels, sorted by ID so checkpoints+// advance deterministically.+func (c *SlackConnector) resolvedChannels(ctx context.Context) ([]slackChannel, error) {+ all, err := c.listChannels(ctx)+ if err != nil {+ return nil, err+ }+ filtered, err := filterSlackChannels(all, c.channels, c.channelRegexEnabled)+ if err != nil {+ return nil, err+ }+ sort.SliceStable(filtered, func(i, j int) bool {+ return filtered[i].ID < filtered[j].ID+ })+ return filtered, nil+}++// listChannels lists public and private channels, falling back to public-only+// when private channel enumeration fails.+func (c *SlackConnector) listChannels(ctx context.Context) ([]slackChannel, error) {+ channels, err := c.collectChannels(ctx, []string{"public_channel", "private_channel"})+ if err != nil {+ channels, err = c.collectChannels(ctx, []string{"public_channel"})+ }+ return channels, err+}++func (c *SlackConnector) collectChannels(ctx context.Context, channelTypes []string) ([]slackChannel, error) {+ var channels []slackChannel+ cursor := ""+ for {+ params := url.Values{+ "limit": {strconv.Itoa(slackPageSize)},+ "types": {strings.Join(channelTypes, ",")},+ "exclude_archived": {"true"},+ }+ if cursor != "" {+ params.Set("cursor", cursor)+ }+ var resp slackListChannelsResponse+ if err := c.callAPI(ctx, "conversations.list", params, &resp); err != nil {+ return nil, err+ }+ channels = append(channels, resp.Channels...)+ if resp.ResponseMetadata.NextCursor == "" {+ return channels, nil+ }+ cursor = resp.ResponseMetadata.NextCursor+ }+}++// channelMessages returns every top-level message in a channel, optionally+// bounded by oldest/latest Slack epoch-second strings (empty means unbounded).+func (c *SlackConnector) channelMessages(ctx context.Context, channel slackChannel, oldest, latest string) ([]slackMessage, error) {+ if err := c.joinChannel(ctx, channel); err != nil {+ return nil, err+ }+ var messages []slackMessage+ cursor := ""+ for {+ params := url.Values{+ "channel": {channel.ID},+ "limit": {strconv.Itoa(slackPageSize)},+ }+ if oldest != "" {+ params.Set("oldest", oldest)+ }+ if latest != "" {+ params.Set("latest", latest)+ }+ if cursor != "" {+ params.Set("cursor", cursor)+ }+ var resp slackHistoryResponse+ if err := c.callAPI(ctx, "conversations.history", params, &resp); err != nil {+ return nil, err+ }+ messages = append(messages, resp.Messages...)+ if resp.ResponseMetadata.NextCursor == "" {+ return messages, nil+ }+ cursor = resp.ResponseMetadata.NextCursor+ }+}++// getThread returns the full reply thread for a channel message timestamp.+func (c *SlackConnector) getThread(ctx context.Context, channelID, threadTS string) ([]slackMessage, error) {+ var messages []slackMessage+ cursor := ""+ for {+ params := url.Values{+ "channel": {channelID},+ "ts": {threadTS},+ "limit": {strconv.Itoa(slackPageSize)},+ }+ if cursor != "" {+ params.Set("cursor", cursor)+ }+ var resp slackHistoryResponse+ if err := c.callAPI(ctx, "conversations.replies", params, &resp); err != nil {+ return nil, err+ }+ messages = append(messages, resp.Messages...)+ if resp.ResponseMetadata.NextCursor == "" {+ return messages, nil+ }+ cursor = resp.ResponseMetadata.NextCursor+ }+}++// errSlackChannelUnavailable marks Slack channels that the bot cannot join so+// callers can skip them instead of failing the whole sync/prune run.+var errSlackChannelUnavailable = errors.New("slack channel unavailable")++// slackUnjoinableErrors are conversations.join failure codes that mean the bot+// cannot join the channel.+var slackUnjoinableErrors = map[string]struct{}{+ "is_archived": {},+ "method_not_supported_for_channel_type": {},+ "channel_not_found": {},+ "restricted_action": {},+}++// joinChannel joins a channel so the bot can read its messages. Channels the+// bot cannot join are wrapped in errSlackChannelUnavailable so callers can+// skip them; unrelated API errors propagate unchanged. Automatic joins are+// limited to the configured channels selection when one is present.+func (c *SlackConnector) joinChannel(ctx context.Context, channel slackChannel) error {+ if channel.IsMember {+ return nil+ }+ if len(c.channels) > 0 && !slackChannelSelected(channel, c.channels, c.channelRegexEnabled) {+ return nil+ }+ var resp struct {+ slackAPIResponse+ }+ if err := c.callAPI(ctx, "conversations.join", url.Values{"channel": {channel.ID}}, &resp); err != nil {+ var apiErr *slackAPIError+ if errors.As(err, &apiErr) {+ if _, ok := slackUnjoinableErrors[apiErr.code]; ok {+ return fmt.Errorf("%w: %v", errSlackChannelUnavailable, err)+ }+ }+ return err+ }+ return nil+}++// slackChannelSelected reports whether a channel is part of the configured+// channels selection (exact names, or regex patterns when enabled). An empty+// selection matches every channel.+func slackChannelSelected(channel slackChannel, configured []string, regexEnabled bool) bool {+ if len(configured) == 0 {+ return true+ }+ for _, pattern := range configured {+ if regexEnabled {+ re, err := regexp.Compile("^" + pattern + "$")+ if err != nil {+ continue+ }+ if re.MatchString(channel.Name) {+ return true+ }+ continue+ }+ if channel.Name == pattern {+ return true+ }+ }+ return false+}++// displayName resolves a Slack user ID to a display name, caching results.+func (c *SlackConnector) displayName(ctx context.Context, userID string) (string, error) {+ if userID == "" {+ return "", nil+ }+ if name, ok := c.userCache[userID]; ok {+ return name, nil+ }+ var resp slackUserInfoResponse+ if err := c.callAPI(ctx, "users.info", url.Values{"user": {userID}}, &resp); err != nil {+ var notFound *slackUserNotFoundError+ if errors.As(err, ¬Found) {+ return "", nil+ }+ return "", err+ }+ name := resp.User.Profile.DisplayName+ if name == "" {+ name = resp.User.RealName+ }+ if name == "" {+ name = resp.User.Profile.RealName+ }+ if name == "" {+ return "", nil+ }+ c.userCache[userID] = name+ return name, nil+}++// filterSlackChannels narrows the workspace channels to the configured set.+func filterSlackChannels(all []slackChannel, channels []string, regexEnabled bool) ([]slackChannel, error) {+ if len(channels) == 0 {+ return all, nil+ }+ if regexEnabled {+ var out []slackChannel+ for _, pattern := range channels {+ re, err := regexp.Compile("^" + pattern + "$")+ if err != nil {+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Invalid Slack channel regex %q", pattern)}+ }+ for _, channel := range all {+ if re.MatchString(channel.Name) {+ out = append(out, channel)+ }+ }+ }+ return out, nil+ }++ available := map[string]slackChannel{}+ for _, channel := range all {+ available[channel.Name] = channel+ }+ var missing []string+ for _, name := range channels {+ if _, ok := available[name]; !ok {+ missing = append(missing, name)+ }+ }+ if len(missing) > 0 {+ names := make([]string, 0, len(available))+ for name := range available {+ names = append(names, name)+ }+ sort.Strings(names)+ if len(names) > slackMaxChannelsToLog {+ names = names[:slackMaxChannelsToLog]+ }+ return nil, &ConnectorValidationError{Message: fmt.Sprintf(+ "Channel(s) %s not found in workspace. Available channels: %s",+ strings.Join(missing, ", "), strings.Join(names, ", "))}+ }+ out := make([]slackChannel, 0, len(channels))+ for _, name := range channels {+ out = append(out, available[name])+ }+ return out, nil+}++// ---------------------------------------------------------------------------+// Message filtering and text cleaning+// ---------------------------------------------------------------------------++// slackDisallowedSubtypes lists message subtypes that carry no informative+// content.+var slackDisallowedSubtypes = map[string]struct{}{+ "channel_join": {},+ "channel_leave": {},+ "channel_archive": {},+ "channel_unarchive": {},+ "pinned_item": {},+ "unpinned_item": {},+ "ekm_access_denied": {},+ "channel_posting_permissions": {},+ "group_join": {},+ "group_leave": {},+ "group_archive": {},+ "group_unarchive": {},+ "channel_name": {},+}++// acceptSlackMessage reports whether a message should be indexed.+func acceptSlackMessage(message slackMessage) bool {+ if message.BotID != "" || message.AppID != "" {+ if message.BotProfile.Name == "DanswerBot Testing" {+ return true+ }+ return false+ }+ if message.Subtype != "" {+ if _, ok := slackDisallowedSubtypes[message.Subtype]; ok {+ return false+ }+ }+ return true+}++var (+ slackUserMentionRe = regexp.MustCompile(`<@(.*?)>`)+ slackChannelMentionRe = regexp.MustCompile(`<#(.*?)\|(.*?)>`)+ slackSpecialCatchallRe = regexp.MustCompile(`<!([^|]+)\|([^>]+)>`)+)++// indexClean normalizes Slack markdown links and mentions into readable text.+func (c *SlackConnector) indexClean(ctx context.Context, text string) (string, error) {+ cleaned, err := c.replaceUserIDsWithNames(ctx, text)+ if err != nil {+ return "", err+ }+ cleaned = slackReplaceChannelsBasic(cleaned)+ cleaned = slackReplaceSpecialMentions(cleaned)+ cleaned = slackReplaceSpecialCatchall(cleaned)+ return cleaned, nil+}++func (c *SlackConnector) replaceUserIDsWithNames(ctx context.Context, text string) (string, error) {+ matches := slackUserMentionRe.FindAllStringSubmatch(text, -1)+ for _, match := range matches {+ if len(match) < 2 {+ continue+ }+ userID := match[1]+ name, err := c.displayName(ctx, userID)+ if err != nil {+ return "", err+ }+ if name != "" {+ text = strings.ReplaceAll(text, match[0], "@"+name)+ }+ }+ return text, nil+}++func slackReplaceChannelsBasic(text string) string {+ return slackChannelMentionRe.ReplaceAllString(text, "#$2")+}++func slackReplaceSpecialMentions(text string) string {+ text = strings.ReplaceAll(text, "<!channel>", "@channel")+ text = strings.ReplaceAll(text, "<!here>", "@here")+ text = strings.ReplaceAll(text, "<!everyone>", "@everyone")+ return text+}++func slackReplaceSpecialCatchall(text string) string {+ return slackSpecialCatchallRe.ReplaceAllString(text, "$2")+}++// ---------------------------------------------------------------------------+// Document building+// ---------------------------------------------------------------------------++func slackDocID(channelID, ts string) string {+ return channelID + "__" + ts+}++// threadToDocument flattens a filtered thread into one text document. docTS is+// the thread root timestamp used as the document identity.+func (c *SlackConnector) threadToDocument(ctx context.Context, channel slackChannel, thread []slackMessage, docTS string) (*SourceDocument, error) {+ senderName := "Unknown"+ if len(thread) > 0 {+ if name, err := c.displayName(ctx, thread[0].User); err != nil {+ return nil, err+ } else if name != "" {+ senderName = name+ }+ }++ cleaned := make([]string, 0, len(thread))+ for _, message := range thread {+ text, err := c.indexClean(ctx, message.Text)+ if err != nil {+ return nil, err+ }+ cleaned = append(cleaned, text)+ }++ first := ""+ if len(cleaned) > 0 {+ first = cleaned[0]+ }+ snippet := first+ if runes := []rune(first); len(runes) > slackSnippetLength {+ snippet = strings.TrimRight(string(runes[:slackSnippetLength]), " \t\n") + "..."+ }++ semantic := fmt.Sprintf("%s in #%s: %s", senderName, channel.Name, snippet)+ semantic = strings.ReplaceAll(semantic, "\n", " ")++ content := strings.Join(cleaned, "\n\n")+ blob := []byte(content)++ var updatedAt time.Time+ for _, message := range thread {+ if ts := slackMessageTime(message.TS); ts.After(updatedAt) {+ updatedAt = ts+ }+ }++ doc := &SourceDocument{+ SourceID: slackDocID(channel.ID, docTS),+ SemanticIdentifier: semantic,+ Extension: ".txt",+ Blob: blob,+ UpdatedAt: updatedAt,+ SizeBytes: int64(len(blob)),+ Metadata: map[string]any{"Channel": channel.Name},+ }+ return doc, nil+}++func slackMessageTime(ts string) time.Time {+ seconds, err := strconv.ParseFloat(ts, 64)+ if err != nil {+ return time.Time{}+ }+ whole := int64(seconds)+ nanos := int64((seconds - float64(whole)) * 1e9)+ return time.Unix(whole, nanos).UTC()+}++func normalizeSlackChannelNames(channels []string) []string {+ out := make([]string, 0, len(channels))+ for _, name := range channels {+ name = strings.TrimSpace(strings.TrimPrefix(name, "#"))+ if name != "" {+ out = append(out, name)+ }+ }+ return out+}++// ---------------------------------------------------------------------------+// Sync session with checkpoint resume+// ---------------------------------------------------------------------------++type slackSyncSession struct {+ connector *SlackConnector+ request SyncRequest+ channels []slackChannel+ batchSize int+ seenThreads map[string]struct{}++ resumeAfterChannelID string+ channelIndex int+ pending []SourceDocument+ pendingCheckpoint *SyncCheckpoint+ done bool+}++// NextBatch returns the next Slack document batch.+func (s *slackSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {+ for {+ if s.done {+ return SyncBatch{}, io.EOF+ }+ if len(s.pending) == 0 {+ if s.channelIndex >= len(s.channels) {+ s.done = true+ return SyncBatch{}, io.EOF+ }+ channel := s.channels[s.channelIndex]+ s.channelIndex+++ if channel.ID <= s.resumeAfterChannelID {+ continue+ }+ documents, err := s.channelDocuments(ctx, channel)+ if err != nil {+ return SyncBatch{}, err+ }+ if len(documents) == 0 {+ continue+ }+ s.pending = documents+ updatedAt := documents[len(documents)-1].UpdatedAt+ checkpoint := &SyncCheckpoint{+ Cursor: fmt.Sprintf("slack_channel_%s", channel.ID),+ SourceID: fmt.Sprintf("slack_channel_%s", channel.ID),+ UpdatedAt: &updatedAt,+ }+ // The checkpoint only advances once the whole channel is flushed so+ // an interrupted run re-processes a partially committed channel.+ s.pendingCheckpoint = checkpoint+ }+ n := s.batchSize+ if n > len(s.pending) {+ n = len(s.pending)+ }+ chunk := s.pending[:n]+ s.pending = s.pending[n:]+ var checkpoint *SyncCheckpoint+ if len(s.pending) == 0 && s.pendingCheckpoint != nil {+ checkpoint = s.pendingCheckpoint+ s.pendingCheckpoint = nil+ }+ return SyncBatch{Documents: chunk, Checkpoint: checkpoint}, nil+ }+}++// Close closes the Slack sync session.+func (s *slackSyncSession) Close() error {+ return nil+}++func (s *slackSyncSession) applyResume(checkpoint *SyncCheckpoint) {+ sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)+ const prefix = "slack_channel_"+ if !strings.HasPrefix(sourceID, prefix) {+ return+ }+ channelID := strings.TrimPrefix(sourceID, prefix)+ if channelID == "" {+ return+ }+ s.resumeAfterChannelID = channelID+}++// channelDocuments builds the documents for one channel.+func (s *slackSyncSession) channelDocuments(ctx context.Context, channel slackChannel) ([]SourceDocument, error) {+ s.seenThreads = map[string]struct{}{}+ oldest, latest := "", ""+ if !s.request.FromBeginning {+ if s.request.WindowStart != nil {+ oldest = strconv.FormatInt(s.request.WindowStart.Unix(), 10)+ }+ if !s.request.WindowEnd.IsZero() {+ latest = strconv.FormatInt(s.request.WindowEnd.Unix(), 10)+ }+ }+ messages, err := s.connector.channelMessages(ctx, channel, oldest, latest)+ if err != nil {+ if errors.Is(err, errSlackChannelUnavailable) {+ return nil, nil+ }+ return nil, err+ }++ var documents []SourceDocument+ for _, message := range messages {+ doc, err := s.processMessage(ctx, channel, message)+ if err != nil {+ return nil, err+ }+ if doc != nil {+ documents = append(documents, *doc)+ }+ }+ return documents, nil+}++// processMessage converts one top-level message into a document, fetching and+// merging its reply thread when present.+func (s *slackSyncSession) processMessage(ctx context.Context, channel slackChannel, message slackMessage) (*SourceDocument, error) {+ threadOrMessageTS := message.ThreadTS+ if threadOrMessageTS == "" {+ threadOrMessageTS = message.TS+ }+ if _, seen := s.seenThreads[threadOrMessageTS]; seen {+ return nil, nil+ }+ s.seenThreads[threadOrMessageTS] = struct{}{}++ if message.ThreadTS != "" {+ thread, err := s.connector.getThread(ctx, channel.ID, message.ThreadTS)+ if err != nil {+ return nil, err+ }+ filtered := filterSlackMessages(thread)+ if len(filtered) == 0 {+ return nil, nil+ }+ return s.connector.threadToDocument(ctx, channel, filtered, threadOrMessageTS)+ }+ if !acceptSlackMessage(message) {+ return nil, nil+ }+ return s.connector.threadToDocument(ctx, channel, []slackMessage{message}, threadOrMessageTS)+}++func filterSlackMessages(messages []slackMessage) []slackMessage {+ filtered := make([]slackMessage, 0, len(messages))+ for _, message := range messages {+ if acceptSlackMessage(message) {+ filtered = append(filtered, message)+ }+ }+ return filtered+}++type slackPruneSession struct {+ connector *SlackConnector+ channels []slackChannel+ batchSize int++ channelIndex int+ pending []SlimDocument+ done bool+}++// NextBatch returns the next Slack prune snapshot batch, paging channels+// lazily so the whole workspace is never collected in memory at once.+func (s *slackPruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {+ for {+ if s.done {+ return PruneBatch{}, io.EOF+ }+ if len(s.pending) == 0 {+ if s.channelIndex >= len(s.channels) {+ s.done = true+ return PruneBatch{}, io.EOF+ }+ channel := s.channels[s.channelIndex]+ s.channelIndex+++ documents, err := s.connector.pruneChannelDocuments(ctx, channel)+ if err != nil {+ return PruneBatch{}, err+ }+ if len(documents) == 0 {+ continue+ }+ s.pending = documents+ }+ n := s.batchSize+ if n > len(s.pending) {+ n = len(s.pending)+ }+ chunk := s.pending[:n]+ s.pending = s.pending[n:]+ return PruneBatch{Documents: chunk}, nil+ }+}++// Close closes the Slack prune session.+func (s *slackPruneSession) Close() error {+ return nil+}++// pruneChannelDocuments returns the slim snapshot documents for one channel.+func (c *SlackConnector) pruneChannelDocuments(ctx context.Context, channel slackChannel) ([]SlimDocument, error) {+ messages, err := c.channelMessages(ctx, channel, "", "")+ if err != nil {+ if errors.Is(err, errSlackChannelUnavailable) {+ return nil, nil+ }+ return nil, err+ }+ return c.slackPruneDocuments(ctx, channel, messages)+}++// slackPruneDocuments maps channel history to slim snapshot documents. Each+// thread is resolved with the same full-thread view and acceptance logic as+// sync (conversations.replies), so prune identities match sync documents.+func (c *SlackConnector) slackPruneDocuments(ctx context.Context, channel slackChannel, messages []slackMessage) ([]SlimDocument, error) {+ seen := map[string]struct{}{}+ var documents []SlimDocument+ for _, message := range messages {+ rootTS := slackThreadRootTS(message)+ if _, ok := seen[rootTS]; ok {+ continue+ }+ seen[rootTS] = struct{}{}+ accepted := false+ if message.ThreadTS != "" {+ thread, err := c.getThread(ctx, channel.ID, message.ThreadTS)+ if err != nil {+ return nil, err+ }+ accepted = len(filterSlackMessages(thread)) > 0+ } else {+ accepted = acceptSlackMessage(message)+ }+ if accepted {+ documents = append(documents, SlimDocument{SourceID: slackDocID(channel.ID, rootTS)})+ }+ }+ return documents, nil+}++// slackThreadRootTS returns the root timestamp of a message's thread, falling+// back to the message's own timestamp for top-level messages.+func slackThreadRootTS(message slackMessage) string {+ if message.ThreadTS != "" {+ return message.ThreadTS+ }+ return message.TS+}diff --git a/internal/syncer/connector/slack_test.go b/internal/syncer/connector/slack_test.gonew file mode 100644index 0000000000..d0a1be22c1--- /dev/null+++ b/internal/syncer/connector/slack_test.go@@ -0,0 +1,1180 @@+package connector++import (+ "context"+ "errors"+ "io"+ "net/http"+ "net/http/httptest"+ "net/url"+ "strings"+ "sync/atomic"+ "testing"+ "time"+)++func withSlackTestHooks(t *testing.T) {+ t.Helper()+ origTries := slackRetryTries+ origBaseDelay := slackRetryBaseDelay+ origBackoff := slackRetryBackoff+ slackRetryTries = 2+ slackRetryBaseDelay = time.Millisecond+ slackRetryBackoff = 2+ t.Cleanup(func() {+ slackRetryTries = origTries+ slackRetryBaseDelay = origBaseDelay+ slackRetryBackoff = origBackoff+ })+}++func mustSlackConnector(t *testing.T, serverURL string, extra map[string]any) *SlackConnector {+ t.Helper()+ config := map[string]any{+ "batch_size": 2,+ "credentials": map[string]any{+ "slack_bot_token": "xoxb-test",+ },+ }+ for key, value := range extra {+ config[key] = value+ }+ connector, err := NewSlackConnector(config)+ if err != nil {+ t.Fatalf("NewSlackConnector: %v", err)+ }+ if serverURL != "" {+ connector.baseURL = strings.TrimRight(serverURL, "/") + "/api"+ }+ return connector+}++type slackTestFixtures struct {+ authError string+ channelsJSON string+ historyByChannel map[string]string+ repliesByKey map[string]string+ usersByID map[string]string++ listCalls *atomic.Int64+ historyCalls *atomic.Int64+ joinCalls *atomic.Int64+ usersCalls *atomic.Int64+ historyParams []url.Values+}++func newTestSlackServer(t *testing.T, fixtures *slackTestFixtures) *httptest.Server {+ t.Helper()+ mux := http.NewServeMux()+ server := httptest.NewServer(mux)+ t.Cleanup(server.Close)++ mux.HandleFunc("/api/auth.test", func(w http.ResponseWriter, r *http.Request) {+ if fixtures.authError != "" {+ writeSlackError(w, fixtures.authError)+ return+ }+ w.Write([]byte(`{"ok":true,"url":"https://example.slack.com/","team":"Test","user":"bot","team_id":"T1","user_id":"U1"}`))+ })+ mux.HandleFunc("/api/conversations.list", func(w http.ResponseWriter, r *http.Request) {+ if fixtures.listCalls != nil {+ fixtures.listCalls.Add(1)+ }+ body := fixtures.channelsJSON+ if body == "" {+ body = `{"ok":true,"channels":[],"response_metadata":{"next_cursor":""}}`+ }+ w.Write([]byte(body))+ })+ mux.HandleFunc("/api/conversations.history", func(w http.ResponseWriter, r *http.Request) {+ if fixtures.historyCalls != nil {+ fixtures.historyCalls.Add(1)+ }+ if err := r.ParseForm(); err != nil {+ http.Error(w, "bad form", http.StatusBadRequest)+ return+ }+ if fixtures.historyParams != nil {+ params := url.Values{}+ for key, values := range r.Form {+ params[key] = append([]string{}, values...)+ }+ fixtures.historyParams = append(fixtures.historyParams, params)+ }+ body := fixtures.historyByChannel[r.Form.Get("channel")]+ if body == "" {+ body = `{"ok":true,"messages":[],"has_more":false,"response_metadata":{"next_cursor":""}}`+ }+ w.Write([]byte(body))+ })+ mux.HandleFunc("/api/conversations.replies", func(w http.ResponseWriter, r *http.Request) {+ if err := r.ParseForm(); err != nil {+ http.Error(w, "bad form", http.StatusBadRequest)+ return+ }+ key := r.Form.Get("channel") + ":" + r.Form.Get("ts")+ body := fixtures.repliesByKey[key]+ if body == "" {+ body = `{"ok":true,"messages":[],"has_more":false,"response_metadata":{"next_cursor":""}}`+ }+ w.Write([]byte(body))+ })+ mux.HandleFunc("/api/conversations.join", func(w http.ResponseWriter, r *http.Request) {+ if fixtures.joinCalls != nil {+ fixtures.joinCalls.Add(1)+ }+ w.Write([]byte(`{"ok":true,"channel":{"id":"C1"}}`))+ })+ mux.HandleFunc("/api/users.info", func(w http.ResponseWriter, r *http.Request) {+ if fixtures.usersCalls != nil {+ fixtures.usersCalls.Add(1)+ }+ if err := r.ParseForm(); err != nil {+ http.Error(w, "bad form", http.StatusBadRequest)+ return+ }+ body := fixtures.usersByID[r.Form.Get("user")]+ if body == "" {+ body = `{"ok":false,"error":"user_not_found"}`+ }+ w.Write([]byte(body))+ })+ return server+}++func writeSlackError(w http.ResponseWriter, apiError string) {+ w.Write([]byte(`{"ok":false,"error":"` + apiError + `"}`))+}++func slackHistoryBody(messages ...string) string {+ return `{"ok":true,"messages":[` + strings.Join(messages, ",") + `],"has_more":false,"response_metadata":{"next_cursor":""}}`+}++const (+ slackChannelsTwo = `{"ok":true,"channels":[+ {"id":"C1","name":"general","is_archived":false,"is_member":true,"is_private":false},+ {"id":"C2","name":"random","is_archived":false,"is_member":false,"is_private":false}+ ],"response_metadata":{"next_cursor":""}}`++ slackMsgPlain = `{"type":"message","ts":"1700000000.000001","user":"U123","text":"Hello <!channel> from <@U456>"}`+ slackMsgThreadParent = `{"type":"message","ts":"1700000001.000001","user":"U123","text":"Thread root","thread_ts":"1700000001.000001"}`+ slackMsgBot = `{"type":"message","ts":"1700000003.000001","user":"U123","text":"bot noise","bot_id":"B1","bot_profile":{"name":"Some Bot"}}`+ slackMsgJoin = `{"type":"message","subtype":"channel_join","ts":"1700000004.000001","user":"U123","text":"<@U123> has joined the channel"}`+ slackMsgDanswerBot = `{"type":"message","ts":"1700000005.000001","user":"U123","text":"danswer note","bot_id":"B2","bot_profile":{"name":"DanswerBot Testing"}}`+ slackMsgNormal = `{"type":"message","ts":"1700000006.000001","user":"U123","text":"A normal message"}`+)++func slackThreadBody() string {+ return `{"ok":true,"messages":[+ {"type":"message","ts":"1700000001.000001","user":"U123","text":"Thread root","thread_ts":"1700000001.000001"},+ {"type":"message","ts":"1700000002.000002","user":"U456","text":"A reply"}+ ],"has_more":false,"response_metadata":{"next_cursor":""}}`+}++func slackUserAlice() string {+ return `{"ok":true,"user":{"id":"U123","real_name":"Alice","profile":{"display_name":"Alice","real_name":"Alice"}}}`+}++func slackUserBob() string {+ return `{"ok":true,"user":{"id":"U456","real_name":"Bob","profile":{"display_name":"","real_name":"Bob"}}}`+}++func collectSlackDocuments(t *testing.T, session SyncSession) []SourceDocument {+ t.Helper()+ var documents []SourceDocument+ for {+ batch, err := session.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ return documents+ }+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ documents = append(documents, batch.Documents...)+ }+}++func TestSlackConnectorValidate(t *testing.T) {+ withSlackTestHooks(t)+ fixtures := &slackTestFixtures{+ channelsJSON: slackChannelsTwo,+ usersByID: map[string]string{"USLACKBOT": slackUserAlice()},+ listCalls: &atomic.Int64{},+ }+ server := newTestSlackServer(t, fixtures)+ connector := mustSlackConnector(t, server.URL, nil)++ if err := connector.Validate(context.Background()); err != nil {+ t.Fatalf("Validate: %v", err)+ }+ if fixtures.listCalls == nil || fixtures.listCalls.Load() == 0 {+ t.Fatalf("conversations.list was not called during validation")+ }+}++func TestSlackConnectorValidateMissingToken(t *testing.T) {+ connector := mustSlackConnector(t, "", map[string]any{"credentials": map[string]any{}})+ err := connector.Validate(context.Background())+ if err == nil {+ t.Fatalf("expected missing credential error")+ }+ var missing *ConnectorMissingCredentialError+ if !errors.As(err, &missing) {+ t.Fatalf("error = %T, want *ConnectorMissingCredentialError", err)+ }+}++func TestSlackConnectorValidateNil(t *testing.T) {+ var connector *SlackConnector+ if err := connector.Validate(context.Background()); err == nil {+ t.Fatalf("expected error for nil connector")+ }+}++func TestSlackConnectorValidateInvalidAuth(t *testing.T) {+ withSlackTestHooks(t)+ fixtures := &slackTestFixtures{authError: "invalid_auth"}+ server := newTestSlackServer(t, fixtures)+ connector := mustSlackConnector(t, server.URL, nil)++ err := connector.Validate(context.Background())+ if err == nil || !strings.Contains(err.Error(), "Invalid Slack bot token") {+ t.Fatalf("Validate error = %v, want invalid bot token error", err)+ }+}++func TestSlackConnectorValidateMissingScope(t *testing.T) {+ withSlackTestHooks(t)+ fixtures := &slackTestFixtures{+ channelsJSON: slackChannelsTwo,+ usersByID: map[string]string{"USLACKBOT": `{"ok":false,"error":"missing_scope"}`},+ }+ server := newTestSlackServer(t, fixtures)+ connector := mustSlackConnector(t, server.URL, nil)++ err := connector.Validate(context.Background())+ if err == nil || !strings.Contains(strings.ToLower(err.Error()), "scope") {+ t.Fatalf("Validate error = %v, want missing scope error", err)+ }+}++func TestSlackConnectorMissingScopeMessageNamesNeededScope(t *testing.T) {+ withSlackTestHooks(t)+ body := []byte(`{"ok":false,"error":"missing_scope","needed":"channels:read","provided":"channels:write"}`)+ err := classifySlackAPIError("conversations.list", "missing_scope", body)+ var valErr *ConnectorValidationError+ if !errors.As(err, &valErr) {+ t.Fatalf("error = %T, want *ConnectorValidationError", err)+ }+ if !strings.Contains(valErr.Message, "channels:read") {+ t.Fatalf("message = %q, want it to contain %q", valErr.Message, "channels:read")+ }+ if strings.Contains(valErr.Message, "channels:write") {+ t.Fatalf("message = %q, want it not to mention provided scopes", valErr.Message)+ }+}++func TestSlackConnectorMissingScopeMessageFallsBackWithoutNeeded(t *testing.T) {+ withSlackTestHooks(t)+ body := []byte(`{"ok":false,"error":"missing_scope"}`)+ err := classifySlackAPIError("users.info", "missing_scope", body)+ var valErr *ConnectorValidationError+ if !errors.As(err, &valErr) {+ t.Fatalf("error = %T, want *ConnectorValidationError", err)+ }+ if !strings.Contains(valErr.Message, "users.info") || !strings.Contains(valErr.Message, "missing_scope") {+ t.Fatalf("message = %q, want generic missing scope message", valErr.Message)+ }+}++func TestSlackConnectorValidateRateLimitedProceeds(t *testing.T) {+ withSlackTestHooks(t)+ fixtures := &slackTestFixtures{authError: "ratelimited"}+ server := newTestSlackServer(t, fixtures)+ connector := mustSlackConnector(t, server.URL, nil)++ if err := connector.Validate(context.Background()); err != nil {+ t.Fatalf("Validate should proceed after rate limit, got %v", err)+ }+}++func TestSlackConnectorOpenSyncBuildsThreadAndMessageDocs(t *testing.T) {+ withSlackTestHooks(t)+ joinCalls := &atomic.Int64{}+ fixtures := &slackTestFixtures{+ channelsJSON: slackChannelsTwo,+ historyByChannel: map[string]string{+ "C1": slackHistoryBody(slackMsgThreadParent, slackMsgPlain),+ },+ repliesByKey: map[string]string{"C1:1700000001.000001": slackThreadBody()},+ usersByID: map[string]string{"U123": slackUserAlice(), "U456": slackUserBob()},+ joinCalls: joinCalls,+ }+ server := newTestSlackServer(t, fixtures)+ connector := mustSlackConnector(t, server.URL, nil)++ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ defer session.Close()+ documents := collectSlackDocuments(t, session)+ if len(documents) != 2 {+ t.Fatalf("documents = %d, want 2", len(documents))+ }++ threadDoc := documents[0]+ if threadDoc.SourceID != "C1__1700000001.000001" {+ t.Fatalf("thread SourceID = %q", threadDoc.SourceID)+ }+ if threadDoc.SemanticIdentifier != "Alice in #general: Thread root" {+ t.Fatalf("thread semantic = %q", threadDoc.SemanticIdentifier)+ }+ if threadDoc.Extension != ".txt" {+ t.Fatalf("thread extension = %q", threadDoc.Extension)+ }+ if string(threadDoc.Blob) != "Thread root\n\nA reply" {+ t.Fatalf("thread blob = %q", string(threadDoc.Blob))+ }+ if threadDoc.UpdatedAt.Unix() != 1700000002 {+ t.Fatalf("thread updated_at = %v", threadDoc.UpdatedAt)+ }+ if threadDoc.Metadata["Channel"] != "general" {+ t.Fatalf("thread metadata = %v", threadDoc.Metadata)+ }++ plainDoc := documents[1]+ if plainDoc.SourceID != "C1__1700000000.000001" {+ t.Fatalf("plain SourceID = %q", plainDoc.SourceID)+ }+ if plainDoc.SemanticIdentifier != "Alice in #general: Hello @channel from @Bob" {+ t.Fatalf("plain semantic = %q", plainDoc.SemanticIdentifier)+ }+ if string(plainDoc.Blob) != "Hello @channel from @Bob" {+ t.Fatalf("plain blob = %q", string(plainDoc.Blob))+ }++ if joinCalls.Load() != 1 {+ t.Fatalf("conversations.join calls = %d, want 1 (C2 is not a member)", joinCalls.Load())+ }+}++func TestSlackConnectorOpenSyncWindowBounds(t *testing.T) {+ withSlackTestHooks(t)+ fixtures := &slackTestFixtures{+ channelsJSON: slackChannelsTwo,+ historyByChannel: map[string]string{"C1": slackHistoryBody(slackMsgPlain)},+ historyParams: []url.Values{},+ }+ server := newTestSlackServer(t, fixtures)+ connector := mustSlackConnector(t, server.URL, nil)++ windowStart := time.Unix(1700000000, 0).UTC()+ windowEnd := time.Unix(1700000100, 0).UTC()+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ WindowStart: &windowStart,+ WindowEnd: windowEnd,+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ defer session.Close()+ collectSlackDocuments(t, session)++ // Both C1 and C2 are enumerated; C1 is processed first.+ if len(fixtures.historyParams) != 2 {+ t.Fatalf("history calls = %d, want 2", len(fixtures.historyParams))+ }+ if got := fixtures.historyParams[0].Get("oldest"); got != "1700000000" {+ t.Fatalf("oldest = %q, want 1700000000", got)+ }+ if got := fixtures.historyParams[0].Get("latest"); got != "1700000100" {+ t.Fatalf("latest = %q, want 1700000100", got)+ }+}++func TestSlackConnectorOpenSyncFromBeginningHasNoWindow(t *testing.T) {+ withSlackTestHooks(t)+ fixtures := &slackTestFixtures{+ channelsJSON: slackChannelsTwo,+ historyByChannel: map[string]string{"C1": slackHistoryBody(slackMsgPlain)},+ historyParams: []url.Values{},+ }+ server := newTestSlackServer(t, fixtures)+ connector := mustSlackConnector(t, server.URL, nil)++ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ defer session.Close()+ collectSlackDocuments(t, session)++ if got := fixtures.historyParams[0].Get("oldest"); got != "" {+ t.Fatalf("oldest = %q, want empty for full sync", got)+ }+ if got := fixtures.historyParams[0].Get("latest"); got != "" {+ t.Fatalf("latest = %q, want empty for full sync", got)+ }+}++func TestSlackConnectorOpenSyncFiltersMessages(t *testing.T) {+ withSlackTestHooks(t)+ fixtures := &slackTestFixtures{+ channelsJSON: slackChannelsTwo,+ historyByChannel: map[string]string{"C1": slackHistoryBody(slackMsgBot, slackMsgJoin, slackMsgDanswerBot, slackMsgNormal)},+ usersByID: map[string]string{"U123": slackUserAlice()},+ }+ server := newTestSlackServer(t, fixtures)+ connector := mustSlackConnector(t, server.URL, nil)++ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ defer session.Close()+ documents := collectSlackDocuments(t, session)+ if len(documents) != 2 {+ t.Fatalf("documents = %d, want 2 (Danswer bot and normal message)", len(documents))+ }+ if documents[0].SourceID != "C1__1700000005.000001" {+ t.Fatalf("first doc SourceID = %q", documents[0].SourceID)+ }+ if documents[1].SourceID != "C1__1700000006.000001" {+ t.Fatalf("second doc SourceID = %q", documents[1].SourceID)+ }+}++func TestSlackConnectorOpenSyncChannelSelection(t *testing.T) {+ withSlackTestHooks(t)+ historyCalls := &atomic.Int64{}+ fixtures := &slackTestFixtures{+ channelsJSON: slackChannelsTwo,+ historyByChannel: map[string]string{"C1": slackHistoryBody(slackMsgPlain)},+ usersByID: map[string]string{"U123": slackUserAlice()},+ historyCalls: historyCalls,+ }+ server := newTestSlackServer(t, fixtures)+ connector := mustSlackConnector(t, server.URL, map[string]any{"channels": "general"})++ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ defer session.Close()+ documents := collectSlackDocuments(t, session)+ if len(documents) != 1 || documents[0].SourceID != "C1__1700000000.000001" {+ t.Fatalf("documents = %+v", documents)+ }+ if historyCalls.Load() != 1 {+ t.Fatalf("history calls = %d, want 1", historyCalls.Load())+ }+}++func TestSlackConnectorOpenSyncChannelNotFound(t *testing.T) {+ withSlackTestHooks(t)+ fixtures := &slackTestFixtures{channelsJSON: slackChannelsTwo}+ server := newTestSlackServer(t, fixtures)+ connector := mustSlackConnector(t, server.URL, map[string]any{"channels": "missing"})++ _, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err == nil || !strings.Contains(err.Error(), "not found") {+ t.Fatalf("OpenSync error = %v, want channel not found", err)+ }+}++func TestSlackConnectorOpenSyncRegexChannels(t *testing.T) {+ withSlackTestHooks(t)+ historyCalls := &atomic.Int64{}+ fixtures := &slackTestFixtures{+ channelsJSON: slackChannelsTwo,+ historyByChannel: map[string]string{"C1": slackHistoryBody(slackMsgPlain)},+ usersByID: map[string]string{"U123": slackUserAlice()},+ historyCalls: historyCalls,+ }+ server := newTestSlackServer(t, fixtures)+ connector := mustSlackConnector(t, server.URL, map[string]any{+ "channels": "gen.*",+ "channel_regex_enabled": true,+ })++ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ defer session.Close()+ documents := collectSlackDocuments(t, session)+ if len(documents) != 1 || documents[0].SourceID != "C1__1700000000.000001" {+ t.Fatalf("documents = %+v", documents)+ }+}++func TestSlackConnectorOpenSyncResumesFromCheckpoint(t *testing.T) {+ withSlackTestHooks(t)+ historyCalls := &atomic.Int64{}+ fixtures := &slackTestFixtures{+ channelsJSON: `{"ok":true,"channels":[+ {"id":"C1","name":"general","is_member":true},+ {"id":"C2","name":"random","is_member":true},+ {"id":"C3","name":"dev","is_member":true}+ ],"response_metadata":{"next_cursor":""}}`,+ historyByChannel: map[string]string{+ "C1": slackHistoryBody(slackMsgPlain),+ "C3": slackHistoryBody(slackMsgNormal),+ },+ usersByID: map[string]string{"U123": slackUserAlice()},+ historyCalls: historyCalls,+ }+ server := newTestSlackServer(t, fixtures)+ connector := mustSlackConnector(t, server.URL, nil)++ session, err := connector.OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ Resume: &SyncCheckpoint{+ Cursor: "slack_channel_C2",+ SourceID: "slack_channel_C2",+ },+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ defer session.Close()+ documents := collectSlackDocuments(t, session)+ if len(documents) != 1 || documents[0].SourceID != "C3__1700000006.000001" {+ t.Fatalf("documents = %+v, want only channel C3", documents)+ }+ if historyCalls.Load() != 1 {+ t.Fatalf("history calls = %d, want 1 (only C3)", historyCalls.Load())+ }+}++func TestSlackConnectorOpenSyncCheckpointAdvancesPerChannel(t *testing.T) {+ withSlackTestHooks(t)+ fixtures := &slackTestFixtures{+ channelsJSON: slackChannelsTwo,+ historyByChannel: map[string]string{+ "C1": slackHistoryBody(slackMsgPlain, slackMsgNormal),+ "C2": slackHistoryBody(slackMsgThreadParent),+ },+ repliesByKey: map[string]string{"C2:1700000001.000001": slackThreadBody()},+ usersByID: map[string]string{"U123": slackUserAlice()},+ }+ server := newTestSlackServer(t, fixtures)+ connector := mustSlackConnector(t, server.URL, map[string]any{"batch_size": 1})++ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ defer session.Close()++ var checkpoints []string+ for {+ batch, err := session.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) == 0 {+ t.Fatalf("empty batch")+ }+ if batch.Checkpoint != nil {+ checkpoints = append(checkpoints, batch.Checkpoint.Cursor)+ }+ }+ // batch_size=1: C1 emits two batches (no checkpoint on the first), C2 emits+ // one; the checkpoint advances only on the last batch of each channel.+ if len(checkpoints) != 2 || checkpoints[0] != "slack_channel_C1" || checkpoints[1] != "slack_channel_C2" {+ t.Fatalf("checkpoints = %v, want [slack_channel_C1 slack_channel_C2]", checkpoints)+ }+}++func TestSlackConnectorOpenPruneSlimSnapshot(t *testing.T) {+ withSlackTestHooks(t)+ fixtures := &slackTestFixtures{+ channelsJSON: slackChannelsTwo,+ historyByChannel: map[string]string{"C1": slackHistoryBody(slackMsgPlain, slackMsgBot)},+ }+ server := newTestSlackServer(t, fixtures)+ connector := mustSlackConnector(t, server.URL, nil)++ session, err := connector.OpenPrune(context.Background(), PruneRequest{TaskID: "t1"})+ if err != nil {+ t.Fatalf("OpenPrune: %v", err)+ }+ defer session.Close()++ var slim []SlimDocument+ for {+ batch, err := session.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ slim = append(slim, batch.Documents...)+ }+ if len(slim) != 1 || slim[0].SourceID != "C1__1700000000.000001" {+ t.Fatalf("slim documents = %+v", slim)+ }+}++func TestSlackConnectorOpenPruneThreadRootIdentity(t *testing.T) {+ withSlackTestHooks(t)+ history := slackHistoryBody(+ slackMsgThreadParent, // root: ts == thread_ts == 1700000001.000001+ `{"type":"message","ts":"1700000002.000002","user":"U456","text":"A reply","thread_ts":"1700000001.000001"}`,+ slackMsgPlain,+ )+ fixtures := &slackTestFixtures{+ channelsJSON: slackChannelsTwo,+ historyByChannel: map[string]string{"C1": history},+ repliesByKey: map[string]string{"C1:1700000001.000001": slackThreadBody()},+ }+ server := newTestSlackServer(t, fixtures)+ connector := mustSlackConnector(t, server.URL, nil)++ session, err := connector.OpenPrune(context.Background(), PruneRequest{TaskID: "t1"})+ if err != nil {+ t.Fatalf("OpenPrune: %v", err)+ }+ defer session.Close()++ var slim []SlimDocument+ for {+ batch, err := session.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ slim = append(slim, batch.Documents...)+ }+ // The thread is emitted once under its root timestamp; the reply and plain+ // message must not leak their own timestamps.+ want := []string{"C1__1700000001.000001", "C1__1700000000.000001"}+ if len(slim) != len(want) {+ t.Fatalf("slim documents = %+v, want %v", slim, want)+ }+ for i, doc := range slim {+ if doc.SourceID != want[i] {+ t.Fatalf("slim[%d].SourceID = %q, want %q", i, doc.SourceID, want[i])+ }+ }+}++func TestSlackConnectorOpenPruneBotRootThread(t *testing.T) {+ withSlackTestHooks(t)+ history := slackHistoryBody(+ `{"type":"message","ts":"1700000001.000001","user":"U123","text":"bot root","thread_ts":"1700000001.000001","bot_id":"B1","bot_profile":{"name":"Some Bot"}}`,+ `{"type":"message","ts":"1700000002.000002","user":"U456","text":"A reply","thread_ts":"1700000001.000001"}`,+ )+ fixtures := &slackTestFixtures{+ channelsJSON: slackChannelsTwo,+ historyByChannel: map[string]string{"C1": history},+ repliesByKey: map[string]string{+ "C1:1700000001.000001": `{"ok":true,"messages":[+ {"type":"message","ts":"1700000001.000001","user":"U123","text":"bot root","thread_ts":"1700000001.000001","bot_id":"B1","bot_profile":{"name":"Some Bot"}},+ {"type":"message","ts":"1700000002.000002","user":"U456","text":"A reply","thread_ts":"1700000001.000001"}+ ],"has_more":false,"response_metadata":{"next_cursor":""}}`,+ },+ }+ server := newTestSlackServer(t, fixtures)+ connector := mustSlackConnector(t, server.URL, nil)++ session, err := connector.OpenPrune(context.Background(), PruneRequest{TaskID: "t1"})+ if err != nil {+ t.Fatalf("OpenPrune: %v", err)+ }+ defer session.Close()++ var slim []SlimDocument+ for {+ batch, err := session.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ slim = append(slim, batch.Documents...)+ }+ // A thread with an accepted reply is pruned under the root timestamp even+ // when the root message itself is filtered out.+ if len(slim) != 1 || slim[0].SourceID != "C1__1700000001.000001" {+ t.Fatalf("slim documents = %+v, want [C1__1700000001.000001]", slim)+ }+}++func TestSlackConnectorOpenPrunePagesChannelsLazily(t *testing.T) {+ withSlackTestHooks(t)+ historyCalls := &atomic.Int64{}+ fixtures := &slackTestFixtures{+ channelsJSON: slackChannelsTwo,+ historyByChannel: map[string]string{"C1": slackHistoryBody(slackMsgPlain)},+ historyCalls: historyCalls,+ }+ server := newTestSlackServer(t, fixtures)+ connector := mustSlackConnector(t, server.URL, nil)++ session, err := connector.OpenPrune(context.Background(), PruneRequest{TaskID: "t1"})+ if err != nil {+ t.Fatalf("OpenPrune: %v", err)+ }+ defer session.Close()+ if historyCalls.Load() != 0 {+ t.Fatalf("history calls after OpenPrune = %d, want 0 (lazy paging)", historyCalls.Load())+ }++ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if historyCalls.Load() == 0 {+ t.Fatalf("history calls after NextBatch = 0, want lazy channel fetch")+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "C1__1700000000.000001" {+ t.Fatalf("batch documents = %+v", batch.Documents)+ }+}++func TestSlackConnectorOpenPruneResolvesRepliesFromThreadAPI(t *testing.T) {+ withSlackTestHooks(t)+ // The bot root is not accepted, and history carries no replies. Only+ // conversations.replies reveals the accepted reply, so prune must resolve+ // the thread like sync does instead of trusting history alone.+ history := slackHistoryBody(+ `{"type":"message","ts":"1700000001.000001","user":"U123","text":"bot root","thread_ts":"1700000001.000001","bot_id":"B1","bot_profile":{"name":"Some Bot"}}`,+ )+ fixtures := &slackTestFixtures{+ channelsJSON: slackChannelsTwo,+ historyByChannel: map[string]string{"C1": history},+ repliesByKey: map[string]string{+ "C1:1700000001.000001": `{"ok":true,"messages":[+ {"type":"message","ts":"1700000001.000001","user":"U123","text":"bot root","thread_ts":"1700000001.000001","bot_id":"B1","bot_profile":{"name":"Some Bot"}},+ {"type":"message","ts":"1700000002.000002","user":"U456","text":"A reply","thread_ts":"1700000001.000001"}+ ],"has_more":false,"response_metadata":{"next_cursor":""}}`,+ },+ }+ server := newTestSlackServer(t, fixtures)+ connector := mustSlackConnector(t, server.URL, nil)++ session, err := connector.OpenPrune(context.Background(), PruneRequest{TaskID: "t1"})+ if err != nil {+ t.Fatalf("OpenPrune: %v", err)+ }+ defer session.Close()++ var slim []SlimDocument+ for {+ batch, err := session.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ slim = append(slim, batch.Documents...)+ }+ if len(slim) != 1 || slim[0].SourceID != "C1__1700000001.000001" {+ t.Fatalf("slim documents = %+v, want [C1__1700000001.000001]", slim)+ }+}++func TestSlackConnectorOpenPrunePropagatesThreadResolutionError(t *testing.T) {+ withSlackTestHooks(t)+ mux := http.NewServeMux()+ mux.HandleFunc("/api/auth.test", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(`{"ok":true}`))+ })+ mux.HandleFunc("/api/conversations.list", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(slackChannelsTwo))+ })+ mux.HandleFunc("/api/conversations.join", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(`{"ok":true}`))+ })+ mux.HandleFunc("/api/conversations.history", func(w http.ResponseWriter, r *http.Request) {+ if r.FormValue("channel") != "C1" {+ w.Write([]byte(`{"ok":true,"messages":[],"has_more":false,"response_metadata":{"next_cursor":""}}`))+ return+ }+ w.Write([]byte(slackHistoryBody(slackMsgThreadParent)))+ })+ mux.HandleFunc("/api/conversations.replies", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(`{"ok":false,"error":"invalid_auth"}`))+ })+ mux.HandleFunc("/api/users.info", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(slackUserAlice()))+ })+ server := httptest.NewServer(mux)+ t.Cleanup(server.Close)++ connector := mustSlackConnector(t, server.URL, nil)+ session, err := connector.OpenPrune(context.Background(), PruneRequest{TaskID: "t1"})+ if err != nil {+ t.Fatalf("OpenPrune: %v", err)+ }+ defer session.Close()++ _, err = session.NextBatch(context.Background())+ if err == nil {+ t.Fatalf("NextBatch: want thread resolution error, got nil")+ }+}++func TestSlackConnectorTextCleaning(t *testing.T) {+ withSlackTestHooks(t)+ usersCalls := &atomic.Int64{}+ fixtures := &slackTestFixtures{usersByID: map[string]string{"U123": slackUserAlice()}, usersCalls: usersCalls}+ server := newTestSlackServer(t, fixtures)+ connector := mustSlackConnector(t, server.URL, nil)++ text := "Hi <@U123> in <#C1|general> <!channel> <!here> <!everyone> <!date|123|formatted> and <@U123> again"+ got, err := connector.indexClean(context.Background(), text)+ if err != nil {+ t.Fatalf("indexClean: %v", err)+ }+ want := "Hi @Alice in #general @channel @here @everyone 123|formatted and @Alice again"+ if got != want {+ t.Fatalf("indexClean = %q, want %q", got, want)+ }+ if usersCalls.Load() != 1 {+ t.Fatalf("users.info calls = %d, want 1 (cached)", usersCalls.Load())+ }+}++func TestSlackConnectorValidateConnectorSetting(t *testing.T) {+ withSlackTestHooks(t)+ fixtures := &slackTestFixtures{channelsJSON: slackChannelsTwo}+ server := newTestSlackServer(t, fixtures)+ connector := mustSlackConnector(t, server.URL, nil)++ if err := connector.ValidateConnectorSetting(context.Background(), map[string]any{}); err != nil {+ t.Fatalf("ValidateConnectorSetting: %v", err)+ }+}++func TestSlackConnectorHistoryPagination(t *testing.T) {+ withSlackTestHooks(t)+ var calls atomic.Int64+ mux := http.NewServeMux()+ mux.HandleFunc("/api/auth.test", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(`{"ok":true}`))+ })+ mux.HandleFunc("/api/conversations.list", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(slackChannelsTwo))+ })+ mux.HandleFunc("/api/conversations.history", func(w http.ResponseWriter, r *http.Request) {+ if r.FormValue("channel") != "C1" {+ w.Write([]byte(`{"ok":true,"messages":[],"has_more":false,"response_metadata":{"next_cursor":""}}`))+ return+ }+ if calls.Add(1) == 1 {+ w.Write([]byte(`{"ok":true,"messages":[` + slackMsgPlain + `],"has_more":true,"response_metadata":{"next_cursor":"nextpage"}}`))+ return+ }+ w.Write([]byte(`{"ok":true,"messages":[` + slackMsgNormal + `],"has_more":false,"response_metadata":{"next_cursor":""}}`))+ })+ mux.HandleFunc("/api/conversations.join", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(`{"ok":true}`))+ })+ mux.HandleFunc("/api/users.info", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(slackUserAlice()))+ })+ server := httptest.NewServer(mux)+ t.Cleanup(server.Close)++ connector := mustSlackConnector(t, server.URL, nil)+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ defer session.Close()+ documents := collectSlackDocuments(t, session)+ if len(documents) != 2 {+ t.Fatalf("documents = %d, want 2 across two history pages", len(documents))+ }+ if calls.Load() != 2 {+ t.Fatalf("C1 history calls = %d, want 2", calls.Load())+ }+}++func TestSlackConnectorChannelListPagination(t *testing.T) {+ withSlackTestHooks(t)+ var calls atomic.Int64+ mux := http.NewServeMux()+ mux.HandleFunc("/api/auth.test", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(`{"ok":true}`))+ })+ mux.HandleFunc("/api/conversations.list", func(w http.ResponseWriter, r *http.Request) {+ if calls.Add(1) == 1 {+ w.Write([]byte(`{"ok":true,"channels":[{"id":"C1","name":"general","is_member":true}],"response_metadata":{"next_cursor":"nextpage"}}`))+ return+ }+ w.Write([]byte(`{"ok":true,"channels":[{"id":"C2","name":"random","is_member":true}],"response_metadata":{"next_cursor":""}}`))+ })+ mux.HandleFunc("/api/conversations.history", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(`{"ok":true,"messages":[],"has_more":false,"response_metadata":{"next_cursor":""}}`))+ })+ mux.HandleFunc("/api/conversations.join", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(`{"ok":true}`))+ })+ mux.HandleFunc("/api/users.info", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(slackUserAlice()))+ })+ server := httptest.NewServer(mux)+ t.Cleanup(server.Close)++ connector := mustSlackConnector(t, server.URL, nil)+ channels, err := connector.resolvedChannels(context.Background())+ if err != nil {+ t.Fatalf("resolvedChannels: %v", err)+ }+ if len(channels) != 2 || channels[0].ID != "C1" || channels[1].ID != "C2" {+ t.Fatalf("channels = %+v", channels)+ }+}++func TestSlackConnectorRetriesTransientHistory(t *testing.T) {+ withSlackTestHooks(t)+ var calls atomic.Int64+ mux := http.NewServeMux()+ mux.HandleFunc("/api/auth.test", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(`{"ok":true}`))+ })+ mux.HandleFunc("/api/conversations.list", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(slackChannelsTwo))+ })+ mux.HandleFunc("/api/conversations.history", func(w http.ResponseWriter, r *http.Request) {+ if r.FormValue("channel") != "C1" {+ w.Write([]byte(`{"ok":true,"messages":[],"has_more":false,"response_metadata":{"next_cursor":""}}`))+ return+ }+ if calls.Add(1) == 1 {+ w.WriteHeader(http.StatusTooManyRequests)+ w.Write([]byte(`{"ok":false,"error":"ratelimited"}`))+ return+ }+ w.Write([]byte(`{"ok":true,"messages":[` + slackMsgPlain + `],"has_more":false,"response_metadata":{"next_cursor":""}}`))+ })+ mux.HandleFunc("/api/conversations.join", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(`{"ok":true}`))+ })+ mux.HandleFunc("/api/users.info", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(slackUserAlice()))+ })+ server := httptest.NewServer(mux)+ t.Cleanup(server.Close)++ connector := mustSlackConnector(t, server.URL, nil)+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ defer session.Close()+ documents := collectSlackDocuments(t, session)+ if len(documents) != 1 {+ t.Fatalf("documents = %d, want 1 after transient retry", len(documents))+ }+ if calls.Load() != 2 {+ t.Fatalf("C1 history calls = %d, want 2 (429 then success)", calls.Load())+ }+}++func TestSlackConnectorOpenSyncSkipsUnjoinableChannel(t *testing.T) {+ withSlackTestHooks(t)+ joinCalls := &atomic.Int64{}+ mux := http.NewServeMux()+ mux.HandleFunc("/api/auth.test", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(`{"ok":true}`))+ })+ mux.HandleFunc("/api/conversations.list", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(slackChannelsTwo))+ })+ mux.HandleFunc("/api/conversations.join", func(w http.ResponseWriter, r *http.Request) {+ joinCalls.Add(1)+ if r.FormValue("channel") == "C2" {+ w.Write([]byte(`{"ok":false,"error":"is_archived"}`))+ return+ }+ w.Write([]byte(`{"ok":true}`))+ })+ mux.HandleFunc("/api/conversations.history", func(w http.ResponseWriter, r *http.Request) {+ if r.FormValue("channel") != "C1" {+ w.Write([]byte(`{"ok":true,"messages":[],"has_more":false,"response_metadata":{"next_cursor":""}}`))+ return+ }+ w.Write([]byte(slackHistoryBody(slackMsgPlain)))+ })+ mux.HandleFunc("/api/users.info", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(slackUserAlice()))+ })+ server := httptest.NewServer(mux)+ t.Cleanup(server.Close)++ connector := mustSlackConnector(t, server.URL, nil)+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ defer session.Close()+ documents := collectSlackDocuments(t, session)+ if len(documents) != 1 || documents[0].SourceID != "C1__1700000000.000001" {+ t.Fatalf("documents = %+v, want only C1", documents)+ }+ if joinCalls.Load() != 1 {+ t.Fatalf("join calls = %d, want 1 (only C2)", joinCalls.Load())+ }+}++func TestSlackConnectorOpenPruneSkipsUnjoinableChannel(t *testing.T) {+ withSlackTestHooks(t)+ joinCalls := &atomic.Int64{}+ mux := http.NewServeMux()+ mux.HandleFunc("/api/auth.test", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(`{"ok":true}`))+ })+ mux.HandleFunc("/api/conversations.list", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(slackChannelsTwo))+ })+ mux.HandleFunc("/api/conversations.join", func(w http.ResponseWriter, r *http.Request) {+ joinCalls.Add(1)+ if r.FormValue("channel") == "C2" {+ w.Write([]byte(`{"ok":false,"error":"method_not_supported_for_channel_type"}`))+ return+ }+ w.Write([]byte(`{"ok":true}`))+ })+ mux.HandleFunc("/api/conversations.history", func(w http.ResponseWriter, r *http.Request) {+ if r.FormValue("channel") != "C1" {+ w.Write([]byte(`{"ok":true,"messages":[],"has_more":false,"response_metadata":{"next_cursor":""}}`))+ return+ }+ w.Write([]byte(slackHistoryBody(slackMsgPlain)))+ })+ mux.HandleFunc("/api/users.info", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(slackUserAlice()))+ })+ server := httptest.NewServer(mux)+ t.Cleanup(server.Close)++ connector := mustSlackConnector(t, server.URL, nil)+ session, err := connector.OpenPrune(context.Background(), PruneRequest{TaskID: "t1"})+ if err != nil {+ t.Fatalf("OpenPrune: %v", err)+ }+ defer session.Close()+ var slim []SlimDocument+ for {+ batch, err := session.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ slim = append(slim, batch.Documents...)+ }+ if len(slim) != 1 || slim[0].SourceID != "C1__1700000000.000001" {+ t.Fatalf("slim documents = %+v, want only C1", slim)+ }+}++func TestSlackConnectorJoinPropagatesUnrelatedErrors(t *testing.T) {+ withSlackTestHooks(t)+ mux := http.NewServeMux()+ mux.HandleFunc("/api/auth.test", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(`{"ok":true}`))+ })+ mux.HandleFunc("/api/conversations.list", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(slackChannelsTwo))+ })+ mux.HandleFunc("/api/conversations.join", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(`{"ok":false,"error":"invalid_auth"}`))+ })+ mux.HandleFunc("/api/conversations.history", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(slackHistoryBody(slackMsgPlain)))+ })+ mux.HandleFunc("/api/users.info", func(w http.ResponseWriter, r *http.Request) {+ w.Write([]byte(slackUserAlice()))+ })+ server := httptest.NewServer(mux)+ t.Cleanup(server.Close)++ connector := mustSlackConnector(t, server.URL, nil)+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ defer session.Close()+ // C2 is not a member, so join fails with invalid_auth, which is unrelated+ // to channel availability and must abort the run.+ var gotErr error+ for {+ _, nextErr := session.NextBatch(context.Background())+ if errors.Is(nextErr, io.EOF) {+ break+ }+ if nextErr != nil {+ gotErr = nextErr+ break+ }+ }+ if gotErr == nil {+ t.Fatalf("NextBatch: want error, got nil")+ }+ if errors.Is(gotErr, errSlackChannelUnavailable) {+ t.Fatalf("NextBatch error = %v, want unrelated error to propagate unwrapped", gotErr)+ }+}++func TestSlackConnectorJoinRestrictedToSelectedChannels(t *testing.T) {+ withSlackTestHooks(t)+ joinCalls := &atomic.Int64{}+ mux := http.NewServeMux()+ mux.HandleFunc("/api/conversations.join", func(w http.ResponseWriter, r *http.Request) {+ joinCalls.Add(1)+ w.Write([]byte(`{"ok":true}`))+ })+ server := httptest.NewServer(mux)+ t.Cleanup(server.Close)++ connector := mustSlackConnector(t, server.URL, map[string]any{"channels": "general"})+ ctx := context.Background()+ selected := slackChannel{ID: "C1", Name: "general", IsMember: false}+ unselected := slackChannel{ID: "C9", Name: "random", IsMember: false}+ if err := connector.joinChannel(ctx, unselected); err != nil {+ t.Fatalf("joinChannel unselected: %v", err)+ }+ if joinCalls.Load() != 0 {+ t.Fatalf("join calls after unselected channel = %d, want 0", joinCalls.Load())+ }+ if err := connector.joinChannel(ctx, selected); err != nil {+ t.Fatalf("joinChannel selected: %v", err)+ }+ if joinCalls.Load() != 1 {+ t.Fatalf("join calls after selected channel = %d, want 1", joinCalls.Load())+ }++ // Regex selection restricts joins the same way.+ regexConnector := mustSlackConnector(t, server.URL, map[string]any{+ "channels": "gen.*",+ "channel_regex_enabled": true,+ })+ if err := regexConnector.joinChannel(ctx, unselected); err != nil {+ t.Fatalf("regex joinChannel unselected: %v", err)+ }+ if joinCalls.Load() != 1 {+ t.Fatalf("join calls after regex unselected channel = %d, want 1", joinCalls.Load())+ }+ if err := regexConnector.joinChannel(ctx, selected); err != nil {+ t.Fatalf("regex joinChannel selected: %v", err)+ }+ if joinCalls.Load() != 2 {+ t.Fatalf("join calls after regex selected channel = %d, want 2", joinCalls.Load())+ }+}