Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Feat: connector go discord (#18349)

mkaaad248a4e696006
diff --git a/internal/syncer/connector/discord.go b/internal/syncer/connector/discord.go
new file mode 100644
index 0000000000..800617a540
--- /dev/null
+++ b/internal/syncer/connector/discord.go
@@ -0,0 +1,869 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "os"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+)
+
+// Discord connector constants.
+const (
+ discordDefaultBaseURL = "https://discord.com/api/v10"
+ discordDefaultBatchSize = 1024
+ discordRequestTimeout = 60 * time.Second
+ discordMessagePageSize = 100
+ discordMaxGuildsPage = 200
+ discord429MaxWaits = 10
+ discordMaxRetryAfter = 60 * time.Second
+ discordSnippetLength = 30
+ discordDocIDPrefix = "DISCORD_"
+)
+
+// discordChannelTypeGuildText is the REST channel type of a text channel.
+const discordChannelTypeGuildText = 0
+
+// discordEpoch is the earliest timestamp Discord can represent.
+var discordEpoch = time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC)
+
+// DiscordConnector reads messages from Discord guild text channels and
+// their threads through the REST API with a bot token.
+type DiscordConnector struct {
+ token string
+ serverIDs map[string]struct{}
+ channelNames []string
+ batchSize int
+ startDate time.Time
+ baseURL string
+ client *http.Client
+}
+
+// discordTarget is one message source: either a text channel or a thread.
+type discordTarget struct {
+ channelID string
+ name string
+ isThread bool
+}
+
+// discordGuild is a Discord server returned by the guilds endpoint.
+type discordGuild struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+}
+
+// discordChannel is a channel or thread object from the REST API.
+type discordChannel struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+ Type int `json:"type"`
+ GuildID string `json:"guild_id"`
+ ParentID string `json:"parent_id"`
+}
+
+// discordThreadsResponse is the archived-threads paginated payload.
+type discordThreadsResponse struct {
+ Threads []discordChannel `json:"threads"`
+ HasMore bool `json:"has_more"`
+}
+
+// discordActiveThreadsResponse is the active-threads payload.
+type discordActiveThreadsResponse struct {
+ Threads []discordChannel `json:"threads"`
+}
+
+// discordMessage is a Discord message from the messages endpoint.
+type discordMessage struct {
+ ID string `json:"id"`
+ ChannelID string `json:"channel_id"`
+ Content string `json:"content"`
+ Type int `json:"type"`
+ Timestamp string `json:"timestamp"`
+ EditedAt string `json:"edited_timestamp"`
+ Author struct {
+ ID string `json:"id"`
+ Name string `json:"username"`
+ Bot bool `json:"bot"`
+ } `json:"author"`
+}
+
+// discordMessageWithTarget pairs a message with the source it came from.
+type discordMessageWithTarget struct {
+ message discordMessage
+ target discordTarget
+}
+
+// NewDiscordConnector parses a stored connector config into a connector.
+func NewDiscordConnector(config map[string]any) (*DiscordConnector, error) {
+ credentials, _ := config["credentials"].(map[string]any)
+ token := discordBotToken(stringConfig(credentials["discord_bot_token"]))
+
+ baseURL := strings.TrimRight(strings.TrimSpace(os.Getenv("DISCORD_CONNECTOR_BASE_URL")), "/")
+ if baseURL == "" {
+ baseURL = discordDefaultBaseURL
+ }
+
+ serverIDs := map[string]struct{}{}
+ for _, raw := range coerceDiscordStringList(config["server_ids"]) {
+ id := strings.TrimSpace(raw)
+ if !discordValidServerID(id) {
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Invalid Discord server_ids entry %q; expected a numeric server ID", raw)}
+ }
+ serverIDs[id] = struct{}{}
+ }
+
+ channelNames := coerceDiscordStringList(config["channels"])
+ if len(channelNames) == 0 {
+ channelNames = coerceDiscordStringList(config["channel_names"])
+ }
+
+ batchSize := configInt(config["batch_size"], discordDefaultBatchSize)
+ if batchSize <= 0 {
+ batchSize = discordDefaultBatchSize
+ }
+
+ startDate := time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)
+ if raw := strings.TrimSpace(stringConfig(config["start_date"])); raw != "" {
+ if parsed, err := time.Parse("2006-01-02", raw); err == nil {
+ startDate = time.Date(parsed.Year(), parsed.Month(), parsed.Day(), 0, 0, 0, 0, time.UTC)
+ }
+ }
+ if startDate.Before(discordEpoch) {
+ startDate = discordEpoch
+ }
+
+ return &DiscordConnector{
+ token: token,
+ serverIDs: serverIDs,
+ channelNames: channelNames,
+ batchSize: batchSize,
+ startDate: startDate,
+ baseURL: baseURL,
+ client: &http.Client{Timeout: discordRequestTimeout},
+ }, nil
+}
+
+// discordValidServerID reports whether s is a plausible Discord snowflake ID.
+func discordValidServerID(s string) bool {
+ if s == "" {
+ return false
+ }
+ for _, r := range s {
+ if r < '0' || r > '9' {
+ return false
+ }
+ }
+ return true
+}
+
+// coerceDiscordStringList normalizes a config field that may be a list or a
+// comma-separated string into a clean list of non-empty values.
+func coerceDiscordStringList(value any) []string {
+ var rawItems []any
+ switch typed := value.(type) {
+ case nil:
+ return nil
+ case string:
+ for _, part := range strings.Split(typed, ",") {
+ rawItems = append(rawItems, part)
+ }
+ case []any:
+ rawItems = typed
+ case []string:
+ for _, item := range typed {
+ rawItems = append(rawItems, item)
+ }
+ default:
+ rawItems = append(rawItems, typed)
+ }
+
+ out := make([]string, 0, len(rawItems))
+ for _, item := range rawItems {
+ if item == nil {
+ continue
+ }
+ text := strings.TrimSpace(fmt.Sprint(item))
+ if text != "" {
+ out = append(out, text)
+ }
+ }
+ return out
+}
+
+// discordBotToken strips an optional "Bot " or "Bearer " prefix.
+func discordBotToken(token string) string {
+ token = strings.TrimSpace(token)
+ lower := strings.ToLower(token)
+ if strings.HasPrefix(lower, "bot ") {
+ return strings.TrimSpace(token[4:])
+ }
+ if strings.HasPrefix(lower, "bearer ") {
+ return strings.TrimSpace(token[7:])
+ }
+ return token
+}
+
+// discordAuthorizationHeader builds the REST Authorization header value.
+func discordAuthorizationHeader(token string) string {
+ token = discordBotToken(token)
+ if token == "" {
+ return ""
+ }
+ return "Bot " + token
+}
+
+// Validate checks configuration and credential presence without network I/O.
+func (c *DiscordConnector) Validate(ctx context.Context) error {
+ if c == nil {
+ return &ConnectorValidationError{Message: "discord connector is nil"}
+ }
+ if strings.TrimSpace(c.token) == "" {
+ return &ConnectorMissingCredentialError{Message: "Discord connector requires 'discord_bot_token' in credentials"}
+ }
+ if c.batchSize <= 0 {
+ return &ConnectorValidationError{Message: "Discord connector batch_size must be a positive integer"}
+ }
+ return nil
+}
+
+// OpenSync opens one sync session over the configured channels and threads.
+func (c *DiscordConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
+ targets, err := c.listTargets(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ lowerBound := c.startDate
+ if !request.FromBeginning && request.WindowStart != nil && request.WindowStart.After(lowerBound) {
+ lowerBound = *request.WindowStart
+ }
+ upperBound := time.Time{}
+ if !request.FromBeginning {
+ upperBound = request.WindowEnd
+ }
+
+ resumeTarget, resumeBefore := discordResumePosition(targets, request.Resume)
+ return &discordSyncSession{
+ connector: c,
+ iter: newDiscordMessageIterator(c, targets, lowerBound, upperBound, resumeTarget, resumeBefore),
+ targetsFingerprint: discordCursorFingerprint(targets),
+ }, nil
+}
+
+// discordResumePosition returns the target and before cursor to continue from,
+// or empty values when the checkpoint does not match the current enumeration.
+func discordResumePosition(targets []discordTarget, checkpoint *SyncCheckpoint) (string, string) {
+ if checkpoint == nil || checkpoint.Cursor == "" {
+ return "", ""
+ }
+ var cursor discordSyncCursor
+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {
+ return "", ""
+ }
+ if cursor.Target == "" || cursor.Message == "" {
+ return "", ""
+ }
+ if cursor.Targets != "" && cursor.Targets != discordCursorFingerprint(targets) {
+ return "", ""
+ }
+ for _, target := range targets {
+ if target.channelID == cursor.Target {
+ return cursor.Target, cursor.Message
+ }
+ }
+ return "", ""
+}
+
+// discordSyncCursor is the resume position serialized into SyncCheckpoint.Cursor.
+// Target is the channel/thread of the last committed group and Message is the
+// oldest message ID of that group, i.e. the next `before` cursor for it.
+type discordSyncCursor struct {
+ Targets string `json:"targets"`
+ Target string `json:"target"`
+ Message string `json:"message"`
+}
+
+// discordCursorFingerprint summarizes the ordered target list so a resume can
+// detect enumeration changes and fall back to a full re-scan.
+func discordCursorFingerprint(targets []discordTarget) string {
+ ids := make([]string, 0, len(targets))
+ for _, target := range targets {
+ ids = append(ids, target.channelID)
+ }
+ return stableFingerprint(ids)
+}
+
+func encodeDiscordCursor(fingerprint, target, message string) string {
+ raw, err := json.Marshal(discordSyncCursor{Targets: fingerprint, Target: target, Message: message})
+ if err != nil {
+ return ""
+ }
+ return string(raw)
+}
+
+// OpenPrune opens one complete slim snapshot session.
+func (c *DiscordConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {
+ targets, err := c.listTargets(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &discordPruneSession{
+ connector: c,
+ iter: newDiscordMessageIterator(c, targets, c.startDate, time.Time{}, "", ""),
+ }, nil
+}
+
+// listTargets enumerates guild text channels and their threads.
+func (c *DiscordConnector) listTargets(ctx context.Context) ([]discordTarget, error) {
+ guildIDs := make([]string, 0, len(c.serverIDs))
+ if len(c.serverIDs) > 0 {
+ for guildID := range c.serverIDs {
+ guildIDs = append(guildIDs, guildID)
+ }
+ } else {
+ guilds, err := c.listGuilds(ctx)
+ if err != nil {
+ return nil, err
+ }
+ for _, guild := range guilds {
+ guildIDs = append(guildIDs, guild.ID)
+ }
+ }
+ sort.Strings(guildIDs)
+
+ var targets []discordTarget
+ for _, guildID := range guildIDs {
+ var channels []discordChannel
+ if status, err := c.getJSON(ctx, http.MethodGet, "/guilds/"+url.PathEscape(guildID)+"/channels", nil, &channels); err != nil {
+ if status == http.StatusForbidden {
+ continue
+ }
+ return nil, err
+ }
+
+ selected := map[string]discordChannel{}
+ var textChannels []discordChannel
+ for _, ch := range channels {
+ if ch.Type != discordChannelTypeGuildText {
+ continue
+ }
+ if len(c.channelNames) > 0 && !discordContainsString(c.channelNames, ch.Name) {
+ continue
+ }
+ selected[ch.ID] = ch
+ textChannels = append(textChannels, ch)
+ }
+
+ for _, ch := range textChannels {
+ targets = append(targets, discordTarget{channelID: ch.ID, name: ch.Name})
+ for _, kind := range []string{"public", "private"} {
+ threads, status, err := c.listArchivedThreads(ctx, ch.ID, kind)
+ if err != nil {
+ if status == http.StatusForbidden {
+ continue
+ }
+ return nil, err
+ }
+ for _, thread := range threads {
+ targets = append(targets, discordTarget{channelID: thread.ID, name: thread.Name, isThread: true})
+ }
+ }
+ }
+
+ activeThreads, status, err := c.listActiveThreads(ctx, guildID)
+ if err != nil {
+ if status == http.StatusForbidden {
+ continue
+ }
+ return nil, err
+ }
+ for _, thread := range activeThreads {
+ if _, ok := selected[thread.ParentID]; ok {
+ targets = append(targets, discordTarget{channelID: thread.ID, name: thread.Name, isThread: true})
+ }
+ }
+ }
+ return targets, nil
+}
+
+// listGuilds returns all guilds the bot can see.
+func (c *DiscordConnector) listGuilds(ctx context.Context) ([]discordGuild, error) {
+ var out []discordGuild
+ after := ""
+ for {
+ query := url.Values{"limit": {strconv.Itoa(discordMaxGuildsPage)}}
+ if after != "" {
+ query.Set("after", after)
+ }
+ var batch []discordGuild
+ if _, err := c.getJSON(ctx, http.MethodGet, "/users/@me/guilds", query, &batch); err != nil {
+ return nil, err
+ }
+ if len(batch) == 0 {
+ break
+ }
+ out = append(out, batch...)
+ after = batch[len(batch)-1].ID
+ if len(batch) < discordMaxGuildsPage {
+ break
+ }
+ }
+ return out, nil
+}
+
+// listActiveThreads returns all active threads of a guild.
+func (c *DiscordConnector) listActiveThreads(ctx context.Context, guildID string) ([]discordChannel, int, error) {
+ var resp discordActiveThreadsResponse
+ status, err := c.getJSON(ctx, http.MethodGet, "/guilds/"+url.PathEscape(guildID)+"/threads/active", nil, &resp)
+ if err != nil {
+ return nil, status, err
+ }
+ return resp.Threads, status, nil
+}
+
+// listArchivedThreads paginates archived public or private threads of a channel.
+func (c *DiscordConnector) listArchivedThreads(ctx context.Context, channelID, kind string) ([]discordChannel, int, error) {
+ var out []discordChannel
+ before := ""
+ for {
+ query := url.Values{"limit": {strconv.Itoa(discordMessagePageSize)}}
+ if before != "" {
+ query.Set("before", before)
+ }
+ var resp discordThreadsResponse
+ status, err := c.getJSON(ctx, http.MethodGet, "/channels/"+url.PathEscape(channelID)+"/threads/archived/"+kind, query, &resp)
+ if err != nil {
+ return nil, status, err
+ }
+ if len(resp.Threads) == 0 {
+ break
+ }
+ out = append(out, resp.Threads...)
+ if !resp.HasMore {
+ break
+ }
+ before = resp.Threads[len(resp.Threads)-1].ID
+ }
+ return out, http.StatusOK, nil
+}
+
+// getJSON performs one authenticated Discord REST request with 429 handling.
+// The returned status code lets callers distinguish permission errors.
+func (c *DiscordConnector) getJSON(ctx context.Context, method, path string, query url.Values, out any) (int, error) {
+ fullURL := strings.TrimRight(c.baseURL, "/") + path
+ if len(query) > 0 {
+ fullURL += "?" + query.Encode()
+ }
+ req, err := http.NewRequestWithContext(ctx, method, fullURL, nil)
+ if err != nil {
+ return 0, err
+ }
+ req.Header.Set("Authorization", discordAuthorizationHeader(c.token))
+ req.Header.Set("Accept", "application/json")
+
+ for wait := 0; wait < discord429MaxWaits; wait++ {
+ resp, err := c.client.Do(req)
+ if err != nil {
+ return 0, err
+ }
+ if resp.StatusCode == http.StatusTooManyRequests {
+ retryAfter := discordRetryAfter(resp)
+ _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1024*1024))
+ resp.Body.Close()
+ select {
+ case <-ctx.Done():
+ return 0, ctx.Err()
+ case <-time.After(retryAfter):
+ }
+ continue
+ }
+
+ respBody, readErr := io.ReadAll(io.LimitReader(resp.Body, 4*1024*1024))
+ resp.Body.Close()
+ if readErr != nil {
+ return resp.StatusCode, readErr
+ }
+ if resp.StatusCode >= 500 {
+ return resp.StatusCode, fmt.Errorf("discord api request failed with http %d", resp.StatusCode)
+ }
+ if resp.StatusCode >= 400 {
+ return resp.StatusCode, fmt.Errorf("discord api request failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(respBody)))
+ }
+ if out != nil && len(bytes.TrimSpace(respBody)) > 0 {
+ if err := json.Unmarshal(respBody, out); err != nil {
+ return resp.StatusCode, &ConnectorValidationError{Message: "discord api response is not valid JSON"}
+ }
+ }
+ return resp.StatusCode, nil
+ }
+ return 0, &RateLimitTriedTooManyTimesError{Message: fmt.Sprintf("Discord API rate limited: exceeded %d retries (too many requests)", discord429MaxWaits)}
+}
+
+// discordRetryAfter extracts the wait duration from a 429 response.
+func discordRetryAfter(resp *http.Response) time.Duration {
+ for _, key := range []string{"Retry-After", "X-RateLimit-Reset-After"} {
+ if raw := resp.Header.Get(key); raw != "" {
+ if seconds, err := strconv.ParseFloat(strings.TrimSpace(raw), 64); err == nil && seconds >= 0 {
+ retryAfter := time.Duration(seconds * float64(time.Second))
+ if retryAfter > discordMaxRetryAfter {
+ return discordMaxRetryAfter
+ }
+ return retryAfter
+ }
+ }
+ }
+ return 0
+}
+
+// discordContainsString reports whether a slice contains a value.
+func discordContainsString(values []string, target string) bool {
+ for _, value := range values {
+ if value == target {
+ return true
+ }
+ }
+ return false
+}
+
+// discordMessageIterator walks messages of the enumerated targets in
+// newest-to-oldest order, honoring a [lowerBound, upperBound) window.
+type discordMessageIterator struct {
+ connector *DiscordConnector
+ targets []discordTarget
+ lowerBound time.Time
+ upperBound time.Time
+
+ targetIndex int
+ page []discordMessage
+ before string
+ targetDone bool
+}
+
+func newDiscordMessageIterator(c *DiscordConnector, targets []discordTarget, lowerBound, upperBound time.Time, resumeTarget, resumeBefore string) *discordMessageIterator {
+ it := &discordMessageIterator{
+ connector: c,
+ targets: targets,
+ lowerBound: lowerBound,
+ upperBound: upperBound,
+ }
+ if resumeTarget != "" {
+ for i, target := range targets {
+ if target.channelID == resumeTarget {
+ it.targetIndex = i
+ it.before = resumeBefore
+ break
+ }
+ }
+ }
+ return it
+}
+
+// next returns the next in-window message, or io.EOF when exhausted.
+func (it *discordMessageIterator) next(ctx context.Context) (discordMessageWithTarget, error) {
+ for {
+ if it.targetIndex >= len(it.targets) {
+ return discordMessageWithTarget{}, io.EOF
+ }
+ target := it.targets[it.targetIndex]
+
+ if len(it.page) == 0 && !it.targetDone {
+ query := url.Values{"limit": {strconv.Itoa(discordMessagePageSize)}}
+ if it.before != "" {
+ query.Set("before", it.before)
+ }
+ var messages []discordMessage
+ if status, err := it.connector.getJSON(ctx, http.MethodGet, "/channels/"+url.PathEscape(target.channelID)+"/messages", query, &messages); err != nil {
+ if status == http.StatusForbidden {
+ it.advanceTarget()
+ continue
+ }
+ return discordMessageWithTarget{}, err
+ }
+ if len(messages) == 0 {
+ it.advanceTarget()
+ continue
+ }
+ it.page = messages
+ it.before = messages[len(messages)-1].ID
+ }
+
+ if it.targetDone && len(it.page) == 0 {
+ it.advanceTarget()
+ continue
+ }
+
+ msg := it.page[0]
+ it.page = it.page[1:]
+
+ createdAt := discordMessageCreatedAt(msg)
+ if createdAt.IsZero() {
+ continue
+ }
+ if !it.upperBound.IsZero() && !createdAt.Before(it.upperBound) {
+ continue
+ }
+ if createdAt.Before(it.lowerBound) {
+ it.targetDone = true
+ it.page = nil
+ continue
+ }
+ return discordMessageWithTarget{message: msg, target: target}, nil
+ }
+}
+
+func (it *discordMessageIterator) advanceTarget() {
+ it.targetIndex++
+ it.page = nil
+ it.before = ""
+ it.targetDone = false
+}
+
+// discordMessageCreatedAt parses the Discord RFC3339 timestamp as UTC.
+func discordMessageCreatedAt(msg discordMessage) time.Time {
+ created, err := time.Parse(time.RFC3339Nano, msg.Timestamp)
+ if err != nil {
+ return time.Time{}
+ }
+ return created.UTC()
+}
+
+// discordMessageUpdatedAt prefers the edit timestamp, falling back to created.
+func discordMessageUpdatedAt(msg discordMessage) time.Time {
+ if strings.TrimSpace(msg.EditedAt) != "" {
+ if edited, err := time.Parse(time.RFC3339Nano, msg.EditedAt); err == nil {
+ return edited.UTC()
+ }
+ }
+ return discordMessageCreatedAt(msg)
+}
+
+// discordSyncSession streams merged documents for one sync window.
+type discordSyncSession struct {
+ connector *DiscordConnector
+ iter *discordMessageIterator
+ targetsFingerprint string
+ pending []discordMessageWithTarget
+ currentTarget string
+ carry *discordMessageWithTarget
+}
+
+// NextBatch returns one merged document per call until io.EOF.
+func (s *discordSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
+ for {
+ for len(s.pending) < s.connector.batchSize {
+ var item discordMessageWithTarget
+ if s.carry != nil {
+ item = *s.carry
+ s.carry = nil
+ } else {
+ next, err := s.iter.next(ctx)
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ return SyncBatch{}, err
+ }
+ item = next
+ }
+ if s.currentTarget != "" && item.target.channelID != s.currentTarget {
+ s.carry = &item
+ s.currentTarget = ""
+ break
+ }
+ s.currentTarget = item.target.channelID
+ s.pending = append(s.pending, item)
+ }
+ if len(s.pending) > 0 {
+ break
+ }
+ if s.carry == nil {
+ return SyncBatch{}, io.EOF
+ }
+ }
+
+ docs := make([]SourceDocument, 0, len(s.pending))
+ for _, item := range s.pending {
+ docs = append(docs, discordMessageDocument(item))
+ }
+ merged := mergeDiscordDocuments(docs)
+ oldest := s.pending[len(s.pending)-1]
+ s.pending = nil
+ if s.carry != nil {
+ s.currentTarget = ""
+ }
+
+ batch := SyncBatch{Documents: []SourceDocument{merged}}
+ if cursor := encodeDiscordCursor(s.targetsFingerprint, oldest.target.channelID, oldest.message.ID); cursor != "" {
+ batch.Checkpoint = &SyncCheckpoint{Cursor: cursor}
+ }
+ return batch, nil
+}
+
+// Close releases the sync session.
+func (s *discordSyncSession) Close() error {
+ return nil
+}
+
+// discordMessageDocument converts one message into a source document.
+func discordMessageDocument(item discordMessageWithTarget) SourceDocument {
+ content := item.message.Content
+ snippet := content
+ if runes := []rune(snippet); len(runes) > discordSnippetLength {
+ snippet = string(runes[:discordSnippetLength]) + "..."
+ }
+
+ semantic := item.message.Author.Name + " said"
+ if item.target.isThread {
+ semantic += " in Thread: " + item.target.name
+ } else {
+ semantic += " in Channel: #" + item.target.name
+ }
+ semantic += ": " + snippet
+
+ updatedAt := discordMessageUpdatedAt(item.message)
+ blob := []byte(content)
+
+ var metadata map[string]any
+ if !item.target.isThread {
+ metadata = map[string]any{"Channel": item.target.name}
+ }
+
+ return SourceDocument{
+ SourceID: discordDocIDPrefix + item.message.ID,
+ SemanticIdentifier: semantic,
+ Extension: ".txt",
+ Blob: blob,
+ UpdatedAt: updatedAt,
+ SizeBytes: int64(len(blob)),
+ Metadata: metadata,
+ Fingerprint: stableFingerprint(map[string]any{
+ "id": item.message.ID,
+ "content": content,
+ "author": item.message.Author.Name,
+ "updated_at": updatedAt,
+ }),
+ }
+}
+
+// mergeDiscordDocuments merges consecutive messages into one document.
+func mergeDiscordDocuments(docs []SourceDocument) SourceDocument {
+ first := docs[0]
+ minUpdated, maxUpdated := first.UpdatedAt, first.UpdatedAt
+ ids := make([]string, 0, len(docs))
+ var blob strings.Builder
+ var size int64
+ for _, doc := range docs {
+ if doc.UpdatedAt.Before(minUpdated) {
+ minUpdated = doc.UpdatedAt
+ }
+ if doc.UpdatedAt.After(maxUpdated) {
+ maxUpdated = doc.UpdatedAt
+ }
+ ids = append(ids, doc.SourceID)
+ if blob.Len() > 0 {
+ blob.WriteString("\n\n")
+ }
+ blob.Write(doc.Blob)
+ size += doc.SizeBytes
+ }
+
+ format := "2006-01-02 15:04:05Z07:00"
+ return SourceDocument{
+ SourceID: first.SourceID,
+ SemanticIdentifier: fmt.Sprintf("%s -> %s", minUpdated.Format(format), maxUpdated.Format(format)),
+ Extension: ".txt",
+ Blob: []byte(blob.String()),
+ UpdatedAt: maxUpdated,
+ SizeBytes: size,
+ Metadata: first.Metadata,
+ Fingerprint: stableFingerprint(map[string]any{
+ "ids": ids,
+ "content": blob.String(),
+ "updated_at": maxUpdated,
+ }),
+ }
+}
+
+// discordPruneSession streams a complete slim message snapshot.
+type discordPruneSession struct {
+ connector *DiscordConnector
+ iter *discordMessageIterator
+
+ groupSize int
+ firstID string
+ currentTarget string
+ groupDocs []SlimDocument
+}
+
+// NextBatch returns slim documents grouped by the connector batch size.
+func (s *discordPruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {
+ for len(s.groupDocs) < s.connector.batchSize {
+ item, err := s.iter.next(ctx)
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ return PruneBatch{}, err
+ }
+
+ if s.currentTarget != "" && item.target.channelID != s.currentTarget {
+ s.groupDocs = append(s.groupDocs, SlimDocument{SourceID: s.firstID})
+ s.groupSize = 0
+ s.firstID = ""
+ }
+ if s.groupSize == 0 {
+ s.firstID = discordDocIDPrefix + item.message.ID
+ s.currentTarget = item.target.channelID
+ }
+ s.groupSize++
+ if s.groupSize >= s.connector.batchSize {
+ s.groupDocs = append(s.groupDocs, SlimDocument{SourceID: s.firstID})
+ s.groupSize = 0
+ s.firstID = ""
+ s.currentTarget = ""
+ }
+ }
+
+ if len(s.groupDocs) == 0 && s.groupSize > 0 {
+ s.groupDocs = append(s.groupDocs, SlimDocument{SourceID: s.firstID})
+ s.groupSize = 0
+ s.firstID = ""
+ s.currentTarget = ""
+ }
+ if len(s.groupDocs) == 0 {
+ return PruneBatch{}, io.EOF
+ }
+
+ batch := s.groupDocs
+ s.groupDocs = nil
+ return PruneBatch{Documents: batch}, nil
+}
+
+// Close releases the prune session.
+func (s *discordPruneSession) Close() error {
+ return nil
+}
diff --git a/internal/syncer/connector/discord_test.go b/internal/syncer/connector/discord_test.go
new file mode 100644
index 0000000000..fb18842fa6
--- /dev/null
+++ b/internal/syncer/connector/discord_test.go
@@ -0,0 +1,882 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+// discordFixture is an in-memory Discord API stub.
+type discordFixture struct {
+ guilds []map[string]any
+ channels map[string][]map[string]any // guildID -> channels
+ activeThread map[string][]map[string]any // guildID -> threads
+ archived map[string][]map[string]any // channelID -> threads (public)
+ messages map[string][]map[string]any // channelID -> messages (newest first)
+
+ forbidPrivateArchived bool
+ forbidChannels map[string]bool // channelID -> messages endpoint returns 403
+ rateLimitOnce atomic.Bool
+ failArchivedThreads bool
+ failActiveThreads bool
+
+ requests atomic.Int64
+}
+
+func newDiscordFixture() *discordFixture {
+ return &discordFixture{
+ guilds: []map[string]any{
+ {"id": "1001", "name": "Guild One"},
+ {"id": "2001", "name": "Guild Two"},
+ },
+ channels: map[string][]map[string]any{
+ "1001": {
+ {"id": "ch-1", "name": "general", "type": 0, "guild_id": "1001"},
+ {"id": "ch-2", "name": "announcements", "type": 0, "guild_id": "1001"},
+ {"id": "ch-3", "name": "mod-only", "type": 0, "guild_id": "1001"},
+ {"id": "cat-1", "name": "Category", "type": 4, "guild_id": "1001"},
+ {"id": "forum-1", "name": "Forum", "type": 15, "guild_id": "1001"},
+ },
+ "2001": {
+ {"id": "ch-g2", "name": "general", "type": 0, "guild_id": "2001"},
+ },
+ },
+ activeThread: map[string][]map[string]any{
+ "1001": {
+ {"id": "thread-a", "name": "Active Thread", "type": 12, "guild_id": "1001", "parent_id": "ch-1"},
+ {"id": "thread-other", "name": "Other Thread", "type": 12, "guild_id": "1001", "parent_id": "ch-2"},
+ },
+ },
+ archived: map[string][]map[string]any{
+ "ch-1": {
+ {"id": "thread-b", "name": "Archived Thread", "type": 11, "guild_id": "1001", "parent_id": "ch-1"},
+ },
+ },
+ messages: map[string][]map[string]any{},
+ }
+}
+
+func (f *discordFixture) setMessages(channelID string, messages ...map[string]any) {
+ f.messages[channelID] = messages
+}
+
+func (f *discordFixture) handler() http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ f.requests.Add(1)
+ path := r.URL.Path
+ query := r.URL.Query()
+
+ switch {
+ case path == "/users/@me/guilds":
+ json.NewEncoder(w).Encode(f.guilds)
+ return
+ case strings.HasPrefix(path, "/guilds/") && strings.HasSuffix(path, "/channels"):
+ guildID := strings.TrimSuffix(strings.TrimPrefix(path, "/guilds/"), "/channels")
+ json.NewEncoder(w).Encode(f.channels[guildID])
+ return
+ case strings.HasPrefix(path, "/guilds/") && strings.HasSuffix(path, "/threads/active"):
+ guildID := strings.TrimSuffix(strings.TrimPrefix(path, "/guilds/"), "/threads/active")
+ if f.failActiveThreads {
+ http.Error(w, "server error", http.StatusInternalServerError)
+ return
+ }
+ json.NewEncoder(w).Encode(map[string]any{"threads": f.activeThread[guildID]})
+ return
+ case strings.HasPrefix(path, "/channels/") && strings.Contains(path, "/threads/archived/"):
+ channelID := strings.TrimPrefix(path, "/channels/")
+ channelID = strings.TrimSuffix(channelID, "/threads/archived/public")
+ channelID = strings.TrimSuffix(channelID, "/threads/archived/private")
+ if strings.Contains(r.URL.Path, "/threads/archived/private") && f.forbidPrivateArchived {
+ http.Error(w, "forbidden", http.StatusForbidden)
+ return
+ }
+ if f.failArchivedThreads {
+ http.Error(w, "server error", http.StatusInternalServerError)
+ return
+ }
+ all := f.archived[channelID]
+ threads := all
+ start := 0
+ if before := query.Get("before"); before != "" {
+ for i, thread := range all {
+ if thread["id"] == before {
+ start = i + 1
+ break
+ }
+ }
+ threads = threads[start:]
+ }
+ json.NewEncoder(w).Encode(map[string]any{"threads": threads, "has_more": len(all) > start+len(threads)})
+ return
+ case strings.HasPrefix(path, "/channels/") && strings.HasSuffix(path, "/messages"):
+ if f.rateLimitOnce.CompareAndSwap(false, true) {
+ w.Header().Set("Retry-After", "0.001")
+ w.WriteHeader(http.StatusTooManyRequests)
+ json.NewEncoder(w).Encode(map[string]any{"message": "rate limited"})
+ return
+ }
+ channelID := strings.TrimSuffix(strings.TrimPrefix(path, "/channels/"), "/messages")
+ if f.forbidChannels[channelID] {
+ http.Error(w, "forbidden", http.StatusForbidden)
+ return
+ }
+ all := f.messages[channelID]
+ start := 0
+ if before := query.Get("before"); before != "" {
+ for i, message := range all {
+ if message["id"] == before {
+ start = i + 1
+ break
+ }
+ }
+ }
+ limit := 100
+ end := start + limit
+ if end > len(all) {
+ end = len(all)
+ }
+ json.NewEncoder(w).Encode(all[start:end])
+ return
+ default:
+ http.NotFound(w, r)
+ }
+ })
+}
+
+func fixtureMessage(id, content, author string, ts time.Time) map[string]any {
+ return map[string]any{
+ "id": id,
+ "content": content,
+ "type": 0,
+ "timestamp": ts.Format("2006-01-02T15:04:05.000000+00:00"),
+ "edited_timestamp": nil,
+ "author": map[string]any{"id": "u-" + id, "username": author, "bot": false},
+ }
+}
+
+func newDiscordTestConnector(t *testing.T, config map[string]any) *DiscordConnector {
+ t.Helper()
+ connector, err := NewDiscordConnector(config)
+ if err != nil {
+ t.Fatalf("NewDiscordConnector: %v", err)
+ }
+ return connector
+}
+
+func TestDiscordConfigParsing(t *testing.T) {
+ connector := newDiscordTestConnector(t, map[string]any{
+ "server_ids": []any{"1", "2"},
+ "channels": []any{"general", "announcements"},
+ "batch_size": 7,
+ "start_date": "2026-01-02",
+ "credentials": map[string]any{"discord_bot_token": "Bot token-1"},
+ })
+ if connector.batchSize != 7 {
+ t.Fatalf("batchSize = %d, want 7", connector.batchSize)
+ }
+ if len(connector.serverIDs) != 2 {
+ t.Fatalf("serverIDs = %d, want 2", len(connector.serverIDs))
+ }
+ if _, ok := connector.serverIDs["2"]; !ok {
+ t.Fatalf("serverIDs missing id 2")
+ }
+ if len(connector.channelNames) != 2 || connector.channelNames[0] != "general" {
+ t.Fatalf("channelNames = %v", connector.channelNames)
+ }
+ if connector.startDate.Format("2006-01-02") != "2026-01-02" {
+ t.Fatalf("startDate = %v", connector.startDate)
+ }
+ if connector.token != "token-1" {
+ t.Fatalf("token = %q, want token-1", connector.token)
+ }
+
+ // comma-separated strings and the legacy channel_names key are accepted too.
+ legacy := newDiscordTestConnector(t, map[string]any{
+ "server_ids": "1, 3",
+ "channel_names": "general",
+ "credentials": map[string]any{"discord_bot_token": "token-2"},
+ })
+ if len(legacy.serverIDs) != 2 {
+ t.Fatalf("legacy serverIDs = %d, want 2", len(legacy.serverIDs))
+ }
+ if len(legacy.channelNames) != 1 || legacy.channelNames[0] != "general" {
+ t.Fatalf("legacy channelNames = %v", legacy.channelNames)
+ }
+
+ // default start date is clamped to the earliest representable Discord time.
+ def := newDiscordTestConnector(t, map[string]any{
+ "credentials": map[string]any{"discord_bot_token": "token-3"},
+ })
+ if !def.startDate.Equal(discordEpoch) {
+ t.Fatalf("default startDate = %v, want %v", def.startDate, discordEpoch)
+ }
+}
+
+func TestDiscordConfigParsingInvalidServerID(t *testing.T) {
+ for _, serverIDs := range []any{
+ []any{"1001", "not-a-number"},
+ "1001, not-a-number",
+ } {
+ _, err := NewDiscordConnector(map[string]any{
+ "server_ids": serverIDs,
+ "credentials": map[string]any{"discord_bot_token": "token"},
+ })
+ var valErr *ConnectorValidationError
+ if !errors.As(err, &valErr) {
+ t.Fatalf("NewDiscordConnector(server_ids=%v) err = %v, want ConnectorValidationError", serverIDs, err)
+ }
+ }
+}
+
+func TestDiscordValidateNoNetwork(t *testing.T) {
+ fixture := newDiscordFixture()
+ server := httptest.NewServer(fixture.handler())
+ defer server.Close()
+
+ connector := newDiscordTestConnector(t, map[string]any{
+ "credentials": map[string]any{"discord_bot_token": "token"},
+ })
+ connector.baseURL = server.URL
+ if err := connector.Validate(context.Background()); err != nil {
+ t.Fatalf("Validate: %v", err)
+ }
+ if fixture.requests.Load() != 0 {
+ t.Fatalf("Validate made %d network requests, want 0", fixture.requests.Load())
+ }
+
+ missing := newDiscordTestConnector(t, map[string]any{"credentials": map[string]any{}})
+ var credErr *ConnectorMissingCredentialError
+ if err := missing.Validate(context.Background()); !errors.As(err, &credErr) {
+ t.Fatalf("Validate missing token err = %v, want ConnectorMissingCredentialError", err)
+ }
+}
+
+func TestDiscordOpenSyncMergesBatches(t *testing.T) {
+ fixture := newDiscordFixture()
+ base := mustTime(t, "2026-08-14T10:00:00Z")
+ fixture.setMessages("ch-1",
+ fixtureMessage("m1", "one", "alice", base),
+ fixtureMessage("m2", "two", "bob", base.Add(-time.Minute)),
+ fixtureMessage("m3", "three", "carol", base.Add(-2*time.Minute)),
+ fixtureMessage("m4", "four", "dave", base.Add(-3*time.Minute)),
+ )
+ server := httptest.NewServer(fixture.handler())
+ defer server.Close()
+
+ connector := newDiscordTestConnector(t, map[string]any{
+ "credentials": map[string]any{"discord_bot_token": "token"},
+ "batch_size": 2,
+ })
+ connector.baseURL = server.URL
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ defer session.Close()
+
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 {
+ t.Fatalf("documents = %d, want 1 merged document", len(batch.Documents))
+ }
+ doc := batch.Documents[0]
+ if doc.SourceID != "DISCORD_m1" {
+ t.Fatalf("SourceID = %q, want DISCORD_m1", doc.SourceID)
+ }
+ if string(doc.Blob) != "one\n\ntwo" {
+ t.Fatalf("blob = %q", string(doc.Blob))
+ }
+ if doc.SizeBytes != 6 {
+ t.Fatalf("size = %d, want 6 (sum of message contents)", doc.SizeBytes)
+ }
+ if doc.Fingerprint == "" {
+ t.Fatalf("fingerprint is empty")
+ }
+ if doc.Metadata == nil || doc.Metadata["Channel"] != "general" {
+ t.Fatalf("metadata = %v", doc.Metadata)
+ }
+ if !doc.UpdatedAt.Equal(base) {
+ t.Fatalf("UpdatedAt = %v, want %v", doc.UpdatedAt, base)
+ }
+
+ second, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("second NextBatch: %v", err)
+ }
+ if second.Documents[0].SourceID != "DISCORD_m3" || string(second.Documents[0].Blob) != "three\n\nfour" {
+ t.Fatalf("second doc = %q %q", second.Documents[0].SourceID, second.Documents[0].Blob)
+ }
+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v, want io.EOF", err)
+ }
+}
+
+func TestDiscordOpenSyncWindowFilter(t *testing.T) {
+ fixture := newDiscordFixture()
+ base := mustTime(t, "2026-08-14T10:00:00Z")
+ fixture.setMessages("ch-1",
+ fixtureMessage("m1", "new", "alice", base),
+ fixtureMessage("m2", "mid-1", "bob", base.Add(-time.Minute)),
+ fixtureMessage("m3", "mid-2", "carol", base.Add(-2*time.Minute)),
+ fixtureMessage("m4", "old", "dave", base.Add(-3*time.Minute)),
+ )
+ server := httptest.NewServer(fixture.handler())
+ defer server.Close()
+
+ connector := newDiscordTestConnector(t, map[string]any{
+ "credentials": map[string]any{"discord_bot_token": "token"},
+ "batch_size": 10,
+ })
+ connector.baseURL = server.URL
+
+ start := base.Add(-2 * time.Minute)
+ end := base
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: false,
+ WindowStart: &start,
+ WindowEnd: end,
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ defer session.Close()
+
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 {
+ t.Fatalf("documents = %d, want 1", len(batch.Documents))
+ }
+ if string(batch.Documents[0].Blob) != "mid-1\n\nmid-2" {
+ t.Fatalf("blob = %q, want mid-1 + mid-2", batch.Documents[0].Blob)
+ }
+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v, want io.EOF", err)
+ }
+}
+
+func TestDiscordOpenSyncChannelsAndThreads(t *testing.T) {
+ fixture := newDiscordFixture()
+ fixture.forbidPrivateArchived = true
+ fixture.forbidChannels = map[string]bool{"ch-3": true}
+ base := mustTime(t, "2026-08-14T10:00:00Z")
+ fixture.setMessages("ch-1", fixtureMessage("m-ch", "channel msg", "alice", base))
+ fixture.setMessages("thread-a", fixtureMessage("m-ta", "active thread msg", "bob", base))
+ fixture.setMessages("thread-b", fixtureMessage("m-tb", "archived thread msg", "carol", base))
+ server := httptest.NewServer(fixture.handler())
+ defer server.Close()
+
+ connector := newDiscordTestConnector(t, map[string]any{
+ "server_ids": []any{"1001"},
+ "channels": []any{"general", "mod-only"},
+ "credentials": map[string]any{"discord_bot_token": "token"},
+ "batch_size": 10,
+ })
+ connector.baseURL = server.URL
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ defer session.Close()
+
+ var blobs []string
+ for {
+ batch, err := session.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ blobs = append(blobs, string(batch.Documents[0].Blob))
+ }
+ want := []string{"channel msg", "archived thread msg", "active thread msg"}
+ if len(blobs) != len(want) {
+ t.Fatalf("documents = %v, want %v", blobs, want)
+ }
+ for i := range want {
+ if blobs[i] != want[i] {
+ t.Fatalf("document %d = %q, want %q", i, blobs[i], want[i])
+ }
+ }
+}
+
+func TestDiscordThreadListErrorsPropagate(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ fail func(*discordFixture)
+ }{
+ {"archived", func(f *discordFixture) { f.failArchivedThreads = true }},
+ {"active", func(f *discordFixture) { f.failActiveThreads = true }},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ fixture := newDiscordFixture()
+ tc.fail(fixture)
+ server := httptest.NewServer(fixture.handler())
+ defer server.Close()
+
+ connector := newDiscordTestConnector(t, map[string]any{
+ "credentials": map[string]any{"discord_bot_token": "token"},
+ })
+ connector.baseURL = server.URL
+
+ if _, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true}); err == nil {
+ t.Fatalf("OpenSync: want error, got nil")
+ }
+ if _, err := connector.OpenPrune(context.Background(), PruneRequest{}); err == nil {
+ t.Fatalf("OpenPrune: want error, got nil")
+ }
+ })
+ }
+}
+
+func TestDiscordResumeSameTarget(t *testing.T) {
+ fixture := newDiscordFixture()
+ base := mustTime(t, "2026-08-14T10:00:00Z")
+ fixture.setMessages("ch-1",
+ fixtureMessage("m1", "one", "alice", base),
+ fixtureMessage("m2", "two", "bob", base.Add(-time.Minute)),
+ fixtureMessage("m3", "three", "carol", base.Add(-2*time.Minute)),
+ fixtureMessage("m4", "four", "dave", base.Add(-3*time.Minute)),
+ )
+ server := httptest.NewServer(fixture.handler())
+ defer server.Close()
+
+ newConnector := func() *DiscordConnector {
+ connector := newDiscordTestConnector(t, map[string]any{
+ "credentials": map[string]any{"discord_bot_token": "token"},
+ "batch_size": 2,
+ })
+ connector.baseURL = server.URL
+ return connector
+ }
+
+ session, err := newConnector().OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ session.Close()
+ if batch.Checkpoint == nil {
+ t.Fatalf("batch checkpoint is nil")
+ }
+ var cursor discordSyncCursor
+ if err := json.Unmarshal([]byte(batch.Checkpoint.Cursor), &cursor); err != nil {
+ t.Fatalf("cursor unmarshal: %v", err)
+ }
+ if cursor.Target != "ch-1" || cursor.Message != "m2" || cursor.Targets == "" {
+ t.Fatalf("cursor = %+v, want target ch-1 message m2", cursor)
+ }
+
+ resumed, err := newConnector().OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: batch.Checkpoint,
+ })
+ if err != nil {
+ t.Fatalf("resumed OpenSync: %v", err)
+ }
+ defer resumed.Close()
+
+ first, err := resumed.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("resumed NextBatch: %v", err)
+ }
+ if first.Documents[0].SourceID != "DISCORD_m3" || string(first.Documents[0].Blob) != "three\n\nfour" {
+ t.Fatalf("resumed doc = %q %q, want DISCORD_m3 three+four", first.Documents[0].SourceID, first.Documents[0].Blob)
+ }
+ if _, err := resumed.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("resumed NextBatch EOF = %v, want io.EOF", err)
+ }
+}
+
+func TestDiscordResumeAcrossTargets(t *testing.T) {
+ fixture := newDiscordFixture()
+ base := mustTime(t, "2026-08-14T10:00:00Z")
+ fixture.setMessages("ch-1",
+ fixtureMessage("m1", "one", "alice", base),
+ fixtureMessage("m2", "two", "bob", base.Add(-time.Minute)),
+ fixtureMessage("m3", "three", "carol", base.Add(-2*time.Minute)),
+ fixtureMessage("m4", "four", "dave", base.Add(-3*time.Minute)),
+ )
+ fixture.setMessages("thread-a",
+ fixtureMessage("t1", "thread one", "eve", base.Add(-4*time.Minute)),
+ fixtureMessage("t2", "thread two", "frank", base.Add(-5*time.Minute)),
+ )
+ server := httptest.NewServer(fixture.handler())
+ defer server.Close()
+
+ newConnector := func() *DiscordConnector {
+ connector := newDiscordTestConnector(t, map[string]any{
+ "server_ids": []any{"1001"},
+ "channels": []any{"general"},
+ "credentials": map[string]any{"discord_bot_token": "token"},
+ "batch_size": 3,
+ })
+ connector.baseURL = server.URL
+ return connector
+ }
+
+ session, err := newConnector().OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ session.Close()
+ if batch.Checkpoint == nil {
+ t.Fatalf("batch checkpoint is nil")
+ }
+
+ resumed, err := newConnector().OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: batch.Checkpoint,
+ })
+ if err != nil {
+ t.Fatalf("resumed OpenSync: %v", err)
+ }
+ defer resumed.Close()
+
+ first, err := resumed.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("resumed NextBatch: %v", err)
+ }
+ if first.Documents[0].SourceID != "DISCORD_m4" || string(first.Documents[0].Blob) != "four" {
+ t.Fatalf("resumed doc = %q %q, want DISCORD_m4", first.Documents[0].SourceID, first.Documents[0].Blob)
+ }
+
+ second, err := resumed.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("resumed second NextBatch: %v", err)
+ }
+ if second.Documents[0].SourceID != "DISCORD_t1" || string(second.Documents[0].Blob) != "thread one\n\nthread two" {
+ t.Fatalf("resumed second doc = %q %q", second.Documents[0].SourceID, second.Documents[0].Blob)
+ }
+ if _, err := resumed.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("resumed NextBatch EOF = %v, want io.EOF", err)
+ }
+}
+
+func TestDiscordResumeFingerprintMismatch(t *testing.T) {
+ fixture := newDiscordFixture()
+ base := mustTime(t, "2026-08-14T10:00:00Z")
+ fixture.setMessages("ch-1",
+ fixtureMessage("m1", "one", "alice", base),
+ fixtureMessage("m2", "two", "bob", base.Add(-time.Minute)),
+ )
+ server := httptest.NewServer(fixture.handler())
+ defer server.Close()
+
+ // Run 1 enumerates all text channels of the guild.
+ firstConnector := newDiscordTestConnector(t, map[string]any{
+ "server_ids": []any{"1001"},
+ "credentials": map[string]any{"discord_bot_token": "token"},
+ "batch_size": 2,
+ })
+ firstConnector.baseURL = server.URL
+ session, err := firstConnector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ session.Close()
+
+ // Run 2 filters to one channel, changing the enumeration: resume is ignored.
+ secondConnector := newDiscordTestConnector(t, map[string]any{
+ "server_ids": []any{"1001"},
+ "channels": []any{"general"},
+ "credentials": map[string]any{"discord_bot_token": "token"},
+ "batch_size": 2,
+ })
+ secondConnector.baseURL = server.URL
+ resumed, err := secondConnector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: batch.Checkpoint,
+ })
+ if err != nil {
+ t.Fatalf("resumed OpenSync: %v", err)
+ }
+ defer resumed.Close()
+
+ first, err := resumed.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("resumed NextBatch: %v", err)
+ }
+ if first.Documents[0].SourceID != "DISCORD_m1" {
+ t.Fatalf("resumed doc = %q, want full re-scan from DISCORD_m1", first.Documents[0].SourceID)
+ }
+}
+
+func TestDiscordResumeInvalidCursor(t *testing.T) {
+ fixture := newDiscordFixture()
+ base := mustTime(t, "2026-08-14T10:00:00Z")
+ fixture.setMessages("ch-1", fixtureMessage("m1", "one", "alice", base))
+ server := httptest.NewServer(fixture.handler())
+ defer server.Close()
+
+ connector := newDiscordTestConnector(t, map[string]any{
+ "credentials": map[string]any{"discord_bot_token": "token"},
+ "batch_size": 2,
+ })
+ connector.baseURL = server.URL
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{Cursor: "not-json"},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ defer session.Close()
+
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if batch.Documents[0].SourceID != "DISCORD_m1" {
+ t.Fatalf("doc = %q, want full re-scan from DISCORD_m1", batch.Documents[0].SourceID)
+ }
+}
+
+func TestDiscordMessagePagination(t *testing.T) {
+ fixture := newDiscordFixture()
+ base := mustTime(t, "2026-08-14T10:00:00Z")
+ messages := make([]map[string]any, 0, 250)
+ for i := 0; i < 250; i++ {
+ messages = append(messages, fixtureMessage(
+ fmt.Sprintf("m%03d", i),
+ fmt.Sprintf("content %d", i),
+ "alice",
+ base.Add(-time.Duration(i)*time.Minute),
+ ))
+ }
+ fixture.setMessages("ch-1", messages...)
+ server := httptest.NewServer(fixture.handler())
+ defer server.Close()
+
+ connector := newDiscordTestConnector(t, map[string]any{
+ "credentials": map[string]any{"discord_bot_token": "token"},
+ "batch_size": 10,
+ })
+ connector.baseURL = server.URL
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ defer session.Close()
+
+ total := 0
+ for {
+ batch, err := session.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ total += len(batch.Documents[0].Blob)
+ }
+ if total == 0 {
+ t.Fatalf("no documents streamed")
+ }
+ if fixture.requests.Load() < 3 {
+ t.Fatalf("message requests = %d, want pagination across multiple pages", fixture.requests.Load())
+ }
+}
+
+func TestDiscord429Retry(t *testing.T) {
+ fixture := newDiscordFixture()
+ base := mustTime(t, "2026-08-14T10:00:00Z")
+ fixture.setMessages("ch-1", fixtureMessage("m1", "hello", "alice", base))
+ server := httptest.NewServer(fixture.handler())
+ defer server.Close()
+
+ connector := newDiscordTestConnector(t, map[string]any{
+ "credentials": map[string]any{"discord_bot_token": "token"},
+ "batch_size": 10,
+ })
+ connector.baseURL = server.URL
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ defer session.Close()
+
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch after 429: %v", err)
+ }
+ if string(batch.Documents[0].Blob) != "hello" {
+ t.Fatalf("blob = %q", batch.Documents[0].Blob)
+ }
+}
+
+func TestDiscordPruneSlimDocuments(t *testing.T) {
+ fixture := newDiscordFixture()
+ base := mustTime(t, "2026-08-14T10:00:00Z")
+ fixture.setMessages("ch-1",
+ fixtureMessage("m1", "one", "alice", base),
+ fixtureMessage("m2", "two", "bob", base.Add(-time.Minute)),
+ fixtureMessage("m3", "three", "carol", base.Add(-2*time.Minute)),
+ fixtureMessage("m4", "four", "dave", base.Add(-3*time.Minute)),
+ fixtureMessage("m5", "five", "eve", base.Add(-4*time.Minute)),
+ )
+ server := httptest.NewServer(fixture.handler())
+ defer server.Close()
+
+ connector := newDiscordTestConnector(t, map[string]any{
+ "credentials": map[string]any{"discord_bot_token": "token"},
+ "batch_size": 2,
+ })
+ connector.baseURL = server.URL
+
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})
+ if err != nil {
+ t.Fatalf("OpenPrune: %v", err)
+ }
+ defer session.Close()
+
+ first, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(first.Documents) != 2 {
+ t.Fatalf("first slim batch = %d, want 2", len(first.Documents))
+ }
+ if first.Documents[0].SourceID != "DISCORD_m1" || first.Documents[1].SourceID != "DISCORD_m3" {
+ t.Fatalf("slim ids = %q, %q", first.Documents[0].SourceID, first.Documents[1].SourceID)
+ }
+
+ second, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("second NextBatch: %v", err)
+ }
+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "DISCORD_m5" {
+ t.Fatalf("second slim batch = %v", second.Documents)
+ }
+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v, want io.EOF", err)
+ }
+}
+
+func TestDiscordPruneGroupBreaksOnTargetSwitch(t *testing.T) {
+ fixture := newDiscordFixture()
+ base := mustTime(t, "2026-08-14T10:00:00Z")
+ fixture.setMessages("ch-1",
+ fixtureMessage("m1", "one", "alice", base),
+ fixtureMessage("m2", "two", "bob", base.Add(-time.Minute)),
+ )
+ fixture.setMessages("thread-a",
+ fixtureMessage("m3", "three", "carol", base.Add(-2*time.Minute)),
+ fixtureMessage("m4", "four", "dave", base.Add(-3*time.Minute)),
+ )
+ server := httptest.NewServer(fixture.handler())
+ defer server.Close()
+
+ connector := newDiscordTestConnector(t, map[string]any{
+ "credentials": map[string]any{"discord_bot_token": "token"},
+ "batch_size": 10,
+ })
+ connector.baseURL = server.URL
+
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})
+ if err != nil {
+ t.Fatalf("OpenPrune: %v", err)
+ }
+ defer session.Close()
+
+ var ids []string
+ for {
+ batch, err := session.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ for _, doc := range batch.Documents {
+ ids = append(ids, doc.SourceID)
+ }
+ }
+ want := []string{"DISCORD_m1", "DISCORD_m3"}
+ if len(ids) != len(want) {
+ t.Fatalf("slim ids = %v, want %v", ids, want)
+ }
+ for i := range want {
+ if ids[i] != want[i] {
+ t.Fatalf("slim id %d = %q, want %q", i, ids[i], want[i])
+ }
+ }
+}
+
+func TestDiscordFingerprintStable(t *testing.T) {
+ base := mustTime(t, "2026-08-14T10:00:00Z")
+ target := discordTarget{channelID: "ch-1", name: "general"}
+ item := discordMessageWithTarget{
+ message: discordMessage{
+ ID: "m1",
+ Content: "hello",
+ Type: 0,
+ Timestamp: base.Format("2006-01-02T15:04:05.000000+00:00"),
+ Author: struct {
+ ID string `json:"id"`
+ Name string `json:"username"`
+ Bot bool `json:"bot"`
+ }{ID: "u1", Name: "alice"},
+ },
+ target: target,
+ }
+
+ first := discordMessageDocument(item)
+ second := discordMessageDocument(item)
+ if first.Fingerprint == "" || first.Fingerprint != second.Fingerprint {
+ t.Fatalf("fingerprint unstable: %q %q", first.Fingerprint, second.Fingerprint)
+ }
+ if first.SemanticIdentifier != "alice said in Channel: #general: hello" {
+ t.Fatalf("semantic identifier = %q", first.SemanticIdentifier)
+ }
+
+ edited := item
+ edited.message.Content = "hello world"
+ if changed := discordMessageDocument(edited); changed.Fingerprint == first.Fingerprint {
+ t.Fatalf("fingerprint did not change with content")
+ }
+}
diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go
index 856a88ff87..dab0eb69cb 100644
--- a/internal/syncer/syncer.go
+++ b/internal/syncer/syncer.go
@@ -178,6 +178,7 @@ func registerBuiltInConnectors(registry *syncerconnector.Registry) {
registerDAOConnector(registry, "rest_api", syncerconnector.NewRestAPIConnector)
registerDAOConnector(registry, "mysql", syncerconnector.NewMySQLConnector)
registerDAOConnector(registry, "postgresql", syncerconnector.NewPostgreSQLConnector)
+ registerDAOConnector(registry, "discord", syncerconnector.NewDiscordConnector)
}
func registerDAOConnector[T syncerconnector.Connector](registry *syncerconnector.Registry, source string, factory func(map[string]any) (T, error)) {