RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Feat: connector go discord (#18349)
mkaaad248a4e696006
diff --git a/internal/syncer/connector/discord.go b/internal/syncer/connector/discord.gonew file mode 100644index 0000000000..800617a540--- /dev/null+++ b/internal/syncer/connector/discord.go@@ -0,0 +1,869 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "bytes"+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "net/http"+ "net/url"+ "os"+ "sort"+ "strconv"+ "strings"+ "time"+)++// Discord connector constants.+const (+ discordDefaultBaseURL = "https://discord.com/api/v10"+ discordDefaultBatchSize = 1024+ discordRequestTimeout = 60 * time.Second+ discordMessagePageSize = 100+ discordMaxGuildsPage = 200+ discord429MaxWaits = 10+ discordMaxRetryAfter = 60 * time.Second+ discordSnippetLength = 30+ discordDocIDPrefix = "DISCORD_"+)++// discordChannelTypeGuildText is the REST channel type of a text channel.+const discordChannelTypeGuildText = 0++// discordEpoch is the earliest timestamp Discord can represent.+var discordEpoch = time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC)++// DiscordConnector reads messages from Discord guild text channels and+// their threads through the REST API with a bot token.+type DiscordConnector struct {+ token string+ serverIDs map[string]struct{}+ channelNames []string+ batchSize int+ startDate time.Time+ baseURL string+ client *http.Client+}++// discordTarget is one message source: either a text channel or a thread.+type discordTarget struct {+ channelID string+ name string+ isThread bool+}++// discordGuild is a Discord server returned by the guilds endpoint.+type discordGuild struct {+ ID string `json:"id"`+ Name string `json:"name"`+}++// discordChannel is a channel or thread object from the REST API.+type discordChannel struct {+ ID string `json:"id"`+ Name string `json:"name"`+ Type int `json:"type"`+ GuildID string `json:"guild_id"`+ ParentID string `json:"parent_id"`+}++// discordThreadsResponse is the archived-threads paginated payload.+type discordThreadsResponse struct {+ Threads []discordChannel `json:"threads"`+ HasMore bool `json:"has_more"`+}++// discordActiveThreadsResponse is the active-threads payload.+type discordActiveThreadsResponse struct {+ Threads []discordChannel `json:"threads"`+}++// discordMessage is a Discord message from the messages endpoint.+type discordMessage struct {+ ID string `json:"id"`+ ChannelID string `json:"channel_id"`+ Content string `json:"content"`+ Type int `json:"type"`+ Timestamp string `json:"timestamp"`+ EditedAt string `json:"edited_timestamp"`+ Author struct {+ ID string `json:"id"`+ Name string `json:"username"`+ Bot bool `json:"bot"`+ } `json:"author"`+}++// discordMessageWithTarget pairs a message with the source it came from.+type discordMessageWithTarget struct {+ message discordMessage+ target discordTarget+}++// NewDiscordConnector parses a stored connector config into a connector.+func NewDiscordConnector(config map[string]any) (*DiscordConnector, error) {+ credentials, _ := config["credentials"].(map[string]any)+ token := discordBotToken(stringConfig(credentials["discord_bot_token"]))++ baseURL := strings.TrimRight(strings.TrimSpace(os.Getenv("DISCORD_CONNECTOR_BASE_URL")), "/")+ if baseURL == "" {+ baseURL = discordDefaultBaseURL+ }++ serverIDs := map[string]struct{}{}+ for _, raw := range coerceDiscordStringList(config["server_ids"]) {+ id := strings.TrimSpace(raw)+ if !discordValidServerID(id) {+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Invalid Discord server_ids entry %q; expected a numeric server ID", raw)}+ }+ serverIDs[id] = struct{}{}+ }++ channelNames := coerceDiscordStringList(config["channels"])+ if len(channelNames) == 0 {+ channelNames = coerceDiscordStringList(config["channel_names"])+ }++ batchSize := configInt(config["batch_size"], discordDefaultBatchSize)+ if batchSize <= 0 {+ batchSize = discordDefaultBatchSize+ }++ startDate := time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)+ if raw := strings.TrimSpace(stringConfig(config["start_date"])); raw != "" {+ if parsed, err := time.Parse("2006-01-02", raw); err == nil {+ startDate = time.Date(parsed.Year(), parsed.Month(), parsed.Day(), 0, 0, 0, 0, time.UTC)+ }+ }+ if startDate.Before(discordEpoch) {+ startDate = discordEpoch+ }++ return &DiscordConnector{+ token: token,+ serverIDs: serverIDs,+ channelNames: channelNames,+ batchSize: batchSize,+ startDate: startDate,+ baseURL: baseURL,+ client: &http.Client{Timeout: discordRequestTimeout},+ }, nil+}++// discordValidServerID reports whether s is a plausible Discord snowflake ID.+func discordValidServerID(s string) bool {+ if s == "" {+ return false+ }+ for _, r := range s {+ if r < '0' || r > '9' {+ return false+ }+ }+ return true+}++// coerceDiscordStringList normalizes a config field that may be a list or a+// comma-separated string into a clean list of non-empty values.+func coerceDiscordStringList(value any) []string {+ var rawItems []any+ switch typed := value.(type) {+ case nil:+ return nil+ case string:+ for _, part := range strings.Split(typed, ",") {+ rawItems = append(rawItems, part)+ }+ case []any:+ rawItems = typed+ case []string:+ for _, item := range typed {+ rawItems = append(rawItems, item)+ }+ default:+ rawItems = append(rawItems, typed)+ }++ out := make([]string, 0, len(rawItems))+ for _, item := range rawItems {+ if item == nil {+ continue+ }+ text := strings.TrimSpace(fmt.Sprint(item))+ if text != "" {+ out = append(out, text)+ }+ }+ return out+}++// discordBotToken strips an optional "Bot " or "Bearer " prefix.+func discordBotToken(token string) string {+ token = strings.TrimSpace(token)+ lower := strings.ToLower(token)+ if strings.HasPrefix(lower, "bot ") {+ return strings.TrimSpace(token[4:])+ }+ if strings.HasPrefix(lower, "bearer ") {+ return strings.TrimSpace(token[7:])+ }+ return token+}++// discordAuthorizationHeader builds the REST Authorization header value.+func discordAuthorizationHeader(token string) string {+ token = discordBotToken(token)+ if token == "" {+ return ""+ }+ return "Bot " + token+}++// Validate checks configuration and credential presence without network I/O.+func (c *DiscordConnector) Validate(ctx context.Context) error {+ if c == nil {+ return &ConnectorValidationError{Message: "discord connector is nil"}+ }+ if strings.TrimSpace(c.token) == "" {+ return &ConnectorMissingCredentialError{Message: "Discord connector requires 'discord_bot_token' in credentials"}+ }+ if c.batchSize <= 0 {+ return &ConnectorValidationError{Message: "Discord connector batch_size must be a positive integer"}+ }+ return nil+}++// OpenSync opens one sync session over the configured channels and threads.+func (c *DiscordConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {+ targets, err := c.listTargets(ctx)+ if err != nil {+ return nil, err+ }++ lowerBound := c.startDate+ if !request.FromBeginning && request.WindowStart != nil && request.WindowStart.After(lowerBound) {+ lowerBound = *request.WindowStart+ }+ upperBound := time.Time{}+ if !request.FromBeginning {+ upperBound = request.WindowEnd+ }++ resumeTarget, resumeBefore := discordResumePosition(targets, request.Resume)+ return &discordSyncSession{+ connector: c,+ iter: newDiscordMessageIterator(c, targets, lowerBound, upperBound, resumeTarget, resumeBefore),+ targetsFingerprint: discordCursorFingerprint(targets),+ }, nil+}++// discordResumePosition returns the target and before cursor to continue from,+// or empty values when the checkpoint does not match the current enumeration.+func discordResumePosition(targets []discordTarget, checkpoint *SyncCheckpoint) (string, string) {+ if checkpoint == nil || checkpoint.Cursor == "" {+ return "", ""+ }+ var cursor discordSyncCursor+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {+ return "", ""+ }+ if cursor.Target == "" || cursor.Message == "" {+ return "", ""+ }+ if cursor.Targets != "" && cursor.Targets != discordCursorFingerprint(targets) {+ return "", ""+ }+ for _, target := range targets {+ if target.channelID == cursor.Target {+ return cursor.Target, cursor.Message+ }+ }+ return "", ""+}++// discordSyncCursor is the resume position serialized into SyncCheckpoint.Cursor.+// Target is the channel/thread of the last committed group and Message is the+// oldest message ID of that group, i.e. the next `before` cursor for it.+type discordSyncCursor struct {+ Targets string `json:"targets"`+ Target string `json:"target"`+ Message string `json:"message"`+}++// discordCursorFingerprint summarizes the ordered target list so a resume can+// detect enumeration changes and fall back to a full re-scan.+func discordCursorFingerprint(targets []discordTarget) string {+ ids := make([]string, 0, len(targets))+ for _, target := range targets {+ ids = append(ids, target.channelID)+ }+ return stableFingerprint(ids)+}++func encodeDiscordCursor(fingerprint, target, message string) string {+ raw, err := json.Marshal(discordSyncCursor{Targets: fingerprint, Target: target, Message: message})+ if err != nil {+ return ""+ }+ return string(raw)+}++// OpenPrune opens one complete slim snapshot session.+func (c *DiscordConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {+ targets, err := c.listTargets(ctx)+ if err != nil {+ return nil, err+ }+ return &discordPruneSession{+ connector: c,+ iter: newDiscordMessageIterator(c, targets, c.startDate, time.Time{}, "", ""),+ }, nil+}++// listTargets enumerates guild text channels and their threads.+func (c *DiscordConnector) listTargets(ctx context.Context) ([]discordTarget, error) {+ guildIDs := make([]string, 0, len(c.serverIDs))+ if len(c.serverIDs) > 0 {+ for guildID := range c.serverIDs {+ guildIDs = append(guildIDs, guildID)+ }+ } else {+ guilds, err := c.listGuilds(ctx)+ if err != nil {+ return nil, err+ }+ for _, guild := range guilds {+ guildIDs = append(guildIDs, guild.ID)+ }+ }+ sort.Strings(guildIDs)++ var targets []discordTarget+ for _, guildID := range guildIDs {+ var channels []discordChannel+ if status, err := c.getJSON(ctx, http.MethodGet, "/guilds/"+url.PathEscape(guildID)+"/channels", nil, &channels); err != nil {+ if status == http.StatusForbidden {+ continue+ }+ return nil, err+ }++ selected := map[string]discordChannel{}+ var textChannels []discordChannel+ for _, ch := range channels {+ if ch.Type != discordChannelTypeGuildText {+ continue+ }+ if len(c.channelNames) > 0 && !discordContainsString(c.channelNames, ch.Name) {+ continue+ }+ selected[ch.ID] = ch+ textChannels = append(textChannels, ch)+ }++ for _, ch := range textChannels {+ targets = append(targets, discordTarget{channelID: ch.ID, name: ch.Name})+ for _, kind := range []string{"public", "private"} {+ threads, status, err := c.listArchivedThreads(ctx, ch.ID, kind)+ if err != nil {+ if status == http.StatusForbidden {+ continue+ }+ return nil, err+ }+ for _, thread := range threads {+ targets = append(targets, discordTarget{channelID: thread.ID, name: thread.Name, isThread: true})+ }+ }+ }++ activeThreads, status, err := c.listActiveThreads(ctx, guildID)+ if err != nil {+ if status == http.StatusForbidden {+ continue+ }+ return nil, err+ }+ for _, thread := range activeThreads {+ if _, ok := selected[thread.ParentID]; ok {+ targets = append(targets, discordTarget{channelID: thread.ID, name: thread.Name, isThread: true})+ }+ }+ }+ return targets, nil+}++// listGuilds returns all guilds the bot can see.+func (c *DiscordConnector) listGuilds(ctx context.Context) ([]discordGuild, error) {+ var out []discordGuild+ after := ""+ for {+ query := url.Values{"limit": {strconv.Itoa(discordMaxGuildsPage)}}+ if after != "" {+ query.Set("after", after)+ }+ var batch []discordGuild+ if _, err := c.getJSON(ctx, http.MethodGet, "/users/@me/guilds", query, &batch); err != nil {+ return nil, err+ }+ if len(batch) == 0 {+ break+ }+ out = append(out, batch...)+ after = batch[len(batch)-1].ID+ if len(batch) < discordMaxGuildsPage {+ break+ }+ }+ return out, nil+}++// listActiveThreads returns all active threads of a guild.+func (c *DiscordConnector) listActiveThreads(ctx context.Context, guildID string) ([]discordChannel, int, error) {+ var resp discordActiveThreadsResponse+ status, err := c.getJSON(ctx, http.MethodGet, "/guilds/"+url.PathEscape(guildID)+"/threads/active", nil, &resp)+ if err != nil {+ return nil, status, err+ }+ return resp.Threads, status, nil+}++// listArchivedThreads paginates archived public or private threads of a channel.+func (c *DiscordConnector) listArchivedThreads(ctx context.Context, channelID, kind string) ([]discordChannel, int, error) {+ var out []discordChannel+ before := ""+ for {+ query := url.Values{"limit": {strconv.Itoa(discordMessagePageSize)}}+ if before != "" {+ query.Set("before", before)+ }+ var resp discordThreadsResponse+ status, err := c.getJSON(ctx, http.MethodGet, "/channels/"+url.PathEscape(channelID)+"/threads/archived/"+kind, query, &resp)+ if err != nil {+ return nil, status, err+ }+ if len(resp.Threads) == 0 {+ break+ }+ out = append(out, resp.Threads...)+ if !resp.HasMore {+ break+ }+ before = resp.Threads[len(resp.Threads)-1].ID+ }+ return out, http.StatusOK, nil+}++// getJSON performs one authenticated Discord REST request with 429 handling.+// The returned status code lets callers distinguish permission errors.+func (c *DiscordConnector) getJSON(ctx context.Context, method, path string, query url.Values, out any) (int, error) {+ fullURL := strings.TrimRight(c.baseURL, "/") + path+ if len(query) > 0 {+ fullURL += "?" + query.Encode()+ }+ req, err := http.NewRequestWithContext(ctx, method, fullURL, nil)+ if err != nil {+ return 0, err+ }+ req.Header.Set("Authorization", discordAuthorizationHeader(c.token))+ req.Header.Set("Accept", "application/json")++ for wait := 0; wait < discord429MaxWaits; wait++ {+ resp, err := c.client.Do(req)+ if err != nil {+ return 0, err+ }+ if resp.StatusCode == http.StatusTooManyRequests {+ retryAfter := discordRetryAfter(resp)+ _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1024*1024))+ resp.Body.Close()+ select {+ case <-ctx.Done():+ return 0, ctx.Err()+ case <-time.After(retryAfter):+ }+ continue+ }++ respBody, readErr := io.ReadAll(io.LimitReader(resp.Body, 4*1024*1024))+ resp.Body.Close()+ if readErr != nil {+ return resp.StatusCode, readErr+ }+ if resp.StatusCode >= 500 {+ return resp.StatusCode, fmt.Errorf("discord api request failed with http %d", resp.StatusCode)+ }+ if resp.StatusCode >= 400 {+ return resp.StatusCode, fmt.Errorf("discord api request failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(respBody)))+ }+ if out != nil && len(bytes.TrimSpace(respBody)) > 0 {+ if err := json.Unmarshal(respBody, out); err != nil {+ return resp.StatusCode, &ConnectorValidationError{Message: "discord api response is not valid JSON"}+ }+ }+ return resp.StatusCode, nil+ }+ return 0, &RateLimitTriedTooManyTimesError{Message: fmt.Sprintf("Discord API rate limited: exceeded %d retries (too many requests)", discord429MaxWaits)}+}++// discordRetryAfter extracts the wait duration from a 429 response.+func discordRetryAfter(resp *http.Response) time.Duration {+ for _, key := range []string{"Retry-After", "X-RateLimit-Reset-After"} {+ if raw := resp.Header.Get(key); raw != "" {+ if seconds, err := strconv.ParseFloat(strings.TrimSpace(raw), 64); err == nil && seconds >= 0 {+ retryAfter := time.Duration(seconds * float64(time.Second))+ if retryAfter > discordMaxRetryAfter {+ return discordMaxRetryAfter+ }+ return retryAfter+ }+ }+ }+ return 0+}++// discordContainsString reports whether a slice contains a value.+func discordContainsString(values []string, target string) bool {+ for _, value := range values {+ if value == target {+ return true+ }+ }+ return false+}++// discordMessageIterator walks messages of the enumerated targets in+// newest-to-oldest order, honoring a [lowerBound, upperBound) window.+type discordMessageIterator struct {+ connector *DiscordConnector+ targets []discordTarget+ lowerBound time.Time+ upperBound time.Time++ targetIndex int+ page []discordMessage+ before string+ targetDone bool+}++func newDiscordMessageIterator(c *DiscordConnector, targets []discordTarget, lowerBound, upperBound time.Time, resumeTarget, resumeBefore string) *discordMessageIterator {+ it := &discordMessageIterator{+ connector: c,+ targets: targets,+ lowerBound: lowerBound,+ upperBound: upperBound,+ }+ if resumeTarget != "" {+ for i, target := range targets {+ if target.channelID == resumeTarget {+ it.targetIndex = i+ it.before = resumeBefore+ break+ }+ }+ }+ return it+}++// next returns the next in-window message, or io.EOF when exhausted.+func (it *discordMessageIterator) next(ctx context.Context) (discordMessageWithTarget, error) {+ for {+ if it.targetIndex >= len(it.targets) {+ return discordMessageWithTarget{}, io.EOF+ }+ target := it.targets[it.targetIndex]++ if len(it.page) == 0 && !it.targetDone {+ query := url.Values{"limit": {strconv.Itoa(discordMessagePageSize)}}+ if it.before != "" {+ query.Set("before", it.before)+ }+ var messages []discordMessage+ if status, err := it.connector.getJSON(ctx, http.MethodGet, "/channels/"+url.PathEscape(target.channelID)+"/messages", query, &messages); err != nil {+ if status == http.StatusForbidden {+ it.advanceTarget()+ continue+ }+ return discordMessageWithTarget{}, err+ }+ if len(messages) == 0 {+ it.advanceTarget()+ continue+ }+ it.page = messages+ it.before = messages[len(messages)-1].ID+ }++ if it.targetDone && len(it.page) == 0 {+ it.advanceTarget()+ continue+ }++ msg := it.page[0]+ it.page = it.page[1:]++ createdAt := discordMessageCreatedAt(msg)+ if createdAt.IsZero() {+ continue+ }+ if !it.upperBound.IsZero() && !createdAt.Before(it.upperBound) {+ continue+ }+ if createdAt.Before(it.lowerBound) {+ it.targetDone = true+ it.page = nil+ continue+ }+ return discordMessageWithTarget{message: msg, target: target}, nil+ }+}++func (it *discordMessageIterator) advanceTarget() {+ it.targetIndex+++ it.page = nil+ it.before = ""+ it.targetDone = false+}++// discordMessageCreatedAt parses the Discord RFC3339 timestamp as UTC.+func discordMessageCreatedAt(msg discordMessage) time.Time {+ created, err := time.Parse(time.RFC3339Nano, msg.Timestamp)+ if err != nil {+ return time.Time{}+ }+ return created.UTC()+}++// discordMessageUpdatedAt prefers the edit timestamp, falling back to created.+func discordMessageUpdatedAt(msg discordMessage) time.Time {+ if strings.TrimSpace(msg.EditedAt) != "" {+ if edited, err := time.Parse(time.RFC3339Nano, msg.EditedAt); err == nil {+ return edited.UTC()+ }+ }+ return discordMessageCreatedAt(msg)+}++// discordSyncSession streams merged documents for one sync window.+type discordSyncSession struct {+ connector *DiscordConnector+ iter *discordMessageIterator+ targetsFingerprint string+ pending []discordMessageWithTarget+ currentTarget string+ carry *discordMessageWithTarget+}++// NextBatch returns one merged document per call until io.EOF.+func (s *discordSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {+ for {+ for len(s.pending) < s.connector.batchSize {+ var item discordMessageWithTarget+ if s.carry != nil {+ item = *s.carry+ s.carry = nil+ } else {+ next, err := s.iter.next(ctx)+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ return SyncBatch{}, err+ }+ item = next+ }+ if s.currentTarget != "" && item.target.channelID != s.currentTarget {+ s.carry = &item+ s.currentTarget = ""+ break+ }+ s.currentTarget = item.target.channelID+ s.pending = append(s.pending, item)+ }+ if len(s.pending) > 0 {+ break+ }+ if s.carry == nil {+ return SyncBatch{}, io.EOF+ }+ }++ docs := make([]SourceDocument, 0, len(s.pending))+ for _, item := range s.pending {+ docs = append(docs, discordMessageDocument(item))+ }+ merged := mergeDiscordDocuments(docs)+ oldest := s.pending[len(s.pending)-1]+ s.pending = nil+ if s.carry != nil {+ s.currentTarget = ""+ }++ batch := SyncBatch{Documents: []SourceDocument{merged}}+ if cursor := encodeDiscordCursor(s.targetsFingerprint, oldest.target.channelID, oldest.message.ID); cursor != "" {+ batch.Checkpoint = &SyncCheckpoint{Cursor: cursor}+ }+ return batch, nil+}++// Close releases the sync session.+func (s *discordSyncSession) Close() error {+ return nil+}++// discordMessageDocument converts one message into a source document.+func discordMessageDocument(item discordMessageWithTarget) SourceDocument {+ content := item.message.Content+ snippet := content+ if runes := []rune(snippet); len(runes) > discordSnippetLength {+ snippet = string(runes[:discordSnippetLength]) + "..."+ }++ semantic := item.message.Author.Name + " said"+ if item.target.isThread {+ semantic += " in Thread: " + item.target.name+ } else {+ semantic += " in Channel: #" + item.target.name+ }+ semantic += ": " + snippet++ updatedAt := discordMessageUpdatedAt(item.message)+ blob := []byte(content)++ var metadata map[string]any+ if !item.target.isThread {+ metadata = map[string]any{"Channel": item.target.name}+ }++ return SourceDocument{+ SourceID: discordDocIDPrefix + item.message.ID,+ SemanticIdentifier: semantic,+ Extension: ".txt",+ Blob: blob,+ UpdatedAt: updatedAt,+ SizeBytes: int64(len(blob)),+ Metadata: metadata,+ Fingerprint: stableFingerprint(map[string]any{+ "id": item.message.ID,+ "content": content,+ "author": item.message.Author.Name,+ "updated_at": updatedAt,+ }),+ }+}++// mergeDiscordDocuments merges consecutive messages into one document.+func mergeDiscordDocuments(docs []SourceDocument) SourceDocument {+ first := docs[0]+ minUpdated, maxUpdated := first.UpdatedAt, first.UpdatedAt+ ids := make([]string, 0, len(docs))+ var blob strings.Builder+ var size int64+ for _, doc := range docs {+ if doc.UpdatedAt.Before(minUpdated) {+ minUpdated = doc.UpdatedAt+ }+ if doc.UpdatedAt.After(maxUpdated) {+ maxUpdated = doc.UpdatedAt+ }+ ids = append(ids, doc.SourceID)+ if blob.Len() > 0 {+ blob.WriteString("\n\n")+ }+ blob.Write(doc.Blob)+ size += doc.SizeBytes+ }++ format := "2006-01-02 15:04:05Z07:00"+ return SourceDocument{+ SourceID: first.SourceID,+ SemanticIdentifier: fmt.Sprintf("%s -> %s", minUpdated.Format(format), maxUpdated.Format(format)),+ Extension: ".txt",+ Blob: []byte(blob.String()),+ UpdatedAt: maxUpdated,+ SizeBytes: size,+ Metadata: first.Metadata,+ Fingerprint: stableFingerprint(map[string]any{+ "ids": ids,+ "content": blob.String(),+ "updated_at": maxUpdated,+ }),+ }+}++// discordPruneSession streams a complete slim message snapshot.+type discordPruneSession struct {+ connector *DiscordConnector+ iter *discordMessageIterator++ groupSize int+ firstID string+ currentTarget string+ groupDocs []SlimDocument+}++// NextBatch returns slim documents grouped by the connector batch size.+func (s *discordPruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {+ for len(s.groupDocs) < s.connector.batchSize {+ item, err := s.iter.next(ctx)+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ return PruneBatch{}, err+ }++ if s.currentTarget != "" && item.target.channelID != s.currentTarget {+ s.groupDocs = append(s.groupDocs, SlimDocument{SourceID: s.firstID})+ s.groupSize = 0+ s.firstID = ""+ }+ if s.groupSize == 0 {+ s.firstID = discordDocIDPrefix + item.message.ID+ s.currentTarget = item.target.channelID+ }+ s.groupSize+++ if s.groupSize >= s.connector.batchSize {+ s.groupDocs = append(s.groupDocs, SlimDocument{SourceID: s.firstID})+ s.groupSize = 0+ s.firstID = ""+ s.currentTarget = ""+ }+ }++ if len(s.groupDocs) == 0 && s.groupSize > 0 {+ s.groupDocs = append(s.groupDocs, SlimDocument{SourceID: s.firstID})+ s.groupSize = 0+ s.firstID = ""+ s.currentTarget = ""+ }+ if len(s.groupDocs) == 0 {+ return PruneBatch{}, io.EOF+ }++ batch := s.groupDocs+ s.groupDocs = nil+ return PruneBatch{Documents: batch}, nil+}++// Close releases the prune session.+func (s *discordPruneSession) Close() error {+ return nil+}diff --git a/internal/syncer/connector/discord_test.go b/internal/syncer/connector/discord_test.gonew file mode 100644index 0000000000..fb18842fa6--- /dev/null+++ b/internal/syncer/connector/discord_test.go@@ -0,0 +1,882 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync/atomic"+ "testing"+ "time"+)++// discordFixture is an in-memory Discord API stub.+type discordFixture struct {+ guilds []map[string]any+ channels map[string][]map[string]any // guildID -> channels+ activeThread map[string][]map[string]any // guildID -> threads+ archived map[string][]map[string]any // channelID -> threads (public)+ messages map[string][]map[string]any // channelID -> messages (newest first)++ forbidPrivateArchived bool+ forbidChannels map[string]bool // channelID -> messages endpoint returns 403+ rateLimitOnce atomic.Bool+ failArchivedThreads bool+ failActiveThreads bool++ requests atomic.Int64+}++func newDiscordFixture() *discordFixture {+ return &discordFixture{+ guilds: []map[string]any{+ {"id": "1001", "name": "Guild One"},+ {"id": "2001", "name": "Guild Two"},+ },+ channels: map[string][]map[string]any{+ "1001": {+ {"id": "ch-1", "name": "general", "type": 0, "guild_id": "1001"},+ {"id": "ch-2", "name": "announcements", "type": 0, "guild_id": "1001"},+ {"id": "ch-3", "name": "mod-only", "type": 0, "guild_id": "1001"},+ {"id": "cat-1", "name": "Category", "type": 4, "guild_id": "1001"},+ {"id": "forum-1", "name": "Forum", "type": 15, "guild_id": "1001"},+ },+ "2001": {+ {"id": "ch-g2", "name": "general", "type": 0, "guild_id": "2001"},+ },+ },+ activeThread: map[string][]map[string]any{+ "1001": {+ {"id": "thread-a", "name": "Active Thread", "type": 12, "guild_id": "1001", "parent_id": "ch-1"},+ {"id": "thread-other", "name": "Other Thread", "type": 12, "guild_id": "1001", "parent_id": "ch-2"},+ },+ },+ archived: map[string][]map[string]any{+ "ch-1": {+ {"id": "thread-b", "name": "Archived Thread", "type": 11, "guild_id": "1001", "parent_id": "ch-1"},+ },+ },+ messages: map[string][]map[string]any{},+ }+}++func (f *discordFixture) setMessages(channelID string, messages ...map[string]any) {+ f.messages[channelID] = messages+}++func (f *discordFixture) handler() http.Handler {+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ f.requests.Add(1)+ path := r.URL.Path+ query := r.URL.Query()++ switch {+ case path == "/users/@me/guilds":+ json.NewEncoder(w).Encode(f.guilds)+ return+ case strings.HasPrefix(path, "/guilds/") && strings.HasSuffix(path, "/channels"):+ guildID := strings.TrimSuffix(strings.TrimPrefix(path, "/guilds/"), "/channels")+ json.NewEncoder(w).Encode(f.channels[guildID])+ return+ case strings.HasPrefix(path, "/guilds/") && strings.HasSuffix(path, "/threads/active"):+ guildID := strings.TrimSuffix(strings.TrimPrefix(path, "/guilds/"), "/threads/active")+ if f.failActiveThreads {+ http.Error(w, "server error", http.StatusInternalServerError)+ return+ }+ json.NewEncoder(w).Encode(map[string]any{"threads": f.activeThread[guildID]})+ return+ case strings.HasPrefix(path, "/channels/") && strings.Contains(path, "/threads/archived/"):+ channelID := strings.TrimPrefix(path, "/channels/")+ channelID = strings.TrimSuffix(channelID, "/threads/archived/public")+ channelID = strings.TrimSuffix(channelID, "/threads/archived/private")+ if strings.Contains(r.URL.Path, "/threads/archived/private") && f.forbidPrivateArchived {+ http.Error(w, "forbidden", http.StatusForbidden)+ return+ }+ if f.failArchivedThreads {+ http.Error(w, "server error", http.StatusInternalServerError)+ return+ }+ all := f.archived[channelID]+ threads := all+ start := 0+ if before := query.Get("before"); before != "" {+ for i, thread := range all {+ if thread["id"] == before {+ start = i + 1+ break+ }+ }+ threads = threads[start:]+ }+ json.NewEncoder(w).Encode(map[string]any{"threads": threads, "has_more": len(all) > start+len(threads)})+ return+ case strings.HasPrefix(path, "/channels/") && strings.HasSuffix(path, "/messages"):+ if f.rateLimitOnce.CompareAndSwap(false, true) {+ w.Header().Set("Retry-After", "0.001")+ w.WriteHeader(http.StatusTooManyRequests)+ json.NewEncoder(w).Encode(map[string]any{"message": "rate limited"})+ return+ }+ channelID := strings.TrimSuffix(strings.TrimPrefix(path, "/channels/"), "/messages")+ if f.forbidChannels[channelID] {+ http.Error(w, "forbidden", http.StatusForbidden)+ return+ }+ all := f.messages[channelID]+ start := 0+ if before := query.Get("before"); before != "" {+ for i, message := range all {+ if message["id"] == before {+ start = i + 1+ break+ }+ }+ }+ limit := 100+ end := start + limit+ if end > len(all) {+ end = len(all)+ }+ json.NewEncoder(w).Encode(all[start:end])+ return+ default:+ http.NotFound(w, r)+ }+ })+}++func fixtureMessage(id, content, author string, ts time.Time) map[string]any {+ return map[string]any{+ "id": id,+ "content": content,+ "type": 0,+ "timestamp": ts.Format("2006-01-02T15:04:05.000000+00:00"),+ "edited_timestamp": nil,+ "author": map[string]any{"id": "u-" + id, "username": author, "bot": false},+ }+}++func newDiscordTestConnector(t *testing.T, config map[string]any) *DiscordConnector {+ t.Helper()+ connector, err := NewDiscordConnector(config)+ if err != nil {+ t.Fatalf("NewDiscordConnector: %v", err)+ }+ return connector+}++func TestDiscordConfigParsing(t *testing.T) {+ connector := newDiscordTestConnector(t, map[string]any{+ "server_ids": []any{"1", "2"},+ "channels": []any{"general", "announcements"},+ "batch_size": 7,+ "start_date": "2026-01-02",+ "credentials": map[string]any{"discord_bot_token": "Bot token-1"},+ })+ if connector.batchSize != 7 {+ t.Fatalf("batchSize = %d, want 7", connector.batchSize)+ }+ if len(connector.serverIDs) != 2 {+ t.Fatalf("serverIDs = %d, want 2", len(connector.serverIDs))+ }+ if _, ok := connector.serverIDs["2"]; !ok {+ t.Fatalf("serverIDs missing id 2")+ }+ if len(connector.channelNames) != 2 || connector.channelNames[0] != "general" {+ t.Fatalf("channelNames = %v", connector.channelNames)+ }+ if connector.startDate.Format("2006-01-02") != "2026-01-02" {+ t.Fatalf("startDate = %v", connector.startDate)+ }+ if connector.token != "token-1" {+ t.Fatalf("token = %q, want token-1", connector.token)+ }++ // comma-separated strings and the legacy channel_names key are accepted too.+ legacy := newDiscordTestConnector(t, map[string]any{+ "server_ids": "1, 3",+ "channel_names": "general",+ "credentials": map[string]any{"discord_bot_token": "token-2"},+ })+ if len(legacy.serverIDs) != 2 {+ t.Fatalf("legacy serverIDs = %d, want 2", len(legacy.serverIDs))+ }+ if len(legacy.channelNames) != 1 || legacy.channelNames[0] != "general" {+ t.Fatalf("legacy channelNames = %v", legacy.channelNames)+ }++ // default start date is clamped to the earliest representable Discord time.+ def := newDiscordTestConnector(t, map[string]any{+ "credentials": map[string]any{"discord_bot_token": "token-3"},+ })+ if !def.startDate.Equal(discordEpoch) {+ t.Fatalf("default startDate = %v, want %v", def.startDate, discordEpoch)+ }+}++func TestDiscordConfigParsingInvalidServerID(t *testing.T) {+ for _, serverIDs := range []any{+ []any{"1001", "not-a-number"},+ "1001, not-a-number",+ } {+ _, err := NewDiscordConnector(map[string]any{+ "server_ids": serverIDs,+ "credentials": map[string]any{"discord_bot_token": "token"},+ })+ var valErr *ConnectorValidationError+ if !errors.As(err, &valErr) {+ t.Fatalf("NewDiscordConnector(server_ids=%v) err = %v, want ConnectorValidationError", serverIDs, err)+ }+ }+}++func TestDiscordValidateNoNetwork(t *testing.T) {+ fixture := newDiscordFixture()+ server := httptest.NewServer(fixture.handler())+ defer server.Close()++ connector := newDiscordTestConnector(t, map[string]any{+ "credentials": map[string]any{"discord_bot_token": "token"},+ })+ connector.baseURL = server.URL+ if err := connector.Validate(context.Background()); err != nil {+ t.Fatalf("Validate: %v", err)+ }+ if fixture.requests.Load() != 0 {+ t.Fatalf("Validate made %d network requests, want 0", fixture.requests.Load())+ }++ missing := newDiscordTestConnector(t, map[string]any{"credentials": map[string]any{}})+ var credErr *ConnectorMissingCredentialError+ if err := missing.Validate(context.Background()); !errors.As(err, &credErr) {+ t.Fatalf("Validate missing token err = %v, want ConnectorMissingCredentialError", err)+ }+}++func TestDiscordOpenSyncMergesBatches(t *testing.T) {+ fixture := newDiscordFixture()+ base := mustTime(t, "2026-08-14T10:00:00Z")+ fixture.setMessages("ch-1",+ fixtureMessage("m1", "one", "alice", base),+ fixtureMessage("m2", "two", "bob", base.Add(-time.Minute)),+ fixtureMessage("m3", "three", "carol", base.Add(-2*time.Minute)),+ fixtureMessage("m4", "four", "dave", base.Add(-3*time.Minute)),+ )+ server := httptest.NewServer(fixture.handler())+ defer server.Close()++ connector := newDiscordTestConnector(t, map[string]any{+ "credentials": map[string]any{"discord_bot_token": "token"},+ "batch_size": 2,+ })+ connector.baseURL = server.URL++ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ defer session.Close()++ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("documents = %d, want 1 merged document", len(batch.Documents))+ }+ doc := batch.Documents[0]+ if doc.SourceID != "DISCORD_m1" {+ t.Fatalf("SourceID = %q, want DISCORD_m1", doc.SourceID)+ }+ if string(doc.Blob) != "one\n\ntwo" {+ t.Fatalf("blob = %q", string(doc.Blob))+ }+ if doc.SizeBytes != 6 {+ t.Fatalf("size = %d, want 6 (sum of message contents)", doc.SizeBytes)+ }+ if doc.Fingerprint == "" {+ t.Fatalf("fingerprint is empty")+ }+ if doc.Metadata == nil || doc.Metadata["Channel"] != "general" {+ t.Fatalf("metadata = %v", doc.Metadata)+ }+ if !doc.UpdatedAt.Equal(base) {+ t.Fatalf("UpdatedAt = %v, want %v", doc.UpdatedAt, base)+ }++ second, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("second NextBatch: %v", err)+ }+ if second.Documents[0].SourceID != "DISCORD_m3" || string(second.Documents[0].Blob) != "three\n\nfour" {+ t.Fatalf("second doc = %q %q", second.Documents[0].SourceID, second.Documents[0].Blob)+ }+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v, want io.EOF", err)+ }+}++func TestDiscordOpenSyncWindowFilter(t *testing.T) {+ fixture := newDiscordFixture()+ base := mustTime(t, "2026-08-14T10:00:00Z")+ fixture.setMessages("ch-1",+ fixtureMessage("m1", "new", "alice", base),+ fixtureMessage("m2", "mid-1", "bob", base.Add(-time.Minute)),+ fixtureMessage("m3", "mid-2", "carol", base.Add(-2*time.Minute)),+ fixtureMessage("m4", "old", "dave", base.Add(-3*time.Minute)),+ )+ server := httptest.NewServer(fixture.handler())+ defer server.Close()++ connector := newDiscordTestConnector(t, map[string]any{+ "credentials": map[string]any{"discord_bot_token": "token"},+ "batch_size": 10,+ })+ connector.baseURL = server.URL++ start := base.Add(-2 * time.Minute)+ end := base+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ FromBeginning: false,+ WindowStart: &start,+ WindowEnd: end,+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ defer session.Close()++ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("documents = %d, want 1", len(batch.Documents))+ }+ if string(batch.Documents[0].Blob) != "mid-1\n\nmid-2" {+ t.Fatalf("blob = %q, want mid-1 + mid-2", batch.Documents[0].Blob)+ }+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v, want io.EOF", err)+ }+}++func TestDiscordOpenSyncChannelsAndThreads(t *testing.T) {+ fixture := newDiscordFixture()+ fixture.forbidPrivateArchived = true+ fixture.forbidChannels = map[string]bool{"ch-3": true}+ base := mustTime(t, "2026-08-14T10:00:00Z")+ fixture.setMessages("ch-1", fixtureMessage("m-ch", "channel msg", "alice", base))+ fixture.setMessages("thread-a", fixtureMessage("m-ta", "active thread msg", "bob", base))+ fixture.setMessages("thread-b", fixtureMessage("m-tb", "archived thread msg", "carol", base))+ server := httptest.NewServer(fixture.handler())+ defer server.Close()++ connector := newDiscordTestConnector(t, map[string]any{+ "server_ids": []any{"1001"},+ "channels": []any{"general", "mod-only"},+ "credentials": map[string]any{"discord_bot_token": "token"},+ "batch_size": 10,+ })+ connector.baseURL = server.URL++ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ defer session.Close()++ var blobs []string+ for {+ batch, err := session.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ blobs = append(blobs, string(batch.Documents[0].Blob))+ }+ want := []string{"channel msg", "archived thread msg", "active thread msg"}+ if len(blobs) != len(want) {+ t.Fatalf("documents = %v, want %v", blobs, want)+ }+ for i := range want {+ if blobs[i] != want[i] {+ t.Fatalf("document %d = %q, want %q", i, blobs[i], want[i])+ }+ }+}++func TestDiscordThreadListErrorsPropagate(t *testing.T) {+ for _, tc := range []struct {+ name string+ fail func(*discordFixture)+ }{+ {"archived", func(f *discordFixture) { f.failArchivedThreads = true }},+ {"active", func(f *discordFixture) { f.failActiveThreads = true }},+ } {+ t.Run(tc.name, func(t *testing.T) {+ fixture := newDiscordFixture()+ tc.fail(fixture)+ server := httptest.NewServer(fixture.handler())+ defer server.Close()++ connector := newDiscordTestConnector(t, map[string]any{+ "credentials": map[string]any{"discord_bot_token": "token"},+ })+ connector.baseURL = server.URL++ if _, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true}); err == nil {+ t.Fatalf("OpenSync: want error, got nil")+ }+ if _, err := connector.OpenPrune(context.Background(), PruneRequest{}); err == nil {+ t.Fatalf("OpenPrune: want error, got nil")+ }+ })+ }+}++func TestDiscordResumeSameTarget(t *testing.T) {+ fixture := newDiscordFixture()+ base := mustTime(t, "2026-08-14T10:00:00Z")+ fixture.setMessages("ch-1",+ fixtureMessage("m1", "one", "alice", base),+ fixtureMessage("m2", "two", "bob", base.Add(-time.Minute)),+ fixtureMessage("m3", "three", "carol", base.Add(-2*time.Minute)),+ fixtureMessage("m4", "four", "dave", base.Add(-3*time.Minute)),+ )+ server := httptest.NewServer(fixture.handler())+ defer server.Close()++ newConnector := func() *DiscordConnector {+ connector := newDiscordTestConnector(t, map[string]any{+ "credentials": map[string]any{"discord_bot_token": "token"},+ "batch_size": 2,+ })+ connector.baseURL = server.URL+ return connector+ }++ session, err := newConnector().OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ session.Close()+ if batch.Checkpoint == nil {+ t.Fatalf("batch checkpoint is nil")+ }+ var cursor discordSyncCursor+ if err := json.Unmarshal([]byte(batch.Checkpoint.Cursor), &cursor); err != nil {+ t.Fatalf("cursor unmarshal: %v", err)+ }+ if cursor.Target != "ch-1" || cursor.Message != "m2" || cursor.Targets == "" {+ t.Fatalf("cursor = %+v, want target ch-1 message m2", cursor)+ }++ resumed, err := newConnector().OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ Resume: batch.Checkpoint,+ })+ if err != nil {+ t.Fatalf("resumed OpenSync: %v", err)+ }+ defer resumed.Close()++ first, err := resumed.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("resumed NextBatch: %v", err)+ }+ if first.Documents[0].SourceID != "DISCORD_m3" || string(first.Documents[0].Blob) != "three\n\nfour" {+ t.Fatalf("resumed doc = %q %q, want DISCORD_m3 three+four", first.Documents[0].SourceID, first.Documents[0].Blob)+ }+ if _, err := resumed.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("resumed NextBatch EOF = %v, want io.EOF", err)+ }+}++func TestDiscordResumeAcrossTargets(t *testing.T) {+ fixture := newDiscordFixture()+ base := mustTime(t, "2026-08-14T10:00:00Z")+ fixture.setMessages("ch-1",+ fixtureMessage("m1", "one", "alice", base),+ fixtureMessage("m2", "two", "bob", base.Add(-time.Minute)),+ fixtureMessage("m3", "three", "carol", base.Add(-2*time.Minute)),+ fixtureMessage("m4", "four", "dave", base.Add(-3*time.Minute)),+ )+ fixture.setMessages("thread-a",+ fixtureMessage("t1", "thread one", "eve", base.Add(-4*time.Minute)),+ fixtureMessage("t2", "thread two", "frank", base.Add(-5*time.Minute)),+ )+ server := httptest.NewServer(fixture.handler())+ defer server.Close()++ newConnector := func() *DiscordConnector {+ connector := newDiscordTestConnector(t, map[string]any{+ "server_ids": []any{"1001"},+ "channels": []any{"general"},+ "credentials": map[string]any{"discord_bot_token": "token"},+ "batch_size": 3,+ })+ connector.baseURL = server.URL+ return connector+ }++ session, err := newConnector().OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ session.Close()+ if batch.Checkpoint == nil {+ t.Fatalf("batch checkpoint is nil")+ }++ resumed, err := newConnector().OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ Resume: batch.Checkpoint,+ })+ if err != nil {+ t.Fatalf("resumed OpenSync: %v", err)+ }+ defer resumed.Close()++ first, err := resumed.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("resumed NextBatch: %v", err)+ }+ if first.Documents[0].SourceID != "DISCORD_m4" || string(first.Documents[0].Blob) != "four" {+ t.Fatalf("resumed doc = %q %q, want DISCORD_m4", first.Documents[0].SourceID, first.Documents[0].Blob)+ }++ second, err := resumed.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("resumed second NextBatch: %v", err)+ }+ if second.Documents[0].SourceID != "DISCORD_t1" || string(second.Documents[0].Blob) != "thread one\n\nthread two" {+ t.Fatalf("resumed second doc = %q %q", second.Documents[0].SourceID, second.Documents[0].Blob)+ }+ if _, err := resumed.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("resumed NextBatch EOF = %v, want io.EOF", err)+ }+}++func TestDiscordResumeFingerprintMismatch(t *testing.T) {+ fixture := newDiscordFixture()+ base := mustTime(t, "2026-08-14T10:00:00Z")+ fixture.setMessages("ch-1",+ fixtureMessage("m1", "one", "alice", base),+ fixtureMessage("m2", "two", "bob", base.Add(-time.Minute)),+ )+ server := httptest.NewServer(fixture.handler())+ defer server.Close()++ // Run 1 enumerates all text channels of the guild.+ firstConnector := newDiscordTestConnector(t, map[string]any{+ "server_ids": []any{"1001"},+ "credentials": map[string]any{"discord_bot_token": "token"},+ "batch_size": 2,+ })+ firstConnector.baseURL = server.URL+ session, err := firstConnector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ session.Close()++ // Run 2 filters to one channel, changing the enumeration: resume is ignored.+ secondConnector := newDiscordTestConnector(t, map[string]any{+ "server_ids": []any{"1001"},+ "channels": []any{"general"},+ "credentials": map[string]any{"discord_bot_token": "token"},+ "batch_size": 2,+ })+ secondConnector.baseURL = server.URL+ resumed, err := secondConnector.OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ Resume: batch.Checkpoint,+ })+ if err != nil {+ t.Fatalf("resumed OpenSync: %v", err)+ }+ defer resumed.Close()++ first, err := resumed.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("resumed NextBatch: %v", err)+ }+ if first.Documents[0].SourceID != "DISCORD_m1" {+ t.Fatalf("resumed doc = %q, want full re-scan from DISCORD_m1", first.Documents[0].SourceID)+ }+}++func TestDiscordResumeInvalidCursor(t *testing.T) {+ fixture := newDiscordFixture()+ base := mustTime(t, "2026-08-14T10:00:00Z")+ fixture.setMessages("ch-1", fixtureMessage("m1", "one", "alice", base))+ server := httptest.NewServer(fixture.handler())+ defer server.Close()++ connector := newDiscordTestConnector(t, map[string]any{+ "credentials": map[string]any{"discord_bot_token": "token"},+ "batch_size": 2,+ })+ connector.baseURL = server.URL++ session, err := connector.OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ Resume: &SyncCheckpoint{Cursor: "not-json"},+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ defer session.Close()++ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if batch.Documents[0].SourceID != "DISCORD_m1" {+ t.Fatalf("doc = %q, want full re-scan from DISCORD_m1", batch.Documents[0].SourceID)+ }+}++func TestDiscordMessagePagination(t *testing.T) {+ fixture := newDiscordFixture()+ base := mustTime(t, "2026-08-14T10:00:00Z")+ messages := make([]map[string]any, 0, 250)+ for i := 0; i < 250; i++ {+ messages = append(messages, fixtureMessage(+ fmt.Sprintf("m%03d", i),+ fmt.Sprintf("content %d", i),+ "alice",+ base.Add(-time.Duration(i)*time.Minute),+ ))+ }+ fixture.setMessages("ch-1", messages...)+ server := httptest.NewServer(fixture.handler())+ defer server.Close()++ connector := newDiscordTestConnector(t, map[string]any{+ "credentials": map[string]any{"discord_bot_token": "token"},+ "batch_size": 10,+ })+ connector.baseURL = server.URL++ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ defer session.Close()++ total := 0+ for {+ batch, err := session.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ total += len(batch.Documents[0].Blob)+ }+ if total == 0 {+ t.Fatalf("no documents streamed")+ }+ if fixture.requests.Load() < 3 {+ t.Fatalf("message requests = %d, want pagination across multiple pages", fixture.requests.Load())+ }+}++func TestDiscord429Retry(t *testing.T) {+ fixture := newDiscordFixture()+ base := mustTime(t, "2026-08-14T10:00:00Z")+ fixture.setMessages("ch-1", fixtureMessage("m1", "hello", "alice", base))+ server := httptest.NewServer(fixture.handler())+ defer server.Close()++ connector := newDiscordTestConnector(t, map[string]any{+ "credentials": map[string]any{"discord_bot_token": "token"},+ "batch_size": 10,+ })+ connector.baseURL = server.URL++ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ defer session.Close()++ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch after 429: %v", err)+ }+ if string(batch.Documents[0].Blob) != "hello" {+ t.Fatalf("blob = %q", batch.Documents[0].Blob)+ }+}++func TestDiscordPruneSlimDocuments(t *testing.T) {+ fixture := newDiscordFixture()+ base := mustTime(t, "2026-08-14T10:00:00Z")+ fixture.setMessages("ch-1",+ fixtureMessage("m1", "one", "alice", base),+ fixtureMessage("m2", "two", "bob", base.Add(-time.Minute)),+ fixtureMessage("m3", "three", "carol", base.Add(-2*time.Minute)),+ fixtureMessage("m4", "four", "dave", base.Add(-3*time.Minute)),+ fixtureMessage("m5", "five", "eve", base.Add(-4*time.Minute)),+ )+ server := httptest.NewServer(fixture.handler())+ defer server.Close()++ connector := newDiscordTestConnector(t, map[string]any{+ "credentials": map[string]any{"discord_bot_token": "token"},+ "batch_size": 2,+ })+ connector.baseURL = server.URL++ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune: %v", err)+ }+ defer session.Close()++ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(first.Documents) != 2 {+ t.Fatalf("first slim batch = %d, want 2", len(first.Documents))+ }+ if first.Documents[0].SourceID != "DISCORD_m1" || first.Documents[1].SourceID != "DISCORD_m3" {+ t.Fatalf("slim ids = %q, %q", first.Documents[0].SourceID, first.Documents[1].SourceID)+ }++ second, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("second NextBatch: %v", err)+ }+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "DISCORD_m5" {+ t.Fatalf("second slim batch = %v", second.Documents)+ }+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v, want io.EOF", err)+ }+}++func TestDiscordPruneGroupBreaksOnTargetSwitch(t *testing.T) {+ fixture := newDiscordFixture()+ base := mustTime(t, "2026-08-14T10:00:00Z")+ fixture.setMessages("ch-1",+ fixtureMessage("m1", "one", "alice", base),+ fixtureMessage("m2", "two", "bob", base.Add(-time.Minute)),+ )+ fixture.setMessages("thread-a",+ fixtureMessage("m3", "three", "carol", base.Add(-2*time.Minute)),+ fixtureMessage("m4", "four", "dave", base.Add(-3*time.Minute)),+ )+ server := httptest.NewServer(fixture.handler())+ defer server.Close()++ connector := newDiscordTestConnector(t, map[string]any{+ "credentials": map[string]any{"discord_bot_token": "token"},+ "batch_size": 10,+ })+ connector.baseURL = server.URL++ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune: %v", err)+ }+ defer session.Close()++ var ids []string+ for {+ batch, err := session.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ for _, doc := range batch.Documents {+ ids = append(ids, doc.SourceID)+ }+ }+ want := []string{"DISCORD_m1", "DISCORD_m3"}+ if len(ids) != len(want) {+ t.Fatalf("slim ids = %v, want %v", ids, want)+ }+ for i := range want {+ if ids[i] != want[i] {+ t.Fatalf("slim id %d = %q, want %q", i, ids[i], want[i])+ }+ }+}++func TestDiscordFingerprintStable(t *testing.T) {+ base := mustTime(t, "2026-08-14T10:00:00Z")+ target := discordTarget{channelID: "ch-1", name: "general"}+ item := discordMessageWithTarget{+ message: discordMessage{+ ID: "m1",+ Content: "hello",+ Type: 0,+ Timestamp: base.Format("2006-01-02T15:04:05.000000+00:00"),+ Author: struct {+ ID string `json:"id"`+ Name string `json:"username"`+ Bot bool `json:"bot"`+ }{ID: "u1", Name: "alice"},+ },+ target: target,+ }++ first := discordMessageDocument(item)+ second := discordMessageDocument(item)+ if first.Fingerprint == "" || first.Fingerprint != second.Fingerprint {+ t.Fatalf("fingerprint unstable: %q %q", first.Fingerprint, second.Fingerprint)+ }+ if first.SemanticIdentifier != "alice said in Channel: #general: hello" {+ t.Fatalf("semantic identifier = %q", first.SemanticIdentifier)+ }++ edited := item+ edited.message.Content = "hello world"+ if changed := discordMessageDocument(edited); changed.Fingerprint == first.Fingerprint {+ t.Fatalf("fingerprint did not change with content")+ }+}diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.goindex 856a88ff87..dab0eb69cb 100644--- a/internal/syncer/syncer.go+++ b/internal/syncer/syncer.go@@ -178,6 +178,7 @@ func registerBuiltInConnectors(registry *syncerconnector.Registry) {registerDAOConnector(registry, "rest_api", syncerconnector.NewRestAPIConnector)registerDAOConnector(registry, "mysql", syncerconnector.NewMySQLConnector)registerDAOConnector(registry, "postgresql", syncerconnector.NewPostgreSQLConnector)+ registerDAOConnector(registry, "discord", syncerconnector.NewDiscordConnector)}func registerDAOConnector[T syncerconnector.Connector](registry *syncerconnector.Registry, source string, factory func(map[string]any) (T, error)) {