Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(go-syncer): add Salesforce connector with checkpoint resume (#18674)

mkaaadb2541bff6881
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.go
index 393698a006..4f79a0fa41 100644
--- a/internal/syncer/connector/builtin.go
+++ b/internal/syncer/connector/builtin.go
@@ -28,6 +28,7 @@ import (
func RegisterBuiltIns(registry *Registry) {
registerBuiltIn(registry, "confluence", NewConfluenceConnector)
registerBuiltIn(registry, "rss", NewRSSConnector)
+ registerBuiltIn(registry, "salesforce", NewSalesforceConnector)
registerBuiltIn(registry, "bitbucket", NewBitbucketConnector)
registerBuiltIn(registry, "dropbox", NewDropboxConnector)
registerBuiltIn(registry, "github", NewGitHubConnector)
diff --git a/internal/syncer/connector/salesforce.go b/internal/syncer/connector/salesforce.go
new file mode 100644
index 0000000000..daa8348e37
--- /dev/null
+++ b/internal/syncer/connector/salesforce.go
@@ -0,0 +1,1095 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "ragflow/internal/utility"
+)
+
+const (
+ defaultSalesforceAPIVersion = "v59.0"
+ defaultSalesforceBatchSize = 2
+ salesforceRequestTimeout = 60 * time.Second
+ salesforceTokenExpiryMargin = 5 * time.Minute
+)
+
+// Default Salesforce CRM objects indexed when the config does not list any.
+// Knowledge__kav is optional: orgs without Salesforce Knowledge silently skip it.
+var salesforceDefaultObjects = []string{"Account", "Contact", "Opportunity", "Case", "Knowledge__kav"}
+
+// salesforceOptionalObjects are silently skipped when absent from the org.
+var salesforceOptionalObjects = map[string]bool{"Knowledge__kav": true}
+
+// salesforceObjectUnavailableError reports that an SObject is genuinely
+// absent or not queryable (HTTP 404, or 400 INVALID_TYPE) rather than a
+// transient/permission failure. Callers skip such objects without failing
+// the whole run; 403/5xx/429 must still abort.
+type salesforceObjectUnavailableError struct {
+ message string
+}
+
+func (e *salesforceObjectUnavailableError) Error() string { return e.message }
+
+// salesforceHTTPError carries a non-2xx Salesforce REST response.
+type salesforceHTTPError struct {
+ status int
+ body string
+}
+
+func (e *salesforceHTTPError) Error() string {
+ return fmt.Sprintf("Salesforce API returned HTTP %d: %s", e.status, e.body)
+}
+
+// salesforceToken is a cached OAuth2 client-credentials token.
+type salesforceToken struct {
+ AccessToken string
+ InstanceURL string
+ ExpiresAt time.Time
+}
+
+// SalesforceConnector reads Salesforce CRM records through the REST + SOQL
+// APIs. It authenticates with OAuth2 client-credentials against a Connected
+// App, describes the configured SObjects, and uses SystemModstamp as the
+// incremental cursor so re-syncs only fetch what changed.
+type SalesforceConnector struct {
+ instanceURL string
+ clientID string
+ clientSecret string
+ objects []string
+ apiVersion string
+ batchSize int
+
+ clientMu sync.Mutex
+ accessToken string
+ tokenExpiry time.Time
+ now func() time.Time
+
+ acquireAccessToken func(ctx context.Context) (salesforceToken, error)
+ doJSON func(ctx context.Context, apiURL string, out any) error
+}
+
+// NewSalesforceConnector creates a Salesforce connector from config.
+func NewSalesforceConnector(config map[string]any) (*SalesforceConnector, error) {
+ credentials, _ := config["credentials"].(map[string]any)
+ instanceURL := strings.TrimRight(strings.TrimSpace(stringConfig(credentials["instance_url"])), "/")
+ objects := salesforceObjects(config["objects"])
+ return &SalesforceConnector{
+ instanceURL: instanceURL,
+ clientID: strings.TrimSpace(stringConfig(credentials["client_id"])),
+ clientSecret: stringConfig(credentials["client_secret"]),
+ objects: objects,
+ apiVersion: firstNonEmpty(stringConfig(config["api_version"]), defaultSalesforceAPIVersion),
+ batchSize: salesforceBatchSize(config["batch_size"]),
+ now: time.Now,
+ }, nil
+}
+
+// salesforceObjects normalizes a comma string or JSON list of object names.
+func salesforceObjects(value any) []string {
+ switch typed := value.(type) {
+ case string:
+ out := []string{}
+ for _, part := range strings.Split(typed, ",") {
+ if part = strings.TrimSpace(part); part != "" {
+ out = append(out, part)
+ }
+ }
+ if len(out) > 0 {
+ return out
+ }
+ case []any:
+ out := []string{}
+ for _, item := range typed {
+ if part := strings.TrimSpace(fmt.Sprint(item)); part != "" {
+ out = append(out, part)
+ }
+ }
+ if len(out) > 0 {
+ return out
+ }
+ }
+ return append([]string(nil), salesforceDefaultObjects...)
+}
+
+// salesforceBatchSize preserves explicit non-positive values so validation can
+// reject them; only missing/unparseable values fall back to the default.
+func salesforceBatchSize(value any) int {
+ switch typed := value.(type) {
+ case int:
+ return typed
+ case int64:
+ return int(typed)
+ case float64:
+ return int(typed)
+ case json.Number:
+ if v, err := typed.Int64(); err == nil {
+ return int(v)
+ }
+ case string:
+ if v, err := strconv.Atoi(strings.TrimSpace(typed)); err == nil {
+ return v
+ }
+ }
+ return defaultSalesforceBatchSize
+}
+
+// Validate validates Salesforce connector settings and credentials.
+func (c *SalesforceConnector) Validate(ctx context.Context) error {
+ if c == nil {
+ return fmt.Errorf("salesforce connector is nil")
+ }
+ if c.instanceURL == "" || c.clientID == "" || c.clientSecret == "" {
+ return &ConnectorMissingCredentialError{Message: "Salesforce credentials are incomplete (instance_url, client_id, client_secret required)"}
+ }
+ if c.batchSize <= 0 {
+ return &ConnectorValidationError{Message: "batch_size must be a positive integer"}
+ }
+ if _, err := c.token(ctx); err != nil {
+ return err
+ }
+
+ var payload salesforceSObjectsResponse
+ if err := c.getJSON(ctx, "/sobjects", &payload); err != nil {
+ var httpErr *salesforceHTTPError
+ if errors.As(err, &httpErr) {
+ switch httpErr.status {
+ case http.StatusUnauthorized:
+ return &ConnectorMissingCredentialError{Message: "Salesforce access token is invalid or expired."}
+ case http.StatusForbidden:
+ return &ConnectorValidationError{Message: "The Salesforce execution user lacks API access; enable the 'API Enabled' profile permission."}
+ default:
+ return &ConnectorValidationError{Message: fmt.Sprintf("Salesforce validation failed (HTTP %d): %s", httpErr.status, httpErr.body)}
+ }
+ }
+ return err
+ }
+
+ queryable := map[string]bool{}
+ for _, so := range payload.SObjects {
+ if so.Name != "" {
+ queryable[so.Name] = so.Queryable
+ }
+ }
+ unknown := []string{}
+ notQueryable := []string{}
+ for _, obj := range c.objects {
+ queryableFlag, ok := queryable[obj]
+ if !ok {
+ if salesforceOptionalObjects[obj] {
+ continue
+ }
+ unknown = append(unknown, obj)
+ } else if !queryableFlag {
+ notQueryable = append(notQueryable, obj)
+ }
+ }
+ if len(unknown) > 0 || len(notQueryable) > 0 {
+ problems := []string{}
+ if len(unknown) > 0 {
+ sort.Strings(unknown)
+ problems = append(problems, fmt.Sprintf("unknown object(s): %s", strings.Join(unknown, ", ")))
+ }
+ if len(notQueryable) > 0 {
+ sort.Strings(notQueryable)
+ problems = append(problems, fmt.Sprintf("non-queryable object(s): %s", strings.Join(notQueryable, ", ")))
+ }
+ return &ConnectorValidationError{Message: "Salesforce 'objects' configuration is invalid — " + strings.Join(problems, "; ") + ". Check for typos and that the execution user has read access to each object."}
+ }
+ return nil
+}
+
+// ValidateConnectorSetting validates Salesforce settings from an unsaved config.
+func (c *SalesforceConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
+ defer cancel()
+ tmp, err := NewSalesforceConnector(request)
+ if err != nil {
+ return err
+ }
+ // Carry the receiver's transport/acquire stubs so tests can validate an
+ // unsaved request without touching the network; production leaves them unset.
+ tmp.acquireAccessToken = c.acquireAccessToken
+ tmp.doJSON = c.doJSON
+ return tmp.Validate(ctx)
+}
+
+// OpenSync opens one Salesforce sync session.
+func (c *SalesforceConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
+ session := &salesforceSyncSession{
+ connector: c,
+ objects: c.objects,
+ batchSize: c.effectiveBatchSize(),
+ windowStart: request.WindowStart,
+ windowEnd: request.WindowEnd,
+ cursors: map[string]salesforceObjectCursor{},
+ }
+ session.applyResume(request.Resume)
+ return session, nil
+}
+
+// OpenPrune opens one complete Salesforce prune snapshot session.
+func (c *SalesforceConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {
+ return &salesforcePruneSession{
+ connector: c,
+ objects: c.objects,
+ batchSize: c.effectiveBatchSize(),
+ }, nil
+}
+
+func (c *SalesforceConnector) effectiveBatchSize() int {
+ if c.batchSize > 0 {
+ return c.batchSize
+ }
+ return defaultSalesforceBatchSize
+}
+
+// instanceBaseURL returns the current canonical instance URL under the auth
+// lock. It is only safe to read through this accessor once requests share the
+// auth lock with token acquisition.
+func (c *SalesforceConnector) instanceBaseURL() string {
+ c.clientMu.Lock()
+ defer c.clientMu.Unlock()
+ return c.instanceURL
+}
+
+// token returns a synchronized authentication snapshot: the access token, its
+// expiry, and the canonical instance URL to build request targets from. The
+// snapshot is captured atomically so callers never read or write instanceURL
+// without the auth lock, and API URLs can be constructed from a single
+// consistent view of the token exchange.
+func (c *SalesforceConnector) token(ctx context.Context) (salesforceToken, error) {
+ c.clientMu.Lock()
+ if c.accessToken != "" && !c.cachedTokenExpiredLocked() {
+ snap := salesforceToken{
+ AccessToken: c.accessToken,
+ InstanceURL: c.instanceURL,
+ ExpiresAt: c.tokenExpiry,
+ }
+ c.clientMu.Unlock()
+ return snap, nil
+ }
+ c.clientMu.Unlock()
+
+ var cached salesforceToken
+ var err error
+ if c.acquireAccessToken != nil {
+ cached, err = c.acquireAccessToken(ctx)
+ } else {
+ cached, err = c.requestAccessToken(ctx)
+ }
+ if err != nil {
+ return salesforceToken{}, err
+ }
+ if cached.AccessToken == "" {
+ return salesforceToken{}, &ConnectorMissingCredentialError{Message: "Salesforce token response did not contain access_token"}
+ }
+ c.clientMu.Lock()
+ c.accessToken = cached.AccessToken
+ c.tokenExpiry = cached.ExpiresAt
+ if cached.InstanceURL != "" {
+ c.instanceURL = strings.TrimRight(cached.InstanceURL, "/")
+ }
+ snap := salesforceToken{
+ AccessToken: cached.AccessToken,
+ InstanceURL: c.instanceURL,
+ ExpiresAt: cached.ExpiresAt,
+ }
+ c.clientMu.Unlock()
+ return snap, nil
+}
+
+func (c *SalesforceConnector) cachedTokenExpiredLocked() bool {
+ return c.tokenExpiry.IsZero() || !c.currentTime().Add(salesforceTokenExpiryMargin).Before(c.tokenExpiry)
+}
+
+func (c *SalesforceConnector) invalidateToken(token string) {
+ c.clientMu.Lock()
+ defer c.clientMu.Unlock()
+ if c.accessToken == token {
+ c.accessToken = ""
+ c.tokenExpiry = time.Time{}
+ }
+}
+
+func (c *SalesforceConnector) currentTime() time.Time {
+ if c.now != nil {
+ return c.now()
+ }
+ return time.Now()
+}
+
+// salesforceHostAllowed reports whether a host is an approved Salesforce
+// instance host. The configured and token-returned instance URLs must stay on
+// Salesforce-owned domains so credentials are only ever transmitted to the
+// intended provider.
+func salesforceHostAllowed(host string) bool {
+ host = strings.ToLower(strings.TrimSpace(host))
+ if host == "" {
+ return false
+ }
+ return host == "salesforce.com" ||
+ host == "force.com" ||
+ strings.HasSuffix(host, ".salesforce.com") ||
+ strings.HasSuffix(host, ".my.salesforce.com") ||
+ strings.HasSuffix(host, ".force.com") ||
+ strings.HasSuffix(host, ".lightning.force.com")
+}
+
+// requestAccessToken performs the OAuth2 client-credentials exchange, validating
+// the token endpoint for SSRF, HTTPS, and the approved Salesforce host policy
+// before any credentials are transmitted.
+func (c *SalesforceConnector) requestAccessToken(ctx context.Context) (salesforceToken, error) {
+ tokenURL := c.instanceBaseURL() + "/services/oauth2/token"
+ hostname, resolvedIP, err := utility.AssertURLSafe(tokenURL)
+ if err != nil {
+ return salesforceToken{}, &ConnectorMissingCredentialError{Message: fmt.Sprintf("Salesforce token request failed: %v", err)}
+ }
+ if !salesforceHostAllowed(hostname) {
+ return salesforceToken{}, &ConnectorMissingCredentialError{Message: "Salesforce instance_url is not an approved Salesforce host"}
+ }
+ parsedURL, err := url.Parse(tokenURL)
+ if err != nil || !strings.EqualFold(parsedURL.Scheme, "https") {
+ return salesforceToken{}, &ConnectorMissingCredentialError{Message: "Salesforce OAuth token endpoint must use HTTPS"}
+ }
+ form := url.Values{
+ "grant_type": {"client_credentials"},
+ "client_id": {c.clientID},
+ "client_secret": {c.clientSecret},
+ }
+ requestCtx, cancel := context.WithTimeout(ctx, salesforceRequestTimeout)
+ defer cancel()
+ req, err := http.NewRequestWithContext(requestCtx, http.MethodPost, tokenURL, strings.NewReader(form.Encode()))
+ if err != nil {
+ return salesforceToken{}, &ConnectorMissingCredentialError{Message: fmt.Sprintf("Salesforce token request failed: %v", err)}
+ }
+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+ client := utility.PinnedHTTPClient(hostname, resolvedIP, salesforceRequestTimeout)
+ resp, err := client.Do(req)
+ if err != nil {
+ return salesforceToken{}, &ConnectorMissingCredentialError{Message: fmt.Sprintf("Salesforce token request failed: %v", err)}
+ }
+ defer resp.Body.Close()
+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+ if resp.StatusCode >= 400 {
+ detail := salesforceTokenErrorDetail(body)
+ return salesforceToken{}, &ConnectorMissingCredentialError{Message: fmt.Sprintf("Failed to acquire Salesforce access token (HTTP %d): %s", resp.StatusCode, detail)}
+ }
+ var payload struct {
+ AccessToken string `json:"access_token"`
+ InstanceURL string `json:"instance_url"`
+ ExpiresIn int `json:"expires_in"`
+ }
+ if err := json.Unmarshal(body, &payload); err != nil {
+ return salesforceToken{}, &ConnectorMissingCredentialError{Message: fmt.Sprintf("Salesforce token response is not JSON: %v", err)}
+ }
+ expiresAt := c.currentTime().Add(time.Hour)
+ if payload.ExpiresIn > 0 {
+ expiresAt = c.currentTime().Add(time.Duration(payload.ExpiresIn) * time.Second)
+ }
+ return salesforceToken{
+ AccessToken: payload.AccessToken,
+ InstanceURL: strings.TrimRight(payload.InstanceURL, "/"),
+ ExpiresAt: expiresAt,
+ }, nil
+}
+
+func salesforceTokenErrorDetail(body []byte) string {
+ var payload struct {
+ Error string `json:"error"`
+ ErrorDescription string `json:"error_description"`
+ }
+ if err := json.Unmarshal(body, &payload); err == nil {
+ if payload.ErrorDescription != "" {
+ return payload.ErrorDescription
+ }
+ if payload.Error != "" {
+ return payload.Error
+ }
+ }
+ text := strings.TrimSpace(string(body))
+ if len(text) > 200 {
+ text = text[:200]
+ }
+ return text
+}
+
+// apiURL builds the full Salesforce REST URL for a service-relative path from
+// the canonical instance URL captured in the authentication snapshot. The
+// token exchange may publish a different canonical instance than the one the
+// operator configured, so every request target is derived from the snapshot
+// instead of stale connector state. Absolute pagination URLs returned by the
+// query API are validated before use: they must use HTTPS on an approved
+// Salesforce host, so a tampered URL can never receive the bearer token.
+func (c *SalesforceConnector) apiURL(snap salesforceToken, path string) (string, error) {
+ if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
+ parsed, err := url.Parse(path)
+ if err != nil {
+ return "", fmt.Errorf("Salesforce pagination URL is invalid: %v", err)
+ }
+ if !strings.EqualFold(parsed.Scheme, "https") || !salesforceHostAllowed(parsed.Hostname()) {
+ return "", fmt.Errorf("Salesforce pagination URL must use HTTPS on an approved Salesforce host")
+ }
+ return path, nil
+ }
+ if strings.HasPrefix(path, "/services/data/") {
+ return snap.InstanceURL + path, nil
+ }
+ return snap.InstanceURL + "/services/data/" + c.apiVersion + path, nil
+}
+
+// getJSON GETs a Salesforce REST endpoint and decodes JSON into out. The apiURL
+// is built only after token acquisition, from the canonical instance URL in the
+// returned snapshot, and a 401 retry rebuilds it from the refreshed snapshot.
+func (c *SalesforceConnector) getJSON(ctx context.Context, path string, out any) error {
+ snap, err := c.token(ctx)
+ if err != nil {
+ return err
+ }
+ apiURL, err := c.apiURL(snap, path)
+ if err != nil {
+ return err
+ }
+ if c.doJSON != nil {
+ return c.doJSON(ctx, apiURL, out)
+ }
+ for attempt := 0; ; attempt++ {
+ status, body, err := c.doGet(ctx, apiURL, snap.AccessToken)
+ if err != nil {
+ return err
+ }
+ if status == http.StatusUnauthorized && attempt == 0 {
+ c.invalidateToken(snap.AccessToken)
+ snap, err = c.token(ctx)
+ if err != nil {
+ return err
+ }
+ apiURL, err = c.apiURL(snap, path)
+ if err != nil {
+ return err
+ }
+ continue
+ }
+ if salesforceObjectUnavailable(status, body) {
+ return &salesforceObjectUnavailableError{message: fmt.Sprintf("Salesforce object unavailable (HTTP %d): %s", status, strings.TrimSpace(string(body)))}
+ }
+ if status >= 400 {
+ return &salesforceHTTPError{status: status, body: strings.TrimSpace(string(body))}
+ }
+ return json.Unmarshal(body, out)
+ }
+}
+
+// doGet performs one authenticated GET with SSRF protection.
+func (c *SalesforceConnector) doGet(ctx context.Context, apiURL, token string) (int, []byte, error) {
+ parsed, err := url.Parse(apiURL)
+ if err != nil || !strings.EqualFold(parsed.Scheme, "https") || !salesforceHostAllowed(parsed.Hostname()) {
+ return 0, nil, fmt.Errorf("Salesforce request URL must use HTTPS on an approved Salesforce host")
+ }
+ hostname, resolvedIP, err := utility.AssertURLSafe(apiURL)
+ if err != nil {
+ return 0, nil, err
+ }
+ client := utility.PinnedHTTPClient(hostname, resolvedIP, salesforceRequestTimeout)
+ requestCtx, cancel := context.WithTimeout(ctx, salesforceRequestTimeout)
+ defer cancel()
+ req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, apiURL, nil)
+ if err != nil {
+ return 0, nil, err
+ }
+ req.Header.Set("Accept", "application/json")
+ req.Header.Set("Authorization", "Bearer "+token)
+ resp, err := client.Do(req)
+ if err != nil {
+ return 0, nil, err
+ }
+ defer resp.Body.Close()
+ body, err := io.ReadAll(io.LimitReader(resp.Body, 32*1024*1024))
+ if err != nil {
+ return 0, nil, err
+ }
+ return resp.StatusCode, body, nil
+}
+
+// salesforceObjectUnavailable reports whether a response indicates a genuinely
+// absent SObject as opposed to a transient, permission, routing, or API-version
+// failure. A 404 is only treated as object-not-found when the response carries
+// the structured Salesforce NOT_FOUND error; a bare 404 (e.g. a bad pagination
+// URL or unknown route) must propagate as a normal HTTP error.
+func salesforceObjectUnavailable(status int, body []byte) bool {
+ if status == http.StatusNotFound {
+ return salesforceNotFoundError(body)
+ }
+ if status != http.StatusBadRequest {
+ return false
+ }
+ return salesforceInvalidTypeError(body)
+}
+
+// salesforceNotFoundError reports whether a response body carries the
+// structured Salesforce "object not found" error code.
+func salesforceNotFoundError(body []byte) bool {
+ var entries []struct {
+ ErrorCode string `json:"errorCode"`
+ }
+ if err := json.Unmarshal(body, &entries); err == nil {
+ for _, entry := range entries {
+ if entry.ErrorCode == "NOT_FOUND" {
+ return true
+ }
+ }
+ return false
+ }
+ var single struct {
+ ErrorCode string `json:"errorCode"`
+ }
+ if err := json.Unmarshal(body, &single); err == nil {
+ return single.ErrorCode == "NOT_FOUND"
+ }
+ return false
+}
+
+// salesforceInvalidTypeError reports whether a 400 response body carries the
+// Salesforce INVALID_TYPE error, which describes an unknowable SOQL object.
+func salesforceInvalidTypeError(body []byte) bool {
+ var entries []struct {
+ ErrorCode string `json:"errorCode"`
+ }
+ if err := json.Unmarshal(body, &entries); err == nil {
+ for _, entry := range entries {
+ if entry.ErrorCode == "INVALID_TYPE" {
+ return true
+ }
+ }
+ return false
+ }
+ var single struct {
+ ErrorCode string `json:"errorCode"`
+ }
+ if err := json.Unmarshal(body, &single); err == nil {
+ return single.ErrorCode == "INVALID_TYPE"
+ }
+ return strings.Contains(string(body), "INVALID_TYPE")
+}
+
+// describeFields returns field API names for an SObject, filtering out
+// compound types (address, location) that SOQL cannot project directly.
+func (c *SalesforceConnector) describeFields(ctx context.Context, obj string) ([]string, error) {
+ var payload struct {
+ Fields []struct {
+ Name string `json:"name"`
+ Type string `json:"type"`
+ } `json:"fields"`
+ }
+ if err := c.getJSON(ctx, "/sobjects/"+url.PathEscape(obj)+"/describe", &payload); err != nil {
+ return nil, err
+ }
+ fields := []string{}
+ for _, field := range payload.Fields {
+ if field.Type == "address" || field.Type == "location" {
+ continue
+ }
+ if field.Name != "" {
+ fields = append(fields, field.Name)
+ }
+ }
+ hasID := false
+ for _, name := range fields {
+ if name == "Id" {
+ hasID = true
+ break
+ }
+ }
+ if !hasID {
+ fields = append([]string{"Id"}, fields...)
+ }
+ return fields, nil
+}
+
+// queryURL builds the SOQL query URL path for one SObject page. When a resume
+// cursor is present, the WHERE clause re-fetches records strictly newer than the
+// checkpoint plus same-instant records whose Id sorts after the checkpoint, so
+// later records sharing the checkpoint timestamp are not skipped.
+func (c *SalesforceConnector) queryURL(obj string, fields []string, cursor *salesforceObjectCursor, until *time.Time) string {
+ fieldList := strings.Join(fields, ",")
+ filters := []string{}
+ if cursor != nil && cursor.SystemModstamp != "" {
+ since := cursor.SystemModstamp
+ if parsed, err := parseSalesforceTime(cursor.SystemModstamp); err == nil {
+ since = salesforceSOQLTime(parsed)
+ }
+ clause := "SystemModstamp > " + since
+ if cursor.Id != "" {
+ clause = "(" + clause + " OR (SystemModstamp = " + since + " AND Id > '" + cursor.Id + "'))"
+ }
+ filters = append(filters, clause)
+ }
+ if until != nil {
+ filters = append(filters, "SystemModstamp <= "+salesforceSOQLTime(*until))
+ }
+ where := ""
+ if len(filters) > 0 {
+ where = " WHERE " + strings.Join(filters, " AND ")
+ }
+ soql := fmt.Sprintf("SELECT %s FROM %s%s ORDER BY SystemModstamp ASC, Id ASC", fieldList, obj, where)
+ return "/query?q=" + url.QueryEscape(soql)
+}
+
+// salesforceSOQLTime formats a timestamp for a SOQL literal.
+func salesforceSOQLTime(t time.Time) string {
+ return t.UTC().Format("2006-01-02T15:04:05Z")
+}
+
+// salesforceWindowEnd returns the inclusive upper window bound, or nil when
+// the window end is unset (e.g. prune scans).
+func salesforceWindowEnd(windowEnd time.Time) *time.Time {
+ if windowEnd.IsZero() {
+ return nil
+ }
+ until := windowEnd
+ return &until
+}
+
+// parseSalesforceTime parses a Salesforce ISO-8601 timestamp.
+func parseSalesforceTime(value string) (time.Time, error) {
+ value = strings.TrimSpace(value)
+ if value == "" {
+ return time.Time{}, fmt.Errorf("empty Salesforce timestamp")
+ }
+ layouts := []string{
+ "2006-01-02T15:04:05.000-0700",
+ "2006-01-02T15:04:05-0700",
+ "2006-01-02T15:04:05.000Z",
+ time.RFC3339Nano,
+ time.RFC3339,
+ }
+ for _, layout := range layouts {
+ if t, err := time.Parse(layout, value); err == nil {
+ return t, nil
+ }
+ }
+ return time.Time{}, fmt.Errorf("parse Salesforce timestamp %q", value)
+}
+
+// salesforceObjectCursor is a per-object resume position keyed by the last
+// ingested record. Storing both SystemModstamp and Id lets a resume re-fetch
+// records that share the checkpoint timestamp but sort after it, so same-instant
+// inserts are not lost across checkpoint boundaries.
+type salesforceObjectCursor struct {
+ SystemModstamp string `json:"system_modstamp,omitempty"`
+ Id string `json:"id,omitempty"`
+}
+
+// salesforceSyncCursor is the per-object composite cursor map.
+type salesforceSyncCursor struct {
+ Cursors map[string]salesforceObjectCursor `json:"cursors,omitempty"`
+}
+
+// salesforceSyncSession streams Salesforce documents for one fixed sync window.
+type salesforceSyncSession struct {
+ connector *SalesforceConnector
+ objects []string
+ batchSize int
+ windowStart *time.Time
+ windowEnd time.Time
+ cursors map[string]salesforceObjectCursor
+
+ objectIndex int
+ pageURL string
+ latestISO string
+ latestID string
+ buffer []salesforceBufferedDocument
+}
+
+type salesforceBufferedDocument struct {
+ document SourceDocument
+ checkpoint *SyncCheckpoint
+}
+
+// applyResume restores the per-object cursor map from a saved checkpoint.
+func (s *salesforceSyncSession) applyResume(checkpoint *SyncCheckpoint) {
+ if checkpoint == nil || checkpoint.Cursor == "" {
+ return
+ }
+ var cursor salesforceSyncCursor
+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {
+ return
+ }
+ if len(cursor.Cursors) > 0 {
+ s.cursors = cursor.Cursors
+ }
+}
+
+// NextBatch returns the next Salesforce document batch.
+func (s *salesforceSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
+ documents := make([]SourceDocument, 0, s.batchSize)
+ var checkpoint *SyncCheckpoint
+ if len(s.buffer) > 0 {
+ n := min(s.batchSize, len(s.buffer))
+ for _, buffered := range s.buffer[:n] {
+ documents = append(documents, buffered.document)
+ checkpoint = buffered.checkpoint
+ }
+ s.buffer = s.buffer[n:]
+ }
+ for len(documents) < s.batchSize {
+ page, err := s.nextDocumentPage(ctx)
+ if err != nil {
+ return SyncBatch{}, err
+ }
+ if len(page) == 0 {
+ if s.objectIndex >= len(s.objects) {
+ if len(documents) == 0 {
+ return SyncBatch{}, io.EOF
+ }
+ break
+ }
+ continue
+ }
+ remaining := s.batchSize - len(documents)
+ if len(page) > remaining {
+ for _, buffered := range page[:remaining] {
+ documents = append(documents, buffered.document)
+ checkpoint = buffered.checkpoint
+ }
+ s.buffer = append(s.buffer, page[remaining:]...)
+ break
+ }
+ for _, buffered := range page {
+ documents = append(documents, buffered.document)
+ checkpoint = buffered.checkpoint
+ }
+ }
+ return SyncBatch{Documents: documents, Checkpoint: checkpoint}, nil
+}
+
+// Close closes the Salesforce sync session.
+func (s *salesforceSyncSession) Close() error {
+ return nil
+}
+
+// nextDocumentPage streams the next batch of raw records for the current
+// object, advancing to the next object when the current one is drained.
+func (s *salesforceSyncSession) nextDocumentPage(ctx context.Context) ([]salesforceBufferedDocument, error) {
+ for s.objectIndex < len(s.objects) {
+ obj := s.objects[s.objectIndex]
+ if s.pageURL == "" {
+ fields, err := s.connector.describeFields(ctx, obj)
+ if err != nil {
+ var unavail *salesforceObjectUnavailableError
+ if errors.As(err, &unavail) {
+ s.objectIndex++
+ continue
+ }
+ return nil, err
+ }
+ cursor := s.objCursor(obj)
+ until := salesforceWindowEnd(s.windowEnd)
+ s.pageURL = s.connector.queryURL(obj, fields, cursor, until)
+ }
+ var page salesforceQueryPage
+ if err := s.connector.getJSON(ctx, s.pageURL, &page); err != nil {
+ var unavail *salesforceObjectUnavailableError
+ if errors.As(err, &unavail) {
+ // Object disappeared mid-query (e.g. Knowledge__kav org lacks
+ // Knowledge). Skip it rather than abort the run.
+ s.objectIndex++
+ s.pageURL = ""
+ continue
+ }
+ return nil, err
+ }
+ raw := make([]SourceDocument, 0, len(page.Records))
+ for _, record := range page.Records {
+ recID := stringRecordValue(record, "Id")
+ if recID == "" {
+ continue
+ }
+ modifiedStr := stringRecordValue(record, "SystemModstamp")
+ if modifiedStr != "" {
+ s.latestISO = modifiedStr
+ s.latestID = recID
+ }
+ raw = append(raw, s.connector.recordToDocument(obj, record, recID, modifiedStr))
+ }
+ done := page.NextRecordsURL == ""
+ // Capture the object's latest cursor before resetting session state.
+ // The cursor only advances once the object is fully drained: the
+ // object's final record checkpoint carries it, while earlier batches
+ // keep the old cursor so a crash between them re-fetches the object
+ // instead of skipping records that were never ingested.
+ finalISO := s.latestISO
+ finalID := s.latestID
+ if page.NextRecordsURL != "" {
+ s.pageURL = page.NextRecordsURL
+ } else {
+ s.objectIndex++
+ s.pageURL = ""
+ s.latestISO = ""
+ s.latestID = ""
+ }
+ documents := make([]salesforceBufferedDocument, 0, len(raw))
+ for index, doc := range raw {
+ if done && index == len(raw)-1 && finalISO != "" {
+ s.cursors[obj] = salesforceObjectCursor{SystemModstamp: finalISO, Id: finalID}
+ }
+ documents = append(documents, salesforceBufferedDocument{
+ document: doc,
+ checkpoint: s.syncCheckpoint(doc),
+ })
+ }
+ if done && len(raw) == 0 && finalISO != "" {
+ // Empty final page: still commit the drained object's cursor so
+ // the next object's batches skip it on resume.
+ s.cursors[obj] = salesforceObjectCursor{SystemModstamp: finalISO, Id: finalID}
+ }
+ if len(documents) > 0 {
+ return documents, nil
+ }
+ }
+ return nil, nil
+}
+
+// objCursor computes the per-object resume cursor: the caller window or the
+// persisted cursor, whichever is later. A window bound that falls after the
+// persisted cursor drops the cursor's Id so same-instant records from the
+// window are not skipped; otherwise the composite timestamp+Id cursor is kept.
+func (s *salesforceSyncSession) objCursor(obj string) *salesforceObjectCursor {
+ cur, ok := s.cursors[obj]
+ if !ok || cur.SystemModstamp == "" {
+ if s.windowStart != nil {
+ return &salesforceObjectCursor{SystemModstamp: salesforceSOQLTime(*s.windowStart)}
+ }
+ return nil
+ }
+ cursorTime, err := parseSalesforceTime(cur.SystemModstamp)
+ if err != nil {
+ if s.windowStart != nil {
+ return &salesforceObjectCursor{SystemModstamp: salesforceSOQLTime(*s.windowStart)}
+ }
+ return &cur
+ }
+ if s.windowStart != nil && cursorTime.Before(*s.windowStart) {
+ return &salesforceObjectCursor{SystemModstamp: salesforceSOQLTime(*s.windowStart)}
+ }
+ return &cur
+}
+
+// syncCheckpoint serializes the current per-object cursor map.
+func (s *salesforceSyncSession) syncCheckpoint(doc SourceDocument) *SyncCheckpoint {
+ cursors := make(map[string]salesforceObjectCursor, len(s.cursors))
+ for obj, cur := range s.cursors {
+ cursors[obj] = cur
+ }
+ data, err := json.Marshal(salesforceSyncCursor{Cursors: cursors})
+ if err != nil {
+ return nil
+ }
+ updatedAt := doc.UpdatedAt
+ return &SyncCheckpoint{
+ Cursor: string(data),
+ SourceID: doc.SourceID,
+ UpdatedAt: &updatedAt,
+ }
+}
+
+// recordToDocument converts a SOQL record into a SourceDocument.
+func (c *SalesforceConnector) recordToDocument(obj string, record map[string]any, recID, modifiedStr string) SourceDocument {
+ updatedAt := time.Now().UTC()
+ if modifiedStr != "" {
+ if t, err := parseSalesforceTime(modifiedStr); err == nil {
+ updatedAt = t
+ }
+ }
+ name := firstNonEmpty(
+ stringRecordValue(record, "Name"),
+ stringRecordValue(record, "Subject"),
+ stringRecordValue(record, "Title"),
+ fmt.Sprintf("%s/%s", obj, recID),
+ )
+ body := salesforceRecordToText(obj, record)
+ blob := []byte(body)
+ return SourceDocument{
+ SourceID: fmt.Sprintf("%s/%s", obj, recID),
+ SemanticIdentifier: name,
+ Extension: ".txt",
+ Blob: blob,
+ UpdatedAt: updatedAt,
+ SizeBytes: int64(len(blob)),
+ Metadata: map[string]any{
+ "object": obj,
+ "record_id": recID,
+ "web_url": fmt.Sprintf("%s/%s", c.instanceBaseURL(), recID),
+ },
+ Fingerprint: contentFingerprint(blob),
+ }
+}
+
+// salesforceRecordToText flattens a SOQL record into a deterministic
+// plain-text body (sorted keys, stable value formatting) so content hashing is
+// stable across polls.
+func salesforceRecordToText(obj string, record map[string]any) string {
+ lines := []string{"Salesforce " + obj}
+ keys := make([]string, 0, len(record))
+ for key := range record {
+ if key == "attributes" {
+ continue
+ }
+ keys = append(keys, key)
+ }
+ sort.Strings(keys)
+ for _, key := range keys {
+ value := record[key]
+ if value == nil || value == "" {
+ continue
+ }
+ if isMapOrSlice(value) {
+ if data, err := json.Marshal(value); err == nil {
+ lines = append(lines, fmt.Sprintf("%s: %s", key, string(data)))
+ continue
+ }
+ }
+ lines = append(lines, fmt.Sprintf("%s: %v", key, value))
+ }
+ return strings.Join(lines, "\n")
+}
+
+func stringRecordValue(record map[string]any, key string) string {
+ value, _ := record[key].(string)
+ return value
+}
+
+func isMapOrSlice(value any) bool {
+ switch value.(type) {
+ case map[string]any, []any:
+ return true
+ default:
+ return false
+ }
+}
+
+// salesforceQueryPage is one SOQL query result page.
+type salesforceQueryPage struct {
+ Records []map[string]any `json:"records"`
+ NextRecordsURL string `json:"nextRecordsUrl"`
+}
+
+// salesforceSObjectsResponse is the /sobjects global describe payload.
+type salesforceSObjectsResponse struct {
+ SObjects []struct {
+ Name string `json:"name"`
+ Queryable bool `json:"queryable"`
+ } `json:"sobjects"`
+}
+
+// salesforcePruneSession streams a complete Salesforce slim snapshot.
+type salesforcePruneSession struct {
+ connector *SalesforceConnector
+ objects []string
+ batchSize int
+ objectIndex int
+ pageURL string
+ buffer []SlimDocument
+}
+
+// NextBatch returns the next Salesforce prune snapshot batch.
+func (s *salesforcePruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {
+ documents := make([]SlimDocument, 0, s.batchSize)
+ if len(s.buffer) > 0 {
+ n := min(s.batchSize, len(s.buffer))
+ documents = append(documents, s.buffer[:n]...)
+ s.buffer = s.buffer[n:]
+ }
+ for len(documents) < s.batchSize {
+ page, err := s.nextSlimPage(ctx)
+ if err != nil {
+ return PruneBatch{}, err
+ }
+ if len(page) == 0 {
+ if s.objectIndex >= len(s.objects) {
+ if len(documents) == 0 {
+ return PruneBatch{}, io.EOF
+ }
+ break
+ }
+ continue
+ }
+ remaining := s.batchSize - len(documents)
+ if len(page) > remaining {
+ documents = append(documents, page[:remaining]...)
+ s.buffer = append(s.buffer, page[remaining:]...)
+ break
+ }
+ documents = append(documents, page...)
+ }
+ return PruneBatch{Documents: documents}, nil
+}
+
+// Close closes the Salesforce prune session.
+func (s *salesforcePruneSession) Close() error {
+ return nil
+}
+
+// nextSlimPage streams the next page of slim IDs for the current object.
+func (s *salesforcePruneSession) nextSlimPage(ctx context.Context) ([]SlimDocument, error) {
+ for s.objectIndex < len(s.objects) {
+ obj := s.objects[s.objectIndex]
+ if s.pageURL == "" {
+ s.pageURL = s.connector.queryURL(obj, []string{"Id"}, nil, nil)
+ }
+ var page salesforceQueryPage
+ if err := s.connector.getJSON(ctx, s.pageURL, &page); err != nil {
+ var unavail *salesforceObjectUnavailableError
+ if errors.As(err, &unavail) {
+ s.objectIndex++
+ s.pageURL = ""
+ continue
+ }
+ return nil, err
+ }
+ documents := make([]SlimDocument, 0, len(page.Records))
+ for _, record := range page.Records {
+ recID := stringRecordValue(record, "Id")
+ if recID == "" {
+ continue
+ }
+ documents = append(documents, SlimDocument{SourceID: fmt.Sprintf("%s/%s", obj, recID)})
+ }
+ if page.NextRecordsURL != "" {
+ s.pageURL = page.NextRecordsURL
+ } else {
+ s.objectIndex++
+ s.pageURL = ""
+ }
+ if len(documents) > 0 {
+ return documents, nil
+ }
+ }
+ return nil, nil
+}
diff --git a/internal/syncer/connector/salesforce_test.go b/internal/syncer/connector/salesforce_test.go
new file mode 100644
index 0000000000..12f9c3d6d5
--- /dev/null
+++ b/internal/syncer/connector/salesforce_test.go
@@ -0,0 +1,1132 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "io"
+ "net/http"
+ "net/url"
+ "strings"
+ "testing"
+ "time"
+
+ "ragflow/internal/utility"
+)
+
+func TestNewSalesforceConnectorDefaults(t *testing.T) {
+ connector, err := NewSalesforceConnector(map[string]any{
+ "credentials": map[string]any{
+ "instance_url": "https://your-domain.my.salesforce.com/",
+ "client_id": "client",
+ "client_secret": "secret",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewSalesforceConnector failed: %v", err)
+ }
+ if connector.instanceURL != "https://your-domain.my.salesforce.com" {
+ t.Fatalf("instance url = %q", connector.instanceURL)
+ }
+ if connector.apiVersion != "v59.0" {
+ t.Fatalf("api version = %q", connector.apiVersion)
+ }
+ if connector.batchSize != 2 {
+ t.Fatalf("batch size = %d", connector.batchSize)
+ }
+ want := []string{"Account", "Contact", "Opportunity", "Case", "Knowledge__kav"}
+ if len(connector.objects) != len(want) {
+ t.Fatalf("objects = %v, want %v", connector.objects, want)
+ }
+ for i := range want {
+ if connector.objects[i] != want[i] {
+ t.Fatalf("objects = %v, want %v", connector.objects, want)
+ }
+ }
+}
+
+func TestNewSalesforceConnectorObjectsAndBatch(t *testing.T) {
+ connector, err := NewSalesforceConnector(map[string]any{
+ "objects": "Account, Contact",
+ "api_version": "v62.0",
+ "batch_size": 5,
+ "credentials": map[string]any{
+ "instance_url": "https://acme.my.salesforce.com",
+ "client_id": "client",
+ "client_secret": "secret",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewSalesforceConnector failed: %v", err)
+ }
+ if len(connector.objects) != 2 || connector.objects[0] != "Account" || connector.objects[1] != "Contact" {
+ t.Fatalf("objects = %v", connector.objects)
+ }
+ if connector.apiVersion != "v62.0" {
+ t.Fatalf("api version = %q", connector.apiVersion)
+ }
+ if connector.batchSize != 5 {
+ t.Fatalf("batch size = %d", connector.batchSize)
+ }
+}
+
+func TestSalesforceConnectorValidateMissingCredentials(t *testing.T) {
+ connector, err := NewSalesforceConnector(map[string]any{"credentials": map[string]any{}})
+ if err != nil {
+ t.Fatalf("NewSalesforceConnector failed: %v", err)
+ }
+ var credErr *ConnectorMissingCredentialError
+ if err := connector.Validate(context.Background()); !errors.As(err, &credErr) {
+ t.Fatalf("Validate err = %v, want ConnectorMissingCredentialError", err)
+ }
+}
+
+func TestSalesforceConnectorValidateRejectsNonPositiveBatch(t *testing.T) {
+ connector, err := NewSalesforceConnector(map[string]any{
+ "batch_size": 0,
+ "credentials": map[string]any{
+ "instance_url": "https://acme.my.salesforce.com",
+ "client_id": "client",
+ "client_secret": "secret",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewSalesforceConnector failed: %v", err)
+ }
+ var valErr *ConnectorValidationError
+ if err := connector.Validate(context.Background()); !errors.As(err, &valErr) {
+ t.Fatalf("Validate err = %v, want ConnectorValidationError", err)
+ }
+}
+
+func TestSalesforceConnectorValidateQueriesObjects(t *testing.T) {
+ connector := newSalesforceFixtureConnector()
+ var probed bool
+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {
+ if !strings.HasSuffix(apiURL, "/services/data/v59.0/sobjects") {
+ t.Fatalf("validate url = %q", apiURL)
+ }
+ probed = true
+ payload := map[string]any{
+ "sobjects": []any{
+ map[string]any{"name": "Account", "queryable": true},
+ map[string]any{"name": "Contact", "queryable": true},
+ map[string]any{"name": "Opportunity", "queryable": true},
+ map[string]any{"name": "Case", "queryable": true},
+ map[string]any{"name": "Knowledge__kav", "queryable": true},
+ },
+ }
+ data, _ := json.Marshal(payload)
+ return json.Unmarshal(data, out)
+ }
+ if err := connector.Validate(context.Background()); err != nil {
+ t.Fatalf("Validate failed: %v", err)
+ }
+ if !probed {
+ t.Fatalf("Validate did not probe /sobjects")
+ }
+}
+
+func TestSalesforceConnectorValidateUnknownObject(t *testing.T) {
+ connector := newSalesforceFixtureConnector()
+ connector.objects = []string{"Account", "Bogus"}
+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {
+ payload := map[string]any{
+ "sobjects": []any{
+ map[string]any{"name": "Account", "queryable": true},
+ },
+ }
+ data, _ := json.Marshal(payload)
+ return json.Unmarshal(data, out)
+ }
+ var valErr *ConnectorValidationError
+ if err := connector.Validate(context.Background()); !errors.As(err, &valErr) {
+ t.Fatalf("Validate err = %v, want ConnectorValidationError", err)
+ }
+}
+
+func TestSalesforceConnectorValidateSkipsOptionalKnowledge(t *testing.T) {
+ connector := newSalesforceFixtureConnector()
+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {
+ payload := map[string]any{
+ "sobjects": []any{
+ map[string]any{"name": "Account", "queryable": true},
+ map[string]any{"name": "Contact", "queryable": true},
+ map[string]any{"name": "Opportunity", "queryable": true},
+ map[string]any{"name": "Case", "queryable": true},
+ // Knowledge__kav absent: must be skipped silently.
+ },
+ }
+ data, _ := json.Marshal(payload)
+ return json.Unmarshal(data, out)
+ }
+ if err := connector.Validate(context.Background()); err != nil {
+ t.Fatalf("Validate failed: %v", err)
+ }
+}
+
+func TestSalesforceConnectorValidateMapsHTTPStatus(t *testing.T) {
+ cases := []struct {
+ name string
+ status int
+ want error
+ }{
+ {name: "unauthorized", status: http.StatusUnauthorized, want: &ConnectorMissingCredentialError{}},
+ {name: "forbidden", status: http.StatusForbidden, want: &ConnectorValidationError{}},
+ {name: "server error", status: http.StatusInternalServerError, want: &ConnectorValidationError{}},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ connector := newSalesforceFixtureConnector()
+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {
+ return &salesforceHTTPError{status: tc.status, body: "boom"}
+ }
+ err := connector.Validate(context.Background())
+ if tc.want == nil {
+ if err != nil {
+ t.Fatalf("Validate err = %v, want nil", err)
+ }
+ return
+ }
+ if err == nil {
+ t.Fatalf("Validate err = nil, want %T", tc.want)
+ }
+ switch tc.want.(type) {
+ case *ConnectorMissingCredentialError:
+ var want *ConnectorMissingCredentialError
+ if !errors.As(err, &want) {
+ t.Fatalf("Validate err = %v, want ConnectorMissingCredentialError", err)
+ }
+ case *ConnectorValidationError:
+ var want *ConnectorValidationError
+ if !errors.As(err, &want) {
+ t.Fatalf("Validate err = %v, want ConnectorValidationError", err)
+ }
+ }
+ })
+ }
+}
+
+func TestSalesforceConnectorOpenSync(t *testing.T) {
+ connector := newSalesforceFixtureConnector()
+ connector.doJSON = salesforceFixtureDoJSON(t)
+
+ start := mustTime(t, "2026-01-02T00:00:00Z")
+ end := mustTime(t, "2026-01-04T00:00:00Z")
+ session, err := connector.OpenSync(context.Background(), SyncRequest{WindowStart: &start, WindowEnd: end})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 2 {
+ t.Fatalf("documents len = %d, want 2", len(batch.Documents))
+ }
+ doc := batch.Documents[0]
+ if doc.SourceID != "Account/0015g00000Example1" {
+ t.Fatalf("source id = %q", doc.SourceID)
+ }
+ if doc.SemanticIdentifier != "Acme Corp" {
+ t.Fatalf("semantic identifier = %q", doc.SemanticIdentifier)
+ }
+ if doc.Extension != ".txt" {
+ t.Fatalf("extension = %q", doc.Extension)
+ }
+ if !doc.UpdatedAt.Equal(mustTime(t, "2026-01-03T00:00:00Z")) {
+ t.Fatalf("updated at = %s", doc.UpdatedAt)
+ }
+ if doc.Metadata["object"] != "Account" || doc.Metadata["record_id"] != "0015g00000Example1" {
+ t.Fatalf("metadata = %+v", doc.Metadata)
+ }
+ if doc.Metadata["web_url"] != "https://acme.my.salesforce.com/0015g00000Example1" {
+ t.Fatalf("web_url = %v", doc.Metadata["web_url"])
+ }
+ if doc.Fingerprint == "" {
+ t.Fatalf("fingerprint is empty")
+ }
+ blob := string(doc.Blob)
+ if !strings.Contains(blob, "Salesforce Account") || !strings.Contains(blob, "Name: Acme Corp") {
+ t.Fatalf("blob = %q", blob)
+ }
+ if batch.Checkpoint == nil || batch.Checkpoint.SourceID != "Account/0015g00000Example2" {
+ t.Fatalf("checkpoint = %+v", batch.Checkpoint)
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v", err)
+ }
+}
+
+func TestSalesforceConnectorOpenSyncWindowAndPagination(t *testing.T) {
+ connector := newSalesforceFixtureConnector()
+ var soql string
+ var fieldsQueried string
+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {
+ parsed, err := url.Parse(apiURL)
+ if err != nil {
+ t.Fatalf("parse url: %v", err)
+ }
+ if strings.Contains(apiURL, "/sobjects/Account/describe") {
+ // Compound address/location fields must be filtered from SOQL.
+ fieldsQueried = ""
+ }
+ if strings.Contains(apiURL, "/query?") {
+ soql, _ = url.QueryUnescape(parsed.Query().Get("q"))
+ }
+ return salesforceFixtureDoJSON(t)(ctx, apiURL, out)
+ }
+
+ start := mustTime(t, "2026-01-02T00:00:00Z")
+ end := mustTime(t, "2026-01-04T00:00:00Z")
+ session, err := connector.OpenSync(context.Background(), SyncRequest{WindowStart: &start, WindowEnd: end})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ for {
+ _, err := session.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ }
+ if !strings.Contains(soql, "SystemModstamp > 2026-01-02T00:00:00Z") {
+ t.Fatalf("soql missing since bound: %q", soql)
+ }
+ if !strings.Contains(soql, "SystemModstamp <= 2026-01-04T00:00:00Z") {
+ t.Fatalf("soql missing until bound: %q", soql)
+ }
+ if !strings.Contains(soql, " ORDER BY SystemModstamp ASC") {
+ t.Fatalf("soql missing ordering: %q", soql)
+ }
+ if strings.Contains(soql, "BillingAddress") || strings.Contains(soql, "Location__c") {
+ t.Fatalf("soql must exclude compound fields: %q", soql)
+ }
+ _ = fieldsQueried
+}
+
+func TestSalesforceConnectorOpenSyncResume(t *testing.T) {
+ connector := newSalesforceFixtureConnector()
+ connector.doJSON = salesforceFixtureDoJSON(t)
+
+ // Drain the whole object so the final record's checkpoint advances the
+ // per-object cursor; the syncer persists that checkpoint per batch.
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ first, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("first NextBatch failed: %v", err)
+ }
+ if len(first.Documents) != 2 {
+ t.Fatalf("first documents len = %d, want 2", len(first.Documents))
+ }
+ if first.Checkpoint == nil {
+ t.Fatalf("first checkpoint is nil")
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("first session EOF = %v", err)
+ }
+
+ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: first.Checkpoint})
+ if err != nil {
+ t.Fatalf("resume OpenSync failed: %v", err)
+ }
+ if _, err = resumed.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("resume NextBatch = %v, want EOF (object already ingested)", err)
+ }
+}
+
+func TestSalesforceConnectorOpenSyncPaginatedResume(t *testing.T) {
+ connector := newSalesforceFixtureConnector()
+ connector.batchSize = 2
+ records := []map[string]any{
+ {"Id": "0015g00000Example1", "Name": "Acme Corp", "SystemModstamp": "2026-01-03T00:00:00.000+0000"},
+ {"Id": "0015g00000Example2", "Name": "Globex", "SystemModstamp": "2026-01-03T01:00:00.000+0000"},
+ {"Id": "0015g00000Example3", "Name": "Initech", "SystemModstamp": "2026-01-03T02:00:00.000+0000"},
+ }
+ var queryCalls int
+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {
+ if strings.Contains(apiURL, "/sobjects/Account/describe") {
+ return json.Unmarshal([]byte(`{"fields":[
+ {"name":"Id","type":"id"},
+ {"name":"Name","type":"string"},
+ {"name":"SystemModstamp","type":"datetime"}
+ ]}`), out)
+ }
+ if strings.Contains(apiURL, "/query?") || strings.Contains(apiURL, "/query/01gExampleNext") {
+ queryCalls++
+ var page salesforceQueryPage
+ if queryCalls == 1 {
+ page = salesforceQueryPage{
+ Records: records[:2],
+ NextRecordsURL: "/services/data/v59.0/query/01gExampleNext",
+ }
+ } else {
+ // Second page: fixture returns the last record with done=true.
+ // The per-object cursor must only advance on this final record.
+ page = salesforceQueryPage{Records: records[2:]}
+ }
+ data, _ := json.Marshal(page)
+ return json.Unmarshal(data, out)
+ }
+ t.Fatalf("unexpected url %s", apiURL)
+ return nil
+ }
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ first, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("first NextBatch failed: %v", err)
+ }
+ if len(first.Documents) != 2 || first.Documents[0].SourceID != "Account/0015g00000Example1" {
+ t.Fatalf("first documents = %+v", first.Documents)
+ }
+ // The first batch ends before the object drains, so its checkpoint must
+ // NOT advance the Account cursor yet (no committed cursor to resume from).
+ if first.Checkpoint == nil {
+ t.Fatalf("first checkpoint is nil")
+ }
+ var firstCursor salesforceSyncCursor
+ if err := json.Unmarshal([]byte(first.Checkpoint.Cursor), &firstCursor); err != nil {
+ t.Fatalf("parse first checkpoint cursor: %v", err)
+ }
+ if _, ok := firstCursor.Cursors["Account"]; ok {
+ t.Fatalf("first batch checkpoint cursor advanced Account too early: %+v", firstCursor.Cursors)
+ }
+
+ second, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("second NextBatch failed: %v", err)
+ }
+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "Account/0015g00000Example3" {
+ t.Fatalf("second documents = %+v", second.Documents)
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("session EOF = %v", err)
+ }
+ // The final batch's checkpoint carries the advanced Account cursor.
+ var finalCursor salesforceSyncCursor
+ if err := json.Unmarshal([]byte(second.Checkpoint.Cursor), &finalCursor); err != nil {
+ t.Fatalf("parse final checkpoint cursor: %v", err)
+ }
+ finalAccount := finalCursor.Cursors["Account"]
+ if finalAccount.SystemModstamp != "2026-01-03T02:00:00.000+0000" || finalAccount.Id != "0015g00000Example3" {
+ t.Fatalf("final checkpoint cursor = %+v, want Account advanced with composite cursor", finalCursor.Cursors)
+ }
+
+ // Resume from the second batch: the object is fully ingested, so the next
+ // run must not re-emit its records. The fixture returns all three records
+ // on its first query page for the resumed run; the since predicate derived
+ // from the advanced cursor must filter them out.
+ connector2 := newSalesforceFixtureConnector()
+ connector2.batchSize = 2
+ var resumedQueryCalls int
+ connector2.doJSON = func(ctx context.Context, apiURL string, out any) error {
+ if strings.Contains(apiURL, "/sobjects/Account/describe") {
+ return json.Unmarshal([]byte(`{"fields":[
+ {"name":"Id","type":"id"},
+ {"name":"Name","type":"string"},
+ {"name":"SystemModstamp","type":"datetime"}
+ ]}`), out)
+ }
+ if strings.Contains(apiURL, "/query?") {
+ resumedQueryCalls++
+ parsed, _ := url.Parse(apiURL)
+ soql, _ := url.QueryUnescape(parsed.Query().Get("q"))
+ if !strings.Contains(soql, "SystemModstamp > 2026-01-03T02:00:00Z") {
+ t.Fatalf("resumed soql missing since predicate: %q", soql)
+ }
+ var page salesforceQueryPage
+ if strings.Contains(soql, "SystemModstamp > 2026-01-03T02:00:00Z") {
+ // No records strictly newer than the cursor.
+ page = salesforceQueryPage{Records: []map[string]any{}}
+ } else {
+ page = salesforceQueryPage{Records: records}
+ }
+ data, _ := json.Marshal(page)
+ return json.Unmarshal(data, out)
+ }
+ t.Fatalf("unexpected url %s", apiURL)
+ return nil
+ }
+ resumed, err := connector2.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: second.Checkpoint})
+ if err != nil {
+ t.Fatalf("resumed OpenSync failed: %v", err)
+ }
+ if _, err = resumed.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("resumed NextBatch = %v, want EOF", err)
+ }
+}
+
+func TestSalesforceConnectorOpenPrune(t *testing.T) {
+ connector := newSalesforceFixtureConnector()
+ connector.doJSON = salesforceFixtureDoJSON(t)
+
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})
+ if err != nil {
+ t.Fatalf("OpenPrune failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ got := []string{}
+ for _, doc := range batch.Documents {
+ got = append(got, doc.SourceID)
+ }
+ want := []string{"Account/0015g00000Example1", "Account/0015g00000Example2"}
+ if len(got) != len(want) {
+ t.Fatalf("prune documents = %v, want %v", got, want)
+ }
+ for i := range want {
+ if got[i] != want[i] {
+ t.Fatalf("prune documents = %v, want %v", got, want)
+ }
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("prune NextBatch EOF = %v", err)
+ }
+}
+
+func TestSalesforceConnectorOpenSyncSkipsMissingObject(t *testing.T) {
+ connector := newSalesforceFixtureConnector()
+ connector.objects = []string{"Account", "Case"}
+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {
+ if strings.Contains(apiURL, "/sobjects/Case/describe") {
+ return &salesforceObjectUnavailableError{message: "Case unavailable"}
+ }
+ return salesforceFixtureDoJSON(t)(ctx, apiURL, out)
+ }
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 2 {
+ t.Fatalf("documents len = %d, want 2 (Case skipped)", len(batch.Documents))
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v", err)
+ }
+}
+
+func TestSalesforceConnectorOpenPruneSkipsMissingObject(t *testing.T) {
+ connector := newSalesforceFixtureConnector()
+ connector.objects = []string{"Account", "Case"}
+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {
+ if strings.Contains(apiURL, "/query?") && strings.Contains(apiURL, "FROM+Case") {
+ return &salesforceObjectUnavailableError{message: "Case unavailable"}
+ }
+ return salesforceFixtureDoJSON(t)(ctx, apiURL, out)
+ }
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})
+ if err != nil {
+ t.Fatalf("OpenPrune failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ got := []string{}
+ for _, doc := range batch.Documents {
+ got = append(got, doc.SourceID)
+ }
+ if len(got) != 2 || got[0] != "Account/0015g00000Example1" {
+ t.Fatalf("prune documents = %v, want Account records only", got)
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("prune NextBatch EOF = %v", err)
+ }
+}
+
+func TestSalesforceRecordToTextDeterministic(t *testing.T) {
+ record := map[string]any{
+ "Name": "Acme Corp",
+ "Industry": "Software",
+ "attributes": map[string]any{"type": "Account"},
+ "AnnualRevenue": 1000.5,
+ "Description": "",
+ }
+ text1 := salesforceRecordToText("Account", record)
+ text2 := salesforceRecordToText("Account", record)
+ if text1 != text2 {
+ t.Fatalf("record text unstable: %q vs %q", text1, text2)
+ }
+ if !strings.HasPrefix(text1, "Salesforce Account\n") {
+ t.Fatalf("record text = %q", text1)
+ }
+ if strings.Contains(text1, "attributes") {
+ t.Fatalf("record text should skip attributes: %q", text1)
+ }
+ if strings.Contains(text1, "Description:") {
+ t.Fatalf("record text should skip empty values: %q", text1)
+ }
+}
+
+func TestParseSalesforceTime(t *testing.T) {
+ cases := []string{
+ "2026-01-03T00:00:00.000+0000",
+ "2026-01-03T00:00:00+0000",
+ "2026-01-03T00:00:00.000Z",
+ "2026-01-03T00:00:00Z",
+ }
+ for _, value := range cases {
+ parsed, err := parseSalesforceTime(value)
+ if err != nil {
+ t.Fatalf("parse %q: %v", value, err)
+ }
+ if parsed.UTC() != mustTime(t, "2026-01-03T00:00:00Z") {
+ t.Fatalf("parse %q = %s", value, parsed)
+ }
+ }
+}
+
+func TestSalesforceConnectorValidateConnectorSetting(t *testing.T) {
+ connector := newSalesforceFixtureConnector()
+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {
+ payload := map[string]any{
+ "sobjects": []any{
+ map[string]any{"name": "Account", "queryable": true},
+ map[string]any{"name": "Contact", "queryable": true},
+ map[string]any{"name": "Opportunity", "queryable": true},
+ map[string]any{"name": "Case", "queryable": true},
+ map[string]any{"name": "Knowledge__kav", "queryable": true},
+ },
+ }
+ data, _ := json.Marshal(payload)
+ return json.Unmarshal(data, out)
+ }
+ // The receiver is a valid fixture; the unsaved request is what must be
+ // validated. Use a request whose credential set is incomplete so the
+ // temporary connector fails fast regardless of the receiver's state.
+ request := map[string]any{
+ "credentials": map[string]any{
+ "instance_url": "https://acme.my.salesforce.com",
+ "client_id": "client",
+ // client_secret intentionally omitted.
+ },
+ }
+ var credErr *ConnectorMissingCredentialError
+ if err := connector.ValidateConnectorSetting(context.Background(), request); !errors.As(err, &credErr) {
+ t.Fatalf("ValidateConnectorSetting err = %v, want ConnectorMissingCredentialError for the request", err)
+ }
+}
+
+func TestSalesforceConnectorValidateConnectorSettingUsesRequest(t *testing.T) {
+ connector := newSalesforceFixtureConnector()
+ // Receiver itself would pass with only Account; the request lists Bogus, so
+ // a successful validation must be derived from the request, not the receiver.
+ connector.objects = []string{"Account"}
+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {
+ payload := map[string]any{
+ "sobjects": []any{
+ map[string]any{"name": "Account", "queryable": true},
+ },
+ }
+ data, _ := json.Marshal(payload)
+ return json.Unmarshal(data, out)
+ }
+ request := map[string]any{
+ "objects": "Account, Bogus",
+ "credentials": map[string]any{
+ "instance_url": "https://acme.my.salesforce.com",
+ "client_id": "client",
+ "client_secret": "secret",
+ },
+ }
+ var valErr *ConnectorValidationError
+ if err := connector.ValidateConnectorSetting(context.Background(), request); !errors.As(err, &valErr) {
+ t.Fatalf("ValidateConnectorSetting err = %v, want ConnectorValidationError for the unsaved request", err)
+ }
+}
+
+func TestRegisterBuiltInsOpensSalesforce(t *testing.T) {
+ registry := NewRegistry()
+ RegisterBuiltIns(registry)
+ connector, err := registry.OpenFromConfig("salesforce", map[string]any{
+ "credentials": map[string]any{
+ "instance_url": "https://acme.my.salesforce.com",
+ "client_id": "client",
+ "client_secret": "secret",
+ },
+ })
+ if err != nil {
+ t.Fatalf("OpenFromConfig failed: %v", err)
+ }
+ if _, ok := connector.(*SalesforceConnector); !ok {
+ t.Fatalf("connector type = %T, want *SalesforceConnector", connector)
+ }
+}
+
+func TestSalesforceConnectorOpenSyncSameTimestampResume(t *testing.T) {
+ ts := "2026-01-03T00:00:00.000+0000"
+ records := []map[string]any{
+ {"Id": "0015g00000Example1", "Name": "Acme Corp", "SystemModstamp": ts},
+ {"Id": "0015g00000Example2", "Name": "Globex", "SystemModstamp": ts},
+ }
+
+ // First run with batchSize 1: each record becomes one batch and the object
+ // only drains on the last one, so the persisted cursor advances to the
+ // composite position (ts, Example2) of the final same-instant record.
+ connector := newSalesforceFixtureConnector()
+ connector.batchSize = 1
+ connector.doJSON = salesforceFixtureRecordsDoJSON(t, records)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ var checkpoint *SyncCheckpoint
+ for {
+ batch, err := session.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if batch.Checkpoint != nil {
+ checkpoint = batch.Checkpoint
+ }
+ }
+ var curs salesforceSyncCursor
+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &curs); err != nil {
+ t.Fatalf("parse cursor: %v", err)
+ }
+ acct := curs.Cursors["Account"]
+ if acct.SystemModstamp != ts || acct.Id != "0015g00000Example2" {
+ t.Fatalf("cursor = %+v, want (ts, Example2)", acct)
+ }
+
+ // A third record created in the same instant but sorting after the
+ // checkpoint by Id must be delivered on resume. The resumed fixture still
+ // carries all records; the composite WHERE clause must select only the
+ // same-instant record whose Id sorts after the cursor.
+ resumedRecords := append(append([]map[string]any{}, records...), map[string]any{
+ "Id": "0015g00000Example3", "Name": "Initech", "SystemModstamp": ts,
+ })
+ resumed := newSalesforceFixtureConnector()
+ resumed.batchSize = 1
+ var resumedSOQL string
+ resumed.doJSON = func(ctx context.Context, apiURL string, out any) error {
+ if strings.Contains(apiURL, "/query?") {
+ parsed, err := url.Parse(apiURL)
+ if err != nil {
+ t.Fatalf("parse query url: %v", err)
+ }
+ resumedSOQL, _ = url.QueryUnescape(parsed.Query().Get("q"))
+ }
+ return salesforceFixtureRecordsDoJSON(t, resumedRecords)(ctx, apiURL, out)
+ }
+ resumedSession, err := resumed.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: checkpoint})
+ if err != nil {
+ t.Fatalf("resumed OpenSync failed: %v", err)
+ }
+ var gotIDs []string
+ for {
+ batch, err := resumedSession.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ t.Fatalf("resumed NextBatch failed: %v", err)
+ }
+ for _, doc := range batch.Documents {
+ gotIDs = append(gotIDs, doc.SourceID)
+ }
+ }
+ if !strings.Contains(resumedSOQL, "AND Id > '0015g00000Example2'") {
+ t.Fatalf("resumed SOQL missing same-instant Id boundary: %q", resumedSOQL)
+ }
+ if len(gotIDs) != 1 || gotIDs[0] != "Account/0015g00000Example3" {
+ t.Fatalf("resumed documents = %v, want only the same-instant record after the cursor", gotIDs)
+ }
+}
+
+func TestSalesforceObjectUnavailable404(t *testing.T) {
+ cases := []struct {
+ name string
+ body string
+ want bool
+ }{
+ {
+ name: "structured NOT_FOUND",
+ body: `[{"message":"sObject type 'Bogus' is not supported. If you intend to use a custom object, make sure it is enabled in \"Setup > Object Manager\" and \"API\" is enabled in the Object's Detail Page.","errorCode":"NOT_FOUND"}]`,
+ want: true,
+ },
+ {
+ name: "object level permissions NOT_FOUND",
+ body: `[{"message":"Object type 'Case' is not supported. If you intend to use a custom object...","errorCode":"NOT_FOUND"}]`,
+ want: true,
+ },
+ {
+ name: "generic not found route",
+ body: `404 page not found`,
+ want: false,
+ },
+ {
+ name: "empty body",
+ body: ``,
+ want: false,
+ },
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := salesforceObjectUnavailable(http.StatusNotFound, []byte(tc.body)); got != tc.want {
+ t.Fatalf("salesforceObjectUnavailable(404, %q) = %v, want %v", tc.body, got, tc.want)
+ }
+ })
+ }
+}
+
+func TestSalesforceObjectUnavailable400InvalidType(t *testing.T) {
+ body := `[{"message":"\nSELECT Account__c FROM Bogus\n ^\nERROR at Row:1:Column:18\nsObject type 'Bogus' is not supported.","errorCode":"INVALID_TYPE"}]`
+ if !salesforceObjectUnavailable(http.StatusBadRequest, []byte(body)) {
+ t.Fatalf("expected 400 INVALID_TYPE to be unavailable")
+ }
+ if salesforceObjectUnavailable(http.StatusBadRequest, []byte(`bad request`)) {
+ t.Fatalf("did not expect generic 400 to be unavailable")
+ }
+ if salesforceObjectUnavailable(http.StatusForbidden, []byte(`forbidden`)) {
+ t.Fatalf("did not expect 403 to be unavailable")
+ }
+}
+
+func TestSalesforceHostAllowed(t *testing.T) {
+ allowed := []string{
+ "acme.my.salesforce.com",
+ "login.salesforce.com",
+ "ACME.MY.SALESFORCE.COM",
+ "instance.salesforce.com",
+ "custom.force.com",
+ "org.lightning.force.com",
+ "salesforce.com",
+ "force.com",
+ }
+ for _, host := range allowed {
+ if !salesforceHostAllowed(host) {
+ t.Fatalf("salesforceHostAllowed(%q) = false, want true", host)
+ }
+ }
+ blocked := []string{
+ "",
+ "acme.my.salesforce.com.evil.com",
+ "evil.example.com",
+ "salesforce.com.evil.com",
+ "force.com.attacker.io",
+ "127.0.0.1",
+ }
+ for _, host := range blocked {
+ if salesforceHostAllowed(host) {
+ t.Fatalf("salesforceHostAllowed(%q) = true, want false", host)
+ }
+ }
+}
+
+func TestRequestAccessTokenRejectsNonSalesforceHost(t *testing.T) {
+ origLookup := utility.LookupHost
+ utility.LookupHost = func(host string) ([]string, error) {
+ return []string{"93.184.216.34"}, nil
+ }
+ t.Cleanup(func() { utility.LookupHost = origLookup })
+
+ connector := &SalesforceConnector{
+ instanceURL: "https://evil.example.com",
+ clientID: "client",
+ clientSecret: "secret",
+ now: time.Now,
+ }
+ var credErr *ConnectorMissingCredentialError
+ if _, err := connector.requestAccessToken(context.Background()); !errors.As(err, &credErr) {
+ t.Fatalf("requestAccessToken err = %v, want ConnectorMissingCredentialError for non-Salesforce host", err)
+ }
+}
+
+func TestRequestAccessTokenRequiresHTTPS(t *testing.T) {
+ origLookup := utility.LookupHost
+ utility.LookupHost = func(host string) ([]string, error) {
+ return []string{"93.184.216.34"}, nil
+ }
+ t.Cleanup(func() { utility.LookupHost = origLookup })
+
+ connector := &SalesforceConnector{
+ instanceURL: "http://acme.my.salesforce.com",
+ clientID: "client",
+ clientSecret: "secret",
+ now: time.Now,
+ }
+ var credErr *ConnectorMissingCredentialError
+ if _, err := connector.requestAccessToken(context.Background()); !errors.As(err, &credErr) {
+ t.Fatalf("requestAccessToken err = %v, want ConnectorMissingCredentialError for non-HTTPS token endpoint", err)
+ }
+}
+
+func TestSalesforceConnectorAPIURLAbsoluteValidation(t *testing.T) {
+ connector := newSalesforceFixtureConnector()
+ snap := salesforceToken{InstanceURL: "https://acme.my.salesforce.com"}
+
+ // A well-formed absolute pagination URL on an approved HTTPS host is kept.
+ valid := "https://instance2.my.salesforce.com/services/data/v59.0/query/01gX"
+ got, err := connector.apiURL(snap, valid)
+ if err != nil {
+ t.Fatalf("apiURL(valid) err = %v", err)
+ }
+ if got != valid {
+ t.Fatalf("apiURL(valid) = %q", got)
+ }
+
+ // Relative Salesforce API paths are still constructed from the snapshot.
+ rel, err := connector.apiURL(snap, "/sobjects/Account/describe")
+ if err != nil {
+ t.Fatalf("apiURL(relative) err = %v", err)
+ }
+ wantRel := "https://acme.my.salesforce.com/services/data/v59.0/sobjects/Account/describe"
+ if rel != wantRel {
+ t.Fatalf("apiURL(relative) = %q, want %q", rel, wantRel)
+ }
+
+ // Absolute pagination URLs with a bad scheme or host must be rejected.
+ for _, bad := range []string{
+ "http://acme.my.salesforce.com/services/data/v59.0/query/01gX",
+ "https://evil.example.com/services/data/v59.0/query/01gX",
+ "https://acme.my.salesforce.com.evil.com/services/data/v59.0/query/01gX",
+ } {
+ if _, err := connector.apiURL(snap, bad); err == nil {
+ t.Fatalf("apiURL(%q) succeeded, want error", bad)
+ }
+ }
+}
+
+func TestSalesforceConnectorGetJSONRejectsUnsafePaginationURL(t *testing.T) {
+ connector := newSalesforceFixtureConnector()
+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {
+ t.Fatalf("doJSON must not run for an unsafe pagination URL")
+ return nil
+ }
+ var page salesforceQueryPage
+ if err := connector.getJSON(context.Background(), "http://evil.example.com/services/data/v59.0/query/01gX", &page); err == nil {
+ t.Fatalf("getJSON succeeded for an unsafe pagination URL, want error")
+ }
+}
+
+func TestSalesforceConnectorDoGetRejectsUnsafeURL(t *testing.T) {
+ connector := newSalesforceFixtureConnector()
+ for _, bad := range []string{
+ "http://acme.my.salesforce.com/services/data/v59.0/sobjects",
+ "https://evil.example.com/services/data/v59.0/sobjects",
+ } {
+ if _, _, err := connector.doGet(context.Background(), bad, "token"); err == nil {
+ t.Fatalf("doGet(%q) succeeded, want error", bad)
+ }
+ }
+}
+
+// newSalesforceFixtureConnector builds a connector with token acquisition
+// short-circuited so unit tests never touch the network.
+func newSalesforceFixtureConnector() *SalesforceConnector {
+ connector := &SalesforceConnector{
+ instanceURL: "https://acme.my.salesforce.com",
+ clientID: "client",
+ clientSecret: "secret",
+ objects: []string{"Account"},
+ apiVersion: defaultSalesforceAPIVersion,
+ batchSize: defaultSalesforceBatchSize,
+ now: time.Now,
+ }
+ connector.acquireAccessToken = func(ctx context.Context) (salesforceToken, error) {
+ return salesforceToken{
+ AccessToken: "token",
+ InstanceURL: "https://acme.my.salesforce.com",
+ ExpiresAt: time.Now().Add(time.Hour),
+ }, nil
+ }
+ return connector
+}
+
+// salesforceFixtureDoJSON serves describe + query responses for unit tests.
+// The query endpoint emulates server-side SOQL filtering on SystemModstamp so
+// resume tests behave like a real org.
+func salesforceFixtureDoJSON(t *testing.T) func(ctx context.Context, apiURL string, out any) error {
+ t.Helper()
+ records := []map[string]any{
+ {
+ "Id": "0015g00000Example1",
+ "Name": "Acme Corp",
+ "Industry": "Software",
+ "SystemModstamp": "2026-01-03T00:00:00.000+0000",
+ },
+ {
+ "Id": "0015g00000Example2",
+ "Name": "Globex",
+ "Industry": "Hardware",
+ "SystemModstamp": "2026-01-03T01:00:00.000+0000",
+ },
+ }
+ return func(ctx context.Context, apiURL string, out any) error {
+ var body string
+ switch {
+ case strings.Contains(apiURL, "/sobjects/Account/describe"):
+ body = `{"fields":[
+ {"name":"Id","type":"id"},
+ {"name":"Name","type":"string"},
+ {"name":"Industry","type":"string"},
+ {"name":"AnnualRevenue","type":"currency"},
+ {"name":"BillingAddress","type":"address"},
+ {"name":"Location__c","type":"location"}
+ ]}`
+ case strings.Contains(apiURL, "/query?"):
+ parsed, err := url.Parse(apiURL)
+ if err != nil {
+ t.Fatalf("parse query url: %v", err)
+ }
+ soql, err := url.QueryUnescape(parsed.Query().Get("q"))
+ if err != nil {
+ t.Fatalf("unescape soql: %v", err)
+ }
+ filtered := []map[string]any{}
+ for _, record := range records {
+ if salesforceFixtureMatchesSOQL(t, soql, record) {
+ filtered = append(filtered, record)
+ }
+ }
+ payload := map[string]any{"totalSize": len(filtered), "done": true, "records": filtered}
+ data, _ := json.Marshal(payload)
+ return json.Unmarshal(data, out)
+ default:
+ t.Fatalf("unexpected api url %s", apiURL)
+ }
+ return json.Unmarshal([]byte(body), out)
+ }
+}
+
+// salesforceFixtureRecordsDoJSON serves describe + query responses for an
+// arbitrary record set, emulating server-side SOQL filtering on SystemModstamp
+// (including the composite resume predicate).
+func salesforceFixtureRecordsDoJSON(t *testing.T, records []map[string]any) func(ctx context.Context, apiURL string, out any) error {
+ t.Helper()
+ return func(ctx context.Context, apiURL string, out any) error {
+ switch {
+ case strings.Contains(apiURL, "/sobjects/Account/describe"):
+ body := `{"fields":[
+ {"name":"Id","type":"id"},
+ {"name":"Name","type":"string"},
+ {"name":"SystemModstamp","type":"datetime"}
+ ]}`
+ return json.Unmarshal([]byte(body), out)
+ case strings.Contains(apiURL, "/query?"):
+ parsed, err := url.Parse(apiURL)
+ if err != nil {
+ t.Fatalf("parse query url: %v", err)
+ }
+ soql, err := url.QueryUnescape(parsed.Query().Get("q"))
+ if err != nil {
+ t.Fatalf("unescape soql: %v", err)
+ }
+ filtered := []map[string]any{}
+ for _, record := range records {
+ if salesforceFixtureMatchesSOQL(t, soql, record) {
+ filtered = append(filtered, record)
+ }
+ }
+ payload := map[string]any{"totalSize": len(filtered), "done": true, "records": filtered}
+ data, _ := json.Marshal(payload)
+ return json.Unmarshal(data, out)
+ default:
+ t.Fatalf("unexpected api url %s", apiURL)
+ }
+ return nil
+ }
+}
+
+// salesforceFixtureMatchesSOQL applies the fixture's SystemModstamp predicates,
+// including the composite resume clause `(SystemModstamp > X OR (SystemModstamp
+// = X AND Id > 'Y'))` so same-instant records that sort after the checkpoint by
+// Id are kept.
+func salesforceFixtureMatchesSOQL(t *testing.T, soql string, record map[string]any) bool {
+ t.Helper()
+ lower := strings.ToLower(soql)
+ since := time.Time{}
+ until := time.Time{}
+ equalSince := false
+ minID := ""
+ if idx := strings.Index(lower, "systemmodstamp >"); idx >= 0 {
+ rest := soql[idx+len("SystemModstamp > "):]
+ value := strings.TrimSpace(strings.Split(rest, " ")[0])
+ parsed, err := parseSalesforceTime(strings.Trim(value, "'"))
+ if err != nil {
+ t.Fatalf("parse soql since %q: %v", value, err)
+ }
+ since = parsed
+ }
+ if idx := strings.Index(lower, "systemmodstamp ="); idx >= 0 {
+ rest := soql[idx+len("SystemModstamp = "):]
+ value := strings.TrimSpace(strings.Split(rest, " ")[0])
+ parsed, err := parseSalesforceTime(strings.Trim(value, "'"))
+ if err != nil {
+ t.Fatalf("parse soql equal timestamp %q: %v", value, err)
+ }
+ equalSince = true
+ since = parsed
+ }
+ if idx := strings.Index(lower, "and id >"); idx >= 0 {
+ value := soql[idx+len("AND Id > "):]
+ minID = strings.Trim(strings.TrimSpace(value), "'")
+ }
+ if idx := strings.Index(lower, "systemmodstamp <= "); idx >= 0 {
+ rest := soql[idx+len("SystemModstamp <= "):]
+ value := strings.TrimSpace(strings.Split(rest, " ")[0])
+ parsed, err := parseSalesforceTime(strings.Trim(value, "'"))
+ if err != nil {
+ t.Fatalf("parse soql until %q: %v", value, err)
+ }
+ until = parsed
+ }
+ modified, err := parseSalesforceTime(record["SystemModstamp"].(string))
+ if err != nil {
+ t.Fatalf("parse fixture timestamp: %v", err)
+ }
+ if !until.IsZero() && modified.After(until) {
+ return false
+ }
+ if !since.IsZero() {
+ if modified.After(since) {
+ return true
+ }
+ if modified.Before(since) {
+ return false
+ }
+ // Same instant as the checkpoint: keep records that sort after it by Id
+ // only when a same-instant Id boundary is present.
+ if equalSince && minID != "" {
+ id, _ := record["Id"].(string)
+ return id > minID
+ }
+ return false
+ }
+ return true
+}