RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(go-syncer): add Salesforce connector with checkpoint resume (#18674)
mkaaadb2541bff6881
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.goindex 393698a006..4f79a0fa41 100644--- a/internal/syncer/connector/builtin.go+++ b/internal/syncer/connector/builtin.go@@ -28,6 +28,7 @@ import (func RegisterBuiltIns(registry *Registry) {registerBuiltIn(registry, "confluence", NewConfluenceConnector)registerBuiltIn(registry, "rss", NewRSSConnector)+ registerBuiltIn(registry, "salesforce", NewSalesforceConnector)registerBuiltIn(registry, "bitbucket", NewBitbucketConnector)registerBuiltIn(registry, "dropbox", NewDropboxConnector)registerBuiltIn(registry, "github", NewGitHubConnector)diff --git a/internal/syncer/connector/salesforce.go b/internal/syncer/connector/salesforce.gonew file mode 100644index 0000000000..daa8348e37--- /dev/null+++ b/internal/syncer/connector/salesforce.go@@ -0,0 +1,1095 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "net/http"+ "net/url"+ "sort"+ "strconv"+ "strings"+ "sync"+ "time"++ "ragflow/internal/utility"+)++const (+ defaultSalesforceAPIVersion = "v59.0"+ defaultSalesforceBatchSize = 2+ salesforceRequestTimeout = 60 * time.Second+ salesforceTokenExpiryMargin = 5 * time.Minute+)++// Default Salesforce CRM objects indexed when the config does not list any.+// Knowledge__kav is optional: orgs without Salesforce Knowledge silently skip it.+var salesforceDefaultObjects = []string{"Account", "Contact", "Opportunity", "Case", "Knowledge__kav"}++// salesforceOptionalObjects are silently skipped when absent from the org.+var salesforceOptionalObjects = map[string]bool{"Knowledge__kav": true}++// salesforceObjectUnavailableError reports that an SObject is genuinely+// absent or not queryable (HTTP 404, or 400 INVALID_TYPE) rather than a+// transient/permission failure. Callers skip such objects without failing+// the whole run; 403/5xx/429 must still abort.+type salesforceObjectUnavailableError struct {+ message string+}++func (e *salesforceObjectUnavailableError) Error() string { return e.message }++// salesforceHTTPError carries a non-2xx Salesforce REST response.+type salesforceHTTPError struct {+ status int+ body string+}++func (e *salesforceHTTPError) Error() string {+ return fmt.Sprintf("Salesforce API returned HTTP %d: %s", e.status, e.body)+}++// salesforceToken is a cached OAuth2 client-credentials token.+type salesforceToken struct {+ AccessToken string+ InstanceURL string+ ExpiresAt time.Time+}++// SalesforceConnector reads Salesforce CRM records through the REST + SOQL+// APIs. It authenticates with OAuth2 client-credentials against a Connected+// App, describes the configured SObjects, and uses SystemModstamp as the+// incremental cursor so re-syncs only fetch what changed.+type SalesforceConnector struct {+ instanceURL string+ clientID string+ clientSecret string+ objects []string+ apiVersion string+ batchSize int++ clientMu sync.Mutex+ accessToken string+ tokenExpiry time.Time+ now func() time.Time++ acquireAccessToken func(ctx context.Context) (salesforceToken, error)+ doJSON func(ctx context.Context, apiURL string, out any) error+}++// NewSalesforceConnector creates a Salesforce connector from config.+func NewSalesforceConnector(config map[string]any) (*SalesforceConnector, error) {+ credentials, _ := config["credentials"].(map[string]any)+ instanceURL := strings.TrimRight(strings.TrimSpace(stringConfig(credentials["instance_url"])), "/")+ objects := salesforceObjects(config["objects"])+ return &SalesforceConnector{+ instanceURL: instanceURL,+ clientID: strings.TrimSpace(stringConfig(credentials["client_id"])),+ clientSecret: stringConfig(credentials["client_secret"]),+ objects: objects,+ apiVersion: firstNonEmpty(stringConfig(config["api_version"]), defaultSalesforceAPIVersion),+ batchSize: salesforceBatchSize(config["batch_size"]),+ now: time.Now,+ }, nil+}++// salesforceObjects normalizes a comma string or JSON list of object names.+func salesforceObjects(value any) []string {+ switch typed := value.(type) {+ case string:+ out := []string{}+ for _, part := range strings.Split(typed, ",") {+ if part = strings.TrimSpace(part); part != "" {+ out = append(out, part)+ }+ }+ if len(out) > 0 {+ return out+ }+ case []any:+ out := []string{}+ for _, item := range typed {+ if part := strings.TrimSpace(fmt.Sprint(item)); part != "" {+ out = append(out, part)+ }+ }+ if len(out) > 0 {+ return out+ }+ }+ return append([]string(nil), salesforceDefaultObjects...)+}++// salesforceBatchSize preserves explicit non-positive values so validation can+// reject them; only missing/unparseable values fall back to the default.+func salesforceBatchSize(value any) int {+ switch typed := value.(type) {+ case int:+ return typed+ case int64:+ return int(typed)+ case float64:+ return int(typed)+ case json.Number:+ if v, err := typed.Int64(); err == nil {+ return int(v)+ }+ case string:+ if v, err := strconv.Atoi(strings.TrimSpace(typed)); err == nil {+ return v+ }+ }+ return defaultSalesforceBatchSize+}++// Validate validates Salesforce connector settings and credentials.+func (c *SalesforceConnector) Validate(ctx context.Context) error {+ if c == nil {+ return fmt.Errorf("salesforce connector is nil")+ }+ if c.instanceURL == "" || c.clientID == "" || c.clientSecret == "" {+ return &ConnectorMissingCredentialError{Message: "Salesforce credentials are incomplete (instance_url, client_id, client_secret required)"}+ }+ if c.batchSize <= 0 {+ return &ConnectorValidationError{Message: "batch_size must be a positive integer"}+ }+ if _, err := c.token(ctx); err != nil {+ return err+ }++ var payload salesforceSObjectsResponse+ if err := c.getJSON(ctx, "/sobjects", &payload); err != nil {+ var httpErr *salesforceHTTPError+ if errors.As(err, &httpErr) {+ switch httpErr.status {+ case http.StatusUnauthorized:+ return &ConnectorMissingCredentialError{Message: "Salesforce access token is invalid or expired."}+ case http.StatusForbidden:+ return &ConnectorValidationError{Message: "The Salesforce execution user lacks API access; enable the 'API Enabled' profile permission."}+ default:+ return &ConnectorValidationError{Message: fmt.Sprintf("Salesforce validation failed (HTTP %d): %s", httpErr.status, httpErr.body)}+ }+ }+ return err+ }++ queryable := map[string]bool{}+ for _, so := range payload.SObjects {+ if so.Name != "" {+ queryable[so.Name] = so.Queryable+ }+ }+ unknown := []string{}+ notQueryable := []string{}+ for _, obj := range c.objects {+ queryableFlag, ok := queryable[obj]+ if !ok {+ if salesforceOptionalObjects[obj] {+ continue+ }+ unknown = append(unknown, obj)+ } else if !queryableFlag {+ notQueryable = append(notQueryable, obj)+ }+ }+ if len(unknown) > 0 || len(notQueryable) > 0 {+ problems := []string{}+ if len(unknown) > 0 {+ sort.Strings(unknown)+ problems = append(problems, fmt.Sprintf("unknown object(s): %s", strings.Join(unknown, ", ")))+ }+ if len(notQueryable) > 0 {+ sort.Strings(notQueryable)+ problems = append(problems, fmt.Sprintf("non-queryable object(s): %s", strings.Join(notQueryable, ", ")))+ }+ return &ConnectorValidationError{Message: "Salesforce 'objects' configuration is invalid — " + strings.Join(problems, "; ") + ". Check for typos and that the execution user has read access to each object."}+ }+ return nil+}++// ValidateConnectorSetting validates Salesforce settings from an unsaved config.+func (c *SalesforceConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)+ defer cancel()+ tmp, err := NewSalesforceConnector(request)+ if err != nil {+ return err+ }+ // Carry the receiver's transport/acquire stubs so tests can validate an+ // unsaved request without touching the network; production leaves them unset.+ tmp.acquireAccessToken = c.acquireAccessToken+ tmp.doJSON = c.doJSON+ return tmp.Validate(ctx)+}++// OpenSync opens one Salesforce sync session.+func (c *SalesforceConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {+ session := &salesforceSyncSession{+ connector: c,+ objects: c.objects,+ batchSize: c.effectiveBatchSize(),+ windowStart: request.WindowStart,+ windowEnd: request.WindowEnd,+ cursors: map[string]salesforceObjectCursor{},+ }+ session.applyResume(request.Resume)+ return session, nil+}++// OpenPrune opens one complete Salesforce prune snapshot session.+func (c *SalesforceConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {+ return &salesforcePruneSession{+ connector: c,+ objects: c.objects,+ batchSize: c.effectiveBatchSize(),+ }, nil+}++func (c *SalesforceConnector) effectiveBatchSize() int {+ if c.batchSize > 0 {+ return c.batchSize+ }+ return defaultSalesforceBatchSize+}++// instanceBaseURL returns the current canonical instance URL under the auth+// lock. It is only safe to read through this accessor once requests share the+// auth lock with token acquisition.+func (c *SalesforceConnector) instanceBaseURL() string {+ c.clientMu.Lock()+ defer c.clientMu.Unlock()+ return c.instanceURL+}++// token returns a synchronized authentication snapshot: the access token, its+// expiry, and the canonical instance URL to build request targets from. The+// snapshot is captured atomically so callers never read or write instanceURL+// without the auth lock, and API URLs can be constructed from a single+// consistent view of the token exchange.+func (c *SalesforceConnector) token(ctx context.Context) (salesforceToken, error) {+ c.clientMu.Lock()+ if c.accessToken != "" && !c.cachedTokenExpiredLocked() {+ snap := salesforceToken{+ AccessToken: c.accessToken,+ InstanceURL: c.instanceURL,+ ExpiresAt: c.tokenExpiry,+ }+ c.clientMu.Unlock()+ return snap, nil+ }+ c.clientMu.Unlock()++ var cached salesforceToken+ var err error+ if c.acquireAccessToken != nil {+ cached, err = c.acquireAccessToken(ctx)+ } else {+ cached, err = c.requestAccessToken(ctx)+ }+ if err != nil {+ return salesforceToken{}, err+ }+ if cached.AccessToken == "" {+ return salesforceToken{}, &ConnectorMissingCredentialError{Message: "Salesforce token response did not contain access_token"}+ }+ c.clientMu.Lock()+ c.accessToken = cached.AccessToken+ c.tokenExpiry = cached.ExpiresAt+ if cached.InstanceURL != "" {+ c.instanceURL = strings.TrimRight(cached.InstanceURL, "/")+ }+ snap := salesforceToken{+ AccessToken: cached.AccessToken,+ InstanceURL: c.instanceURL,+ ExpiresAt: cached.ExpiresAt,+ }+ c.clientMu.Unlock()+ return snap, nil+}++func (c *SalesforceConnector) cachedTokenExpiredLocked() bool {+ return c.tokenExpiry.IsZero() || !c.currentTime().Add(salesforceTokenExpiryMargin).Before(c.tokenExpiry)+}++func (c *SalesforceConnector) invalidateToken(token string) {+ c.clientMu.Lock()+ defer c.clientMu.Unlock()+ if c.accessToken == token {+ c.accessToken = ""+ c.tokenExpiry = time.Time{}+ }+}++func (c *SalesforceConnector) currentTime() time.Time {+ if c.now != nil {+ return c.now()+ }+ return time.Now()+}++// salesforceHostAllowed reports whether a host is an approved Salesforce+// instance host. The configured and token-returned instance URLs must stay on+// Salesforce-owned domains so credentials are only ever transmitted to the+// intended provider.+func salesforceHostAllowed(host string) bool {+ host = strings.ToLower(strings.TrimSpace(host))+ if host == "" {+ return false+ }+ return host == "salesforce.com" ||+ host == "force.com" ||+ strings.HasSuffix(host, ".salesforce.com") ||+ strings.HasSuffix(host, ".my.salesforce.com") ||+ strings.HasSuffix(host, ".force.com") ||+ strings.HasSuffix(host, ".lightning.force.com")+}++// requestAccessToken performs the OAuth2 client-credentials exchange, validating+// the token endpoint for SSRF, HTTPS, and the approved Salesforce host policy+// before any credentials are transmitted.+func (c *SalesforceConnector) requestAccessToken(ctx context.Context) (salesforceToken, error) {+ tokenURL := c.instanceBaseURL() + "/services/oauth2/token"+ hostname, resolvedIP, err := utility.AssertURLSafe(tokenURL)+ if err != nil {+ return salesforceToken{}, &ConnectorMissingCredentialError{Message: fmt.Sprintf("Salesforce token request failed: %v", err)}+ }+ if !salesforceHostAllowed(hostname) {+ return salesforceToken{}, &ConnectorMissingCredentialError{Message: "Salesforce instance_url is not an approved Salesforce host"}+ }+ parsedURL, err := url.Parse(tokenURL)+ if err != nil || !strings.EqualFold(parsedURL.Scheme, "https") {+ return salesforceToken{}, &ConnectorMissingCredentialError{Message: "Salesforce OAuth token endpoint must use HTTPS"}+ }+ form := url.Values{+ "grant_type": {"client_credentials"},+ "client_id": {c.clientID},+ "client_secret": {c.clientSecret},+ }+ requestCtx, cancel := context.WithTimeout(ctx, salesforceRequestTimeout)+ defer cancel()+ req, err := http.NewRequestWithContext(requestCtx, http.MethodPost, tokenURL, strings.NewReader(form.Encode()))+ if err != nil {+ return salesforceToken{}, &ConnectorMissingCredentialError{Message: fmt.Sprintf("Salesforce token request failed: %v", err)}+ }+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded")+ client := utility.PinnedHTTPClient(hostname, resolvedIP, salesforceRequestTimeout)+ resp, err := client.Do(req)+ if err != nil {+ return salesforceToken{}, &ConnectorMissingCredentialError{Message: fmt.Sprintf("Salesforce token request failed: %v", err)}+ }+ defer resp.Body.Close()+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))+ if resp.StatusCode >= 400 {+ detail := salesforceTokenErrorDetail(body)+ return salesforceToken{}, &ConnectorMissingCredentialError{Message: fmt.Sprintf("Failed to acquire Salesforce access token (HTTP %d): %s", resp.StatusCode, detail)}+ }+ var payload struct {+ AccessToken string `json:"access_token"`+ InstanceURL string `json:"instance_url"`+ ExpiresIn int `json:"expires_in"`+ }+ if err := json.Unmarshal(body, &payload); err != nil {+ return salesforceToken{}, &ConnectorMissingCredentialError{Message: fmt.Sprintf("Salesforce token response is not JSON: %v", err)}+ }+ expiresAt := c.currentTime().Add(time.Hour)+ if payload.ExpiresIn > 0 {+ expiresAt = c.currentTime().Add(time.Duration(payload.ExpiresIn) * time.Second)+ }+ return salesforceToken{+ AccessToken: payload.AccessToken,+ InstanceURL: strings.TrimRight(payload.InstanceURL, "/"),+ ExpiresAt: expiresAt,+ }, nil+}++func salesforceTokenErrorDetail(body []byte) string {+ var payload struct {+ Error string `json:"error"`+ ErrorDescription string `json:"error_description"`+ }+ if err := json.Unmarshal(body, &payload); err == nil {+ if payload.ErrorDescription != "" {+ return payload.ErrorDescription+ }+ if payload.Error != "" {+ return payload.Error+ }+ }+ text := strings.TrimSpace(string(body))+ if len(text) > 200 {+ text = text[:200]+ }+ return text+}++// apiURL builds the full Salesforce REST URL for a service-relative path from+// the canonical instance URL captured in the authentication snapshot. The+// token exchange may publish a different canonical instance than the one the+// operator configured, so every request target is derived from the snapshot+// instead of stale connector state. Absolute pagination URLs returned by the+// query API are validated before use: they must use HTTPS on an approved+// Salesforce host, so a tampered URL can never receive the bearer token.+func (c *SalesforceConnector) apiURL(snap salesforceToken, path string) (string, error) {+ if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {+ parsed, err := url.Parse(path)+ if err != nil {+ return "", fmt.Errorf("Salesforce pagination URL is invalid: %v", err)+ }+ if !strings.EqualFold(parsed.Scheme, "https") || !salesforceHostAllowed(parsed.Hostname()) {+ return "", fmt.Errorf("Salesforce pagination URL must use HTTPS on an approved Salesforce host")+ }+ return path, nil+ }+ if strings.HasPrefix(path, "/services/data/") {+ return snap.InstanceURL + path, nil+ }+ return snap.InstanceURL + "/services/data/" + c.apiVersion + path, nil+}++// getJSON GETs a Salesforce REST endpoint and decodes JSON into out. The apiURL+// is built only after token acquisition, from the canonical instance URL in the+// returned snapshot, and a 401 retry rebuilds it from the refreshed snapshot.+func (c *SalesforceConnector) getJSON(ctx context.Context, path string, out any) error {+ snap, err := c.token(ctx)+ if err != nil {+ return err+ }+ apiURL, err := c.apiURL(snap, path)+ if err != nil {+ return err+ }+ if c.doJSON != nil {+ return c.doJSON(ctx, apiURL, out)+ }+ for attempt := 0; ; attempt++ {+ status, body, err := c.doGet(ctx, apiURL, snap.AccessToken)+ if err != nil {+ return err+ }+ if status == http.StatusUnauthorized && attempt == 0 {+ c.invalidateToken(snap.AccessToken)+ snap, err = c.token(ctx)+ if err != nil {+ return err+ }+ apiURL, err = c.apiURL(snap, path)+ if err != nil {+ return err+ }+ continue+ }+ if salesforceObjectUnavailable(status, body) {+ return &salesforceObjectUnavailableError{message: fmt.Sprintf("Salesforce object unavailable (HTTP %d): %s", status, strings.TrimSpace(string(body)))}+ }+ if status >= 400 {+ return &salesforceHTTPError{status: status, body: strings.TrimSpace(string(body))}+ }+ return json.Unmarshal(body, out)+ }+}++// doGet performs one authenticated GET with SSRF protection.+func (c *SalesforceConnector) doGet(ctx context.Context, apiURL, token string) (int, []byte, error) {+ parsed, err := url.Parse(apiURL)+ if err != nil || !strings.EqualFold(parsed.Scheme, "https") || !salesforceHostAllowed(parsed.Hostname()) {+ return 0, nil, fmt.Errorf("Salesforce request URL must use HTTPS on an approved Salesforce host")+ }+ hostname, resolvedIP, err := utility.AssertURLSafe(apiURL)+ if err != nil {+ return 0, nil, err+ }+ client := utility.PinnedHTTPClient(hostname, resolvedIP, salesforceRequestTimeout)+ requestCtx, cancel := context.WithTimeout(ctx, salesforceRequestTimeout)+ defer cancel()+ req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, apiURL, nil)+ if err != nil {+ return 0, nil, err+ }+ req.Header.Set("Accept", "application/json")+ req.Header.Set("Authorization", "Bearer "+token)+ resp, err := client.Do(req)+ if err != nil {+ return 0, nil, err+ }+ defer resp.Body.Close()+ body, err := io.ReadAll(io.LimitReader(resp.Body, 32*1024*1024))+ if err != nil {+ return 0, nil, err+ }+ return resp.StatusCode, body, nil+}++// salesforceObjectUnavailable reports whether a response indicates a genuinely+// absent SObject as opposed to a transient, permission, routing, or API-version+// failure. A 404 is only treated as object-not-found when the response carries+// the structured Salesforce NOT_FOUND error; a bare 404 (e.g. a bad pagination+// URL or unknown route) must propagate as a normal HTTP error.+func salesforceObjectUnavailable(status int, body []byte) bool {+ if status == http.StatusNotFound {+ return salesforceNotFoundError(body)+ }+ if status != http.StatusBadRequest {+ return false+ }+ return salesforceInvalidTypeError(body)+}++// salesforceNotFoundError reports whether a response body carries the+// structured Salesforce "object not found" error code.+func salesforceNotFoundError(body []byte) bool {+ var entries []struct {+ ErrorCode string `json:"errorCode"`+ }+ if err := json.Unmarshal(body, &entries); err == nil {+ for _, entry := range entries {+ if entry.ErrorCode == "NOT_FOUND" {+ return true+ }+ }+ return false+ }+ var single struct {+ ErrorCode string `json:"errorCode"`+ }+ if err := json.Unmarshal(body, &single); err == nil {+ return single.ErrorCode == "NOT_FOUND"+ }+ return false+}++// salesforceInvalidTypeError reports whether a 400 response body carries the+// Salesforce INVALID_TYPE error, which describes an unknowable SOQL object.+func salesforceInvalidTypeError(body []byte) bool {+ var entries []struct {+ ErrorCode string `json:"errorCode"`+ }+ if err := json.Unmarshal(body, &entries); err == nil {+ for _, entry := range entries {+ if entry.ErrorCode == "INVALID_TYPE" {+ return true+ }+ }+ return false+ }+ var single struct {+ ErrorCode string `json:"errorCode"`+ }+ if err := json.Unmarshal(body, &single); err == nil {+ return single.ErrorCode == "INVALID_TYPE"+ }+ return strings.Contains(string(body), "INVALID_TYPE")+}++// describeFields returns field API names for an SObject, filtering out+// compound types (address, location) that SOQL cannot project directly.+func (c *SalesforceConnector) describeFields(ctx context.Context, obj string) ([]string, error) {+ var payload struct {+ Fields []struct {+ Name string `json:"name"`+ Type string `json:"type"`+ } `json:"fields"`+ }+ if err := c.getJSON(ctx, "/sobjects/"+url.PathEscape(obj)+"/describe", &payload); err != nil {+ return nil, err+ }+ fields := []string{}+ for _, field := range payload.Fields {+ if field.Type == "address" || field.Type == "location" {+ continue+ }+ if field.Name != "" {+ fields = append(fields, field.Name)+ }+ }+ hasID := false+ for _, name := range fields {+ if name == "Id" {+ hasID = true+ break+ }+ }+ if !hasID {+ fields = append([]string{"Id"}, fields...)+ }+ return fields, nil+}++// queryURL builds the SOQL query URL path for one SObject page. When a resume+// cursor is present, the WHERE clause re-fetches records strictly newer than the+// checkpoint plus same-instant records whose Id sorts after the checkpoint, so+// later records sharing the checkpoint timestamp are not skipped.+func (c *SalesforceConnector) queryURL(obj string, fields []string, cursor *salesforceObjectCursor, until *time.Time) string {+ fieldList := strings.Join(fields, ",")+ filters := []string{}+ if cursor != nil && cursor.SystemModstamp != "" {+ since := cursor.SystemModstamp+ if parsed, err := parseSalesforceTime(cursor.SystemModstamp); err == nil {+ since = salesforceSOQLTime(parsed)+ }+ clause := "SystemModstamp > " + since+ if cursor.Id != "" {+ clause = "(" + clause + " OR (SystemModstamp = " + since + " AND Id > '" + cursor.Id + "'))"+ }+ filters = append(filters, clause)+ }+ if until != nil {+ filters = append(filters, "SystemModstamp <= "+salesforceSOQLTime(*until))+ }+ where := ""+ if len(filters) > 0 {+ where = " WHERE " + strings.Join(filters, " AND ")+ }+ soql := fmt.Sprintf("SELECT %s FROM %s%s ORDER BY SystemModstamp ASC, Id ASC", fieldList, obj, where)+ return "/query?q=" + url.QueryEscape(soql)+}++// salesforceSOQLTime formats a timestamp for a SOQL literal.+func salesforceSOQLTime(t time.Time) string {+ return t.UTC().Format("2006-01-02T15:04:05Z")+}++// salesforceWindowEnd returns the inclusive upper window bound, or nil when+// the window end is unset (e.g. prune scans).+func salesforceWindowEnd(windowEnd time.Time) *time.Time {+ if windowEnd.IsZero() {+ return nil+ }+ until := windowEnd+ return &until+}++// parseSalesforceTime parses a Salesforce ISO-8601 timestamp.+func parseSalesforceTime(value string) (time.Time, error) {+ value = strings.TrimSpace(value)+ if value == "" {+ return time.Time{}, fmt.Errorf("empty Salesforce timestamp")+ }+ layouts := []string{+ "2006-01-02T15:04:05.000-0700",+ "2006-01-02T15:04:05-0700",+ "2006-01-02T15:04:05.000Z",+ time.RFC3339Nano,+ time.RFC3339,+ }+ for _, layout := range layouts {+ if t, err := time.Parse(layout, value); err == nil {+ return t, nil+ }+ }+ return time.Time{}, fmt.Errorf("parse Salesforce timestamp %q", value)+}++// salesforceObjectCursor is a per-object resume position keyed by the last+// ingested record. Storing both SystemModstamp and Id lets a resume re-fetch+// records that share the checkpoint timestamp but sort after it, so same-instant+// inserts are not lost across checkpoint boundaries.+type salesforceObjectCursor struct {+ SystemModstamp string `json:"system_modstamp,omitempty"`+ Id string `json:"id,omitempty"`+}++// salesforceSyncCursor is the per-object composite cursor map.+type salesforceSyncCursor struct {+ Cursors map[string]salesforceObjectCursor `json:"cursors,omitempty"`+}++// salesforceSyncSession streams Salesforce documents for one fixed sync window.+type salesforceSyncSession struct {+ connector *SalesforceConnector+ objects []string+ batchSize int+ windowStart *time.Time+ windowEnd time.Time+ cursors map[string]salesforceObjectCursor++ objectIndex int+ pageURL string+ latestISO string+ latestID string+ buffer []salesforceBufferedDocument+}++type salesforceBufferedDocument struct {+ document SourceDocument+ checkpoint *SyncCheckpoint+}++// applyResume restores the per-object cursor map from a saved checkpoint.+func (s *salesforceSyncSession) applyResume(checkpoint *SyncCheckpoint) {+ if checkpoint == nil || checkpoint.Cursor == "" {+ return+ }+ var cursor salesforceSyncCursor+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {+ return+ }+ if len(cursor.Cursors) > 0 {+ s.cursors = cursor.Cursors+ }+}++// NextBatch returns the next Salesforce document batch.+func (s *salesforceSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {+ documents := make([]SourceDocument, 0, s.batchSize)+ var checkpoint *SyncCheckpoint+ if len(s.buffer) > 0 {+ n := min(s.batchSize, len(s.buffer))+ for _, buffered := range s.buffer[:n] {+ documents = append(documents, buffered.document)+ checkpoint = buffered.checkpoint+ }+ s.buffer = s.buffer[n:]+ }+ for len(documents) < s.batchSize {+ page, err := s.nextDocumentPage(ctx)+ if err != nil {+ return SyncBatch{}, err+ }+ if len(page) == 0 {+ if s.objectIndex >= len(s.objects) {+ if len(documents) == 0 {+ return SyncBatch{}, io.EOF+ }+ break+ }+ continue+ }+ remaining := s.batchSize - len(documents)+ if len(page) > remaining {+ for _, buffered := range page[:remaining] {+ documents = append(documents, buffered.document)+ checkpoint = buffered.checkpoint+ }+ s.buffer = append(s.buffer, page[remaining:]...)+ break+ }+ for _, buffered := range page {+ documents = append(documents, buffered.document)+ checkpoint = buffered.checkpoint+ }+ }+ return SyncBatch{Documents: documents, Checkpoint: checkpoint}, nil+}++// Close closes the Salesforce sync session.+func (s *salesforceSyncSession) Close() error {+ return nil+}++// nextDocumentPage streams the next batch of raw records for the current+// object, advancing to the next object when the current one is drained.+func (s *salesforceSyncSession) nextDocumentPage(ctx context.Context) ([]salesforceBufferedDocument, error) {+ for s.objectIndex < len(s.objects) {+ obj := s.objects[s.objectIndex]+ if s.pageURL == "" {+ fields, err := s.connector.describeFields(ctx, obj)+ if err != nil {+ var unavail *salesforceObjectUnavailableError+ if errors.As(err, &unavail) {+ s.objectIndex+++ continue+ }+ return nil, err+ }+ cursor := s.objCursor(obj)+ until := salesforceWindowEnd(s.windowEnd)+ s.pageURL = s.connector.queryURL(obj, fields, cursor, until)+ }+ var page salesforceQueryPage+ if err := s.connector.getJSON(ctx, s.pageURL, &page); err != nil {+ var unavail *salesforceObjectUnavailableError+ if errors.As(err, &unavail) {+ // Object disappeared mid-query (e.g. Knowledge__kav org lacks+ // Knowledge). Skip it rather than abort the run.+ s.objectIndex+++ s.pageURL = ""+ continue+ }+ return nil, err+ }+ raw := make([]SourceDocument, 0, len(page.Records))+ for _, record := range page.Records {+ recID := stringRecordValue(record, "Id")+ if recID == "" {+ continue+ }+ modifiedStr := stringRecordValue(record, "SystemModstamp")+ if modifiedStr != "" {+ s.latestISO = modifiedStr+ s.latestID = recID+ }+ raw = append(raw, s.connector.recordToDocument(obj, record, recID, modifiedStr))+ }+ done := page.NextRecordsURL == ""+ // Capture the object's latest cursor before resetting session state.+ // The cursor only advances once the object is fully drained: the+ // object's final record checkpoint carries it, while earlier batches+ // keep the old cursor so a crash between them re-fetches the object+ // instead of skipping records that were never ingested.+ finalISO := s.latestISO+ finalID := s.latestID+ if page.NextRecordsURL != "" {+ s.pageURL = page.NextRecordsURL+ } else {+ s.objectIndex+++ s.pageURL = ""+ s.latestISO = ""+ s.latestID = ""+ }+ documents := make([]salesforceBufferedDocument, 0, len(raw))+ for index, doc := range raw {+ if done && index == len(raw)-1 && finalISO != "" {+ s.cursors[obj] = salesforceObjectCursor{SystemModstamp: finalISO, Id: finalID}+ }+ documents = append(documents, salesforceBufferedDocument{+ document: doc,+ checkpoint: s.syncCheckpoint(doc),+ })+ }+ if done && len(raw) == 0 && finalISO != "" {+ // Empty final page: still commit the drained object's cursor so+ // the next object's batches skip it on resume.+ s.cursors[obj] = salesforceObjectCursor{SystemModstamp: finalISO, Id: finalID}+ }+ if len(documents) > 0 {+ return documents, nil+ }+ }+ return nil, nil+}++// objCursor computes the per-object resume cursor: the caller window or the+// persisted cursor, whichever is later. A window bound that falls after the+// persisted cursor drops the cursor's Id so same-instant records from the+// window are not skipped; otherwise the composite timestamp+Id cursor is kept.+func (s *salesforceSyncSession) objCursor(obj string) *salesforceObjectCursor {+ cur, ok := s.cursors[obj]+ if !ok || cur.SystemModstamp == "" {+ if s.windowStart != nil {+ return &salesforceObjectCursor{SystemModstamp: salesforceSOQLTime(*s.windowStart)}+ }+ return nil+ }+ cursorTime, err := parseSalesforceTime(cur.SystemModstamp)+ if err != nil {+ if s.windowStart != nil {+ return &salesforceObjectCursor{SystemModstamp: salesforceSOQLTime(*s.windowStart)}+ }+ return &cur+ }+ if s.windowStart != nil && cursorTime.Before(*s.windowStart) {+ return &salesforceObjectCursor{SystemModstamp: salesforceSOQLTime(*s.windowStart)}+ }+ return &cur+}++// syncCheckpoint serializes the current per-object cursor map.+func (s *salesforceSyncSession) syncCheckpoint(doc SourceDocument) *SyncCheckpoint {+ cursors := make(map[string]salesforceObjectCursor, len(s.cursors))+ for obj, cur := range s.cursors {+ cursors[obj] = cur+ }+ data, err := json.Marshal(salesforceSyncCursor{Cursors: cursors})+ if err != nil {+ return nil+ }+ updatedAt := doc.UpdatedAt+ return &SyncCheckpoint{+ Cursor: string(data),+ SourceID: doc.SourceID,+ UpdatedAt: &updatedAt,+ }+}++// recordToDocument converts a SOQL record into a SourceDocument.+func (c *SalesforceConnector) recordToDocument(obj string, record map[string]any, recID, modifiedStr string) SourceDocument {+ updatedAt := time.Now().UTC()+ if modifiedStr != "" {+ if t, err := parseSalesforceTime(modifiedStr); err == nil {+ updatedAt = t+ }+ }+ name := firstNonEmpty(+ stringRecordValue(record, "Name"),+ stringRecordValue(record, "Subject"),+ stringRecordValue(record, "Title"),+ fmt.Sprintf("%s/%s", obj, recID),+ )+ body := salesforceRecordToText(obj, record)+ blob := []byte(body)+ return SourceDocument{+ SourceID: fmt.Sprintf("%s/%s", obj, recID),+ SemanticIdentifier: name,+ Extension: ".txt",+ Blob: blob,+ UpdatedAt: updatedAt,+ SizeBytes: int64(len(blob)),+ Metadata: map[string]any{+ "object": obj,+ "record_id": recID,+ "web_url": fmt.Sprintf("%s/%s", c.instanceBaseURL(), recID),+ },+ Fingerprint: contentFingerprint(blob),+ }+}++// salesforceRecordToText flattens a SOQL record into a deterministic+// plain-text body (sorted keys, stable value formatting) so content hashing is+// stable across polls.+func salesforceRecordToText(obj string, record map[string]any) string {+ lines := []string{"Salesforce " + obj}+ keys := make([]string, 0, len(record))+ for key := range record {+ if key == "attributes" {+ continue+ }+ keys = append(keys, key)+ }+ sort.Strings(keys)+ for _, key := range keys {+ value := record[key]+ if value == nil || value == "" {+ continue+ }+ if isMapOrSlice(value) {+ if data, err := json.Marshal(value); err == nil {+ lines = append(lines, fmt.Sprintf("%s: %s", key, string(data)))+ continue+ }+ }+ lines = append(lines, fmt.Sprintf("%s: %v", key, value))+ }+ return strings.Join(lines, "\n")+}++func stringRecordValue(record map[string]any, key string) string {+ value, _ := record[key].(string)+ return value+}++func isMapOrSlice(value any) bool {+ switch value.(type) {+ case map[string]any, []any:+ return true+ default:+ return false+ }+}++// salesforceQueryPage is one SOQL query result page.+type salesforceQueryPage struct {+ Records []map[string]any `json:"records"`+ NextRecordsURL string `json:"nextRecordsUrl"`+}++// salesforceSObjectsResponse is the /sobjects global describe payload.+type salesforceSObjectsResponse struct {+ SObjects []struct {+ Name string `json:"name"`+ Queryable bool `json:"queryable"`+ } `json:"sobjects"`+}++// salesforcePruneSession streams a complete Salesforce slim snapshot.+type salesforcePruneSession struct {+ connector *SalesforceConnector+ objects []string+ batchSize int+ objectIndex int+ pageURL string+ buffer []SlimDocument+}++// NextBatch returns the next Salesforce prune snapshot batch.+func (s *salesforcePruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {+ documents := make([]SlimDocument, 0, s.batchSize)+ if len(s.buffer) > 0 {+ n := min(s.batchSize, len(s.buffer))+ documents = append(documents, s.buffer[:n]...)+ s.buffer = s.buffer[n:]+ }+ for len(documents) < s.batchSize {+ page, err := s.nextSlimPage(ctx)+ if err != nil {+ return PruneBatch{}, err+ }+ if len(page) == 0 {+ if s.objectIndex >= len(s.objects) {+ if len(documents) == 0 {+ return PruneBatch{}, io.EOF+ }+ break+ }+ continue+ }+ remaining := s.batchSize - len(documents)+ if len(page) > remaining {+ documents = append(documents, page[:remaining]...)+ s.buffer = append(s.buffer, page[remaining:]...)+ break+ }+ documents = append(documents, page...)+ }+ return PruneBatch{Documents: documents}, nil+}++// Close closes the Salesforce prune session.+func (s *salesforcePruneSession) Close() error {+ return nil+}++// nextSlimPage streams the next page of slim IDs for the current object.+func (s *salesforcePruneSession) nextSlimPage(ctx context.Context) ([]SlimDocument, error) {+ for s.objectIndex < len(s.objects) {+ obj := s.objects[s.objectIndex]+ if s.pageURL == "" {+ s.pageURL = s.connector.queryURL(obj, []string{"Id"}, nil, nil)+ }+ var page salesforceQueryPage+ if err := s.connector.getJSON(ctx, s.pageURL, &page); err != nil {+ var unavail *salesforceObjectUnavailableError+ if errors.As(err, &unavail) {+ s.objectIndex+++ s.pageURL = ""+ continue+ }+ return nil, err+ }+ documents := make([]SlimDocument, 0, len(page.Records))+ for _, record := range page.Records {+ recID := stringRecordValue(record, "Id")+ if recID == "" {+ continue+ }+ documents = append(documents, SlimDocument{SourceID: fmt.Sprintf("%s/%s", obj, recID)})+ }+ if page.NextRecordsURL != "" {+ s.pageURL = page.NextRecordsURL+ } else {+ s.objectIndex+++ s.pageURL = ""+ }+ if len(documents) > 0 {+ return documents, nil+ }+ }+ return nil, nil+}diff --git a/internal/syncer/connector/salesforce_test.go b/internal/syncer/connector/salesforce_test.gonew file mode 100644index 0000000000..12f9c3d6d5--- /dev/null+++ b/internal/syncer/connector/salesforce_test.go@@ -0,0 +1,1132 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "encoding/json"+ "errors"+ "io"+ "net/http"+ "net/url"+ "strings"+ "testing"+ "time"++ "ragflow/internal/utility"+)++func TestNewSalesforceConnectorDefaults(t *testing.T) {+ connector, err := NewSalesforceConnector(map[string]any{+ "credentials": map[string]any{+ "instance_url": "https://your-domain.my.salesforce.com/",+ "client_id": "client",+ "client_secret": "secret",+ },+ })+ if err != nil {+ t.Fatalf("NewSalesforceConnector failed: %v", err)+ }+ if connector.instanceURL != "https://your-domain.my.salesforce.com" {+ t.Fatalf("instance url = %q", connector.instanceURL)+ }+ if connector.apiVersion != "v59.0" {+ t.Fatalf("api version = %q", connector.apiVersion)+ }+ if connector.batchSize != 2 {+ t.Fatalf("batch size = %d", connector.batchSize)+ }+ want := []string{"Account", "Contact", "Opportunity", "Case", "Knowledge__kav"}+ if len(connector.objects) != len(want) {+ t.Fatalf("objects = %v, want %v", connector.objects, want)+ }+ for i := range want {+ if connector.objects[i] != want[i] {+ t.Fatalf("objects = %v, want %v", connector.objects, want)+ }+ }+}++func TestNewSalesforceConnectorObjectsAndBatch(t *testing.T) {+ connector, err := NewSalesforceConnector(map[string]any{+ "objects": "Account, Contact",+ "api_version": "v62.0",+ "batch_size": 5,+ "credentials": map[string]any{+ "instance_url": "https://acme.my.salesforce.com",+ "client_id": "client",+ "client_secret": "secret",+ },+ })+ if err != nil {+ t.Fatalf("NewSalesforceConnector failed: %v", err)+ }+ if len(connector.objects) != 2 || connector.objects[0] != "Account" || connector.objects[1] != "Contact" {+ t.Fatalf("objects = %v", connector.objects)+ }+ if connector.apiVersion != "v62.0" {+ t.Fatalf("api version = %q", connector.apiVersion)+ }+ if connector.batchSize != 5 {+ t.Fatalf("batch size = %d", connector.batchSize)+ }+}++func TestSalesforceConnectorValidateMissingCredentials(t *testing.T) {+ connector, err := NewSalesforceConnector(map[string]any{"credentials": map[string]any{}})+ if err != nil {+ t.Fatalf("NewSalesforceConnector failed: %v", err)+ }+ var credErr *ConnectorMissingCredentialError+ if err := connector.Validate(context.Background()); !errors.As(err, &credErr) {+ t.Fatalf("Validate err = %v, want ConnectorMissingCredentialError", err)+ }+}++func TestSalesforceConnectorValidateRejectsNonPositiveBatch(t *testing.T) {+ connector, err := NewSalesforceConnector(map[string]any{+ "batch_size": 0,+ "credentials": map[string]any{+ "instance_url": "https://acme.my.salesforce.com",+ "client_id": "client",+ "client_secret": "secret",+ },+ })+ if err != nil {+ t.Fatalf("NewSalesforceConnector failed: %v", err)+ }+ var valErr *ConnectorValidationError+ if err := connector.Validate(context.Background()); !errors.As(err, &valErr) {+ t.Fatalf("Validate err = %v, want ConnectorValidationError", err)+ }+}++func TestSalesforceConnectorValidateQueriesObjects(t *testing.T) {+ connector := newSalesforceFixtureConnector()+ var probed bool+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {+ if !strings.HasSuffix(apiURL, "/services/data/v59.0/sobjects") {+ t.Fatalf("validate url = %q", apiURL)+ }+ probed = true+ payload := map[string]any{+ "sobjects": []any{+ map[string]any{"name": "Account", "queryable": true},+ map[string]any{"name": "Contact", "queryable": true},+ map[string]any{"name": "Opportunity", "queryable": true},+ map[string]any{"name": "Case", "queryable": true},+ map[string]any{"name": "Knowledge__kav", "queryable": true},+ },+ }+ data, _ := json.Marshal(payload)+ return json.Unmarshal(data, out)+ }+ if err := connector.Validate(context.Background()); err != nil {+ t.Fatalf("Validate failed: %v", err)+ }+ if !probed {+ t.Fatalf("Validate did not probe /sobjects")+ }+}++func TestSalesforceConnectorValidateUnknownObject(t *testing.T) {+ connector := newSalesforceFixtureConnector()+ connector.objects = []string{"Account", "Bogus"}+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {+ payload := map[string]any{+ "sobjects": []any{+ map[string]any{"name": "Account", "queryable": true},+ },+ }+ data, _ := json.Marshal(payload)+ return json.Unmarshal(data, out)+ }+ var valErr *ConnectorValidationError+ if err := connector.Validate(context.Background()); !errors.As(err, &valErr) {+ t.Fatalf("Validate err = %v, want ConnectorValidationError", err)+ }+}++func TestSalesforceConnectorValidateSkipsOptionalKnowledge(t *testing.T) {+ connector := newSalesforceFixtureConnector()+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {+ payload := map[string]any{+ "sobjects": []any{+ map[string]any{"name": "Account", "queryable": true},+ map[string]any{"name": "Contact", "queryable": true},+ map[string]any{"name": "Opportunity", "queryable": true},+ map[string]any{"name": "Case", "queryable": true},+ // Knowledge__kav absent: must be skipped silently.+ },+ }+ data, _ := json.Marshal(payload)+ return json.Unmarshal(data, out)+ }+ if err := connector.Validate(context.Background()); err != nil {+ t.Fatalf("Validate failed: %v", err)+ }+}++func TestSalesforceConnectorValidateMapsHTTPStatus(t *testing.T) {+ cases := []struct {+ name string+ status int+ want error+ }{+ {name: "unauthorized", status: http.StatusUnauthorized, want: &ConnectorMissingCredentialError{}},+ {name: "forbidden", status: http.StatusForbidden, want: &ConnectorValidationError{}},+ {name: "server error", status: http.StatusInternalServerError, want: &ConnectorValidationError{}},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ connector := newSalesforceFixtureConnector()+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {+ return &salesforceHTTPError{status: tc.status, body: "boom"}+ }+ err := connector.Validate(context.Background())+ if tc.want == nil {+ if err != nil {+ t.Fatalf("Validate err = %v, want nil", err)+ }+ return+ }+ if err == nil {+ t.Fatalf("Validate err = nil, want %T", tc.want)+ }+ switch tc.want.(type) {+ case *ConnectorMissingCredentialError:+ var want *ConnectorMissingCredentialError+ if !errors.As(err, &want) {+ t.Fatalf("Validate err = %v, want ConnectorMissingCredentialError", err)+ }+ case *ConnectorValidationError:+ var want *ConnectorValidationError+ if !errors.As(err, &want) {+ t.Fatalf("Validate err = %v, want ConnectorValidationError", err)+ }+ }+ })+ }+}++func TestSalesforceConnectorOpenSync(t *testing.T) {+ connector := newSalesforceFixtureConnector()+ connector.doJSON = salesforceFixtureDoJSON(t)++ start := mustTime(t, "2026-01-02T00:00:00Z")+ end := mustTime(t, "2026-01-04T00:00:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{WindowStart: &start, WindowEnd: end})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 2 {+ t.Fatalf("documents len = %d, want 2", len(batch.Documents))+ }+ doc := batch.Documents[0]+ if doc.SourceID != "Account/0015g00000Example1" {+ t.Fatalf("source id = %q", doc.SourceID)+ }+ if doc.SemanticIdentifier != "Acme Corp" {+ t.Fatalf("semantic identifier = %q", doc.SemanticIdentifier)+ }+ if doc.Extension != ".txt" {+ t.Fatalf("extension = %q", doc.Extension)+ }+ if !doc.UpdatedAt.Equal(mustTime(t, "2026-01-03T00:00:00Z")) {+ t.Fatalf("updated at = %s", doc.UpdatedAt)+ }+ if doc.Metadata["object"] != "Account" || doc.Metadata["record_id"] != "0015g00000Example1" {+ t.Fatalf("metadata = %+v", doc.Metadata)+ }+ if doc.Metadata["web_url"] != "https://acme.my.salesforce.com/0015g00000Example1" {+ t.Fatalf("web_url = %v", doc.Metadata["web_url"])+ }+ if doc.Fingerprint == "" {+ t.Fatalf("fingerprint is empty")+ }+ blob := string(doc.Blob)+ if !strings.Contains(blob, "Salesforce Account") || !strings.Contains(blob, "Name: Acme Corp") {+ t.Fatalf("blob = %q", blob)+ }+ if batch.Checkpoint == nil || batch.Checkpoint.SourceID != "Account/0015g00000Example2" {+ t.Fatalf("checkpoint = %+v", batch.Checkpoint)+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}++func TestSalesforceConnectorOpenSyncWindowAndPagination(t *testing.T) {+ connector := newSalesforceFixtureConnector()+ var soql string+ var fieldsQueried string+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {+ parsed, err := url.Parse(apiURL)+ if err != nil {+ t.Fatalf("parse url: %v", err)+ }+ if strings.Contains(apiURL, "/sobjects/Account/describe") {+ // Compound address/location fields must be filtered from SOQL.+ fieldsQueried = ""+ }+ if strings.Contains(apiURL, "/query?") {+ soql, _ = url.QueryUnescape(parsed.Query().Get("q"))+ }+ return salesforceFixtureDoJSON(t)(ctx, apiURL, out)+ }++ start := mustTime(t, "2026-01-02T00:00:00Z")+ end := mustTime(t, "2026-01-04T00:00:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{WindowStart: &start, WindowEnd: end})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ for {+ _, err := session.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ }+ if !strings.Contains(soql, "SystemModstamp > 2026-01-02T00:00:00Z") {+ t.Fatalf("soql missing since bound: %q", soql)+ }+ if !strings.Contains(soql, "SystemModstamp <= 2026-01-04T00:00:00Z") {+ t.Fatalf("soql missing until bound: %q", soql)+ }+ if !strings.Contains(soql, " ORDER BY SystemModstamp ASC") {+ t.Fatalf("soql missing ordering: %q", soql)+ }+ if strings.Contains(soql, "BillingAddress") || strings.Contains(soql, "Location__c") {+ t.Fatalf("soql must exclude compound fields: %q", soql)+ }+ _ = fieldsQueried+}++func TestSalesforceConnectorOpenSyncResume(t *testing.T) {+ connector := newSalesforceFixtureConnector()+ connector.doJSON = salesforceFixtureDoJSON(t)++ // Drain the whole object so the final record's checkpoint advances the+ // per-object cursor; the syncer persists that checkpoint per batch.+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("first NextBatch failed: %v", err)+ }+ if len(first.Documents) != 2 {+ t.Fatalf("first documents len = %d, want 2", len(first.Documents))+ }+ if first.Checkpoint == nil {+ t.Fatalf("first checkpoint is nil")+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("first session EOF = %v", err)+ }++ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: first.Checkpoint})+ if err != nil {+ t.Fatalf("resume OpenSync failed: %v", err)+ }+ if _, err = resumed.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("resume NextBatch = %v, want EOF (object already ingested)", err)+ }+}++func TestSalesforceConnectorOpenSyncPaginatedResume(t *testing.T) {+ connector := newSalesforceFixtureConnector()+ connector.batchSize = 2+ records := []map[string]any{+ {"Id": "0015g00000Example1", "Name": "Acme Corp", "SystemModstamp": "2026-01-03T00:00:00.000+0000"},+ {"Id": "0015g00000Example2", "Name": "Globex", "SystemModstamp": "2026-01-03T01:00:00.000+0000"},+ {"Id": "0015g00000Example3", "Name": "Initech", "SystemModstamp": "2026-01-03T02:00:00.000+0000"},+ }+ var queryCalls int+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {+ if strings.Contains(apiURL, "/sobjects/Account/describe") {+ return json.Unmarshal([]byte(`{"fields":[+ {"name":"Id","type":"id"},+ {"name":"Name","type":"string"},+ {"name":"SystemModstamp","type":"datetime"}+ ]}`), out)+ }+ if strings.Contains(apiURL, "/query?") || strings.Contains(apiURL, "/query/01gExampleNext") {+ queryCalls+++ var page salesforceQueryPage+ if queryCalls == 1 {+ page = salesforceQueryPage{+ Records: records[:2],+ NextRecordsURL: "/services/data/v59.0/query/01gExampleNext",+ }+ } else {+ // Second page: fixture returns the last record with done=true.+ // The per-object cursor must only advance on this final record.+ page = salesforceQueryPage{Records: records[2:]}+ }+ data, _ := json.Marshal(page)+ return json.Unmarshal(data, out)+ }+ t.Fatalf("unexpected url %s", apiURL)+ return nil+ }++ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("first NextBatch failed: %v", err)+ }+ if len(first.Documents) != 2 || first.Documents[0].SourceID != "Account/0015g00000Example1" {+ t.Fatalf("first documents = %+v", first.Documents)+ }+ // The first batch ends before the object drains, so its checkpoint must+ // NOT advance the Account cursor yet (no committed cursor to resume from).+ if first.Checkpoint == nil {+ t.Fatalf("first checkpoint is nil")+ }+ var firstCursor salesforceSyncCursor+ if err := json.Unmarshal([]byte(first.Checkpoint.Cursor), &firstCursor); err != nil {+ t.Fatalf("parse first checkpoint cursor: %v", err)+ }+ if _, ok := firstCursor.Cursors["Account"]; ok {+ t.Fatalf("first batch checkpoint cursor advanced Account too early: %+v", firstCursor.Cursors)+ }++ second, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("second NextBatch failed: %v", err)+ }+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "Account/0015g00000Example3" {+ t.Fatalf("second documents = %+v", second.Documents)+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("session EOF = %v", err)+ }+ // The final batch's checkpoint carries the advanced Account cursor.+ var finalCursor salesforceSyncCursor+ if err := json.Unmarshal([]byte(second.Checkpoint.Cursor), &finalCursor); err != nil {+ t.Fatalf("parse final checkpoint cursor: %v", err)+ }+ finalAccount := finalCursor.Cursors["Account"]+ if finalAccount.SystemModstamp != "2026-01-03T02:00:00.000+0000" || finalAccount.Id != "0015g00000Example3" {+ t.Fatalf("final checkpoint cursor = %+v, want Account advanced with composite cursor", finalCursor.Cursors)+ }++ // Resume from the second batch: the object is fully ingested, so the next+ // run must not re-emit its records. The fixture returns all three records+ // on its first query page for the resumed run; the since predicate derived+ // from the advanced cursor must filter them out.+ connector2 := newSalesforceFixtureConnector()+ connector2.batchSize = 2+ var resumedQueryCalls int+ connector2.doJSON = func(ctx context.Context, apiURL string, out any) error {+ if strings.Contains(apiURL, "/sobjects/Account/describe") {+ return json.Unmarshal([]byte(`{"fields":[+ {"name":"Id","type":"id"},+ {"name":"Name","type":"string"},+ {"name":"SystemModstamp","type":"datetime"}+ ]}`), out)+ }+ if strings.Contains(apiURL, "/query?") {+ resumedQueryCalls+++ parsed, _ := url.Parse(apiURL)+ soql, _ := url.QueryUnescape(parsed.Query().Get("q"))+ if !strings.Contains(soql, "SystemModstamp > 2026-01-03T02:00:00Z") {+ t.Fatalf("resumed soql missing since predicate: %q", soql)+ }+ var page salesforceQueryPage+ if strings.Contains(soql, "SystemModstamp > 2026-01-03T02:00:00Z") {+ // No records strictly newer than the cursor.+ page = salesforceQueryPage{Records: []map[string]any{}}+ } else {+ page = salesforceQueryPage{Records: records}+ }+ data, _ := json.Marshal(page)+ return json.Unmarshal(data, out)+ }+ t.Fatalf("unexpected url %s", apiURL)+ return nil+ }+ resumed, err := connector2.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: second.Checkpoint})+ if err != nil {+ t.Fatalf("resumed OpenSync failed: %v", err)+ }+ if _, err = resumed.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("resumed NextBatch = %v, want EOF", err)+ }+}++func TestSalesforceConnectorOpenPrune(t *testing.T) {+ connector := newSalesforceFixtureConnector()+ connector.doJSON = salesforceFixtureDoJSON(t)++ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ got := []string{}+ for _, doc := range batch.Documents {+ got = append(got, doc.SourceID)+ }+ want := []string{"Account/0015g00000Example1", "Account/0015g00000Example2"}+ if len(got) != len(want) {+ t.Fatalf("prune documents = %v, want %v", got, want)+ }+ for i := range want {+ if got[i] != want[i] {+ t.Fatalf("prune documents = %v, want %v", got, want)+ }+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("prune NextBatch EOF = %v", err)+ }+}++func TestSalesforceConnectorOpenSyncSkipsMissingObject(t *testing.T) {+ connector := newSalesforceFixtureConnector()+ connector.objects = []string{"Account", "Case"}+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {+ if strings.Contains(apiURL, "/sobjects/Case/describe") {+ return &salesforceObjectUnavailableError{message: "Case unavailable"}+ }+ return salesforceFixtureDoJSON(t)(ctx, apiURL, out)+ }+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 2 {+ t.Fatalf("documents len = %d, want 2 (Case skipped)", len(batch.Documents))+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}++func TestSalesforceConnectorOpenPruneSkipsMissingObject(t *testing.T) {+ connector := newSalesforceFixtureConnector()+ connector.objects = []string{"Account", "Case"}+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {+ if strings.Contains(apiURL, "/query?") && strings.Contains(apiURL, "FROM+Case") {+ return &salesforceObjectUnavailableError{message: "Case unavailable"}+ }+ return salesforceFixtureDoJSON(t)(ctx, apiURL, out)+ }+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ got := []string{}+ for _, doc := range batch.Documents {+ got = append(got, doc.SourceID)+ }+ if len(got) != 2 || got[0] != "Account/0015g00000Example1" {+ t.Fatalf("prune documents = %v, want Account records only", got)+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("prune NextBatch EOF = %v", err)+ }+}++func TestSalesforceRecordToTextDeterministic(t *testing.T) {+ record := map[string]any{+ "Name": "Acme Corp",+ "Industry": "Software",+ "attributes": map[string]any{"type": "Account"},+ "AnnualRevenue": 1000.5,+ "Description": "",+ }+ text1 := salesforceRecordToText("Account", record)+ text2 := salesforceRecordToText("Account", record)+ if text1 != text2 {+ t.Fatalf("record text unstable: %q vs %q", text1, text2)+ }+ if !strings.HasPrefix(text1, "Salesforce Account\n") {+ t.Fatalf("record text = %q", text1)+ }+ if strings.Contains(text1, "attributes") {+ t.Fatalf("record text should skip attributes: %q", text1)+ }+ if strings.Contains(text1, "Description:") {+ t.Fatalf("record text should skip empty values: %q", text1)+ }+}++func TestParseSalesforceTime(t *testing.T) {+ cases := []string{+ "2026-01-03T00:00:00.000+0000",+ "2026-01-03T00:00:00+0000",+ "2026-01-03T00:00:00.000Z",+ "2026-01-03T00:00:00Z",+ }+ for _, value := range cases {+ parsed, err := parseSalesforceTime(value)+ if err != nil {+ t.Fatalf("parse %q: %v", value, err)+ }+ if parsed.UTC() != mustTime(t, "2026-01-03T00:00:00Z") {+ t.Fatalf("parse %q = %s", value, parsed)+ }+ }+}++func TestSalesforceConnectorValidateConnectorSetting(t *testing.T) {+ connector := newSalesforceFixtureConnector()+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {+ payload := map[string]any{+ "sobjects": []any{+ map[string]any{"name": "Account", "queryable": true},+ map[string]any{"name": "Contact", "queryable": true},+ map[string]any{"name": "Opportunity", "queryable": true},+ map[string]any{"name": "Case", "queryable": true},+ map[string]any{"name": "Knowledge__kav", "queryable": true},+ },+ }+ data, _ := json.Marshal(payload)+ return json.Unmarshal(data, out)+ }+ // The receiver is a valid fixture; the unsaved request is what must be+ // validated. Use a request whose credential set is incomplete so the+ // temporary connector fails fast regardless of the receiver's state.+ request := map[string]any{+ "credentials": map[string]any{+ "instance_url": "https://acme.my.salesforce.com",+ "client_id": "client",+ // client_secret intentionally omitted.+ },+ }+ var credErr *ConnectorMissingCredentialError+ if err := connector.ValidateConnectorSetting(context.Background(), request); !errors.As(err, &credErr) {+ t.Fatalf("ValidateConnectorSetting err = %v, want ConnectorMissingCredentialError for the request", err)+ }+}++func TestSalesforceConnectorValidateConnectorSettingUsesRequest(t *testing.T) {+ connector := newSalesforceFixtureConnector()+ // Receiver itself would pass with only Account; the request lists Bogus, so+ // a successful validation must be derived from the request, not the receiver.+ connector.objects = []string{"Account"}+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {+ payload := map[string]any{+ "sobjects": []any{+ map[string]any{"name": "Account", "queryable": true},+ },+ }+ data, _ := json.Marshal(payload)+ return json.Unmarshal(data, out)+ }+ request := map[string]any{+ "objects": "Account, Bogus",+ "credentials": map[string]any{+ "instance_url": "https://acme.my.salesforce.com",+ "client_id": "client",+ "client_secret": "secret",+ },+ }+ var valErr *ConnectorValidationError+ if err := connector.ValidateConnectorSetting(context.Background(), request); !errors.As(err, &valErr) {+ t.Fatalf("ValidateConnectorSetting err = %v, want ConnectorValidationError for the unsaved request", err)+ }+}++func TestRegisterBuiltInsOpensSalesforce(t *testing.T) {+ registry := NewRegistry()+ RegisterBuiltIns(registry)+ connector, err := registry.OpenFromConfig("salesforce", map[string]any{+ "credentials": map[string]any{+ "instance_url": "https://acme.my.salesforce.com",+ "client_id": "client",+ "client_secret": "secret",+ },+ })+ if err != nil {+ t.Fatalf("OpenFromConfig failed: %v", err)+ }+ if _, ok := connector.(*SalesforceConnector); !ok {+ t.Fatalf("connector type = %T, want *SalesforceConnector", connector)+ }+}++func TestSalesforceConnectorOpenSyncSameTimestampResume(t *testing.T) {+ ts := "2026-01-03T00:00:00.000+0000"+ records := []map[string]any{+ {"Id": "0015g00000Example1", "Name": "Acme Corp", "SystemModstamp": ts},+ {"Id": "0015g00000Example2", "Name": "Globex", "SystemModstamp": ts},+ }++ // First run with batchSize 1: each record becomes one batch and the object+ // only drains on the last one, so the persisted cursor advances to the+ // composite position (ts, Example2) of the final same-instant record.+ connector := newSalesforceFixtureConnector()+ connector.batchSize = 1+ connector.doJSON = salesforceFixtureRecordsDoJSON(t, records)+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ var checkpoint *SyncCheckpoint+ for {+ batch, err := session.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if batch.Checkpoint != nil {+ checkpoint = batch.Checkpoint+ }+ }+ var curs salesforceSyncCursor+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &curs); err != nil {+ t.Fatalf("parse cursor: %v", err)+ }+ acct := curs.Cursors["Account"]+ if acct.SystemModstamp != ts || acct.Id != "0015g00000Example2" {+ t.Fatalf("cursor = %+v, want (ts, Example2)", acct)+ }++ // A third record created in the same instant but sorting after the+ // checkpoint by Id must be delivered on resume. The resumed fixture still+ // carries all records; the composite WHERE clause must select only the+ // same-instant record whose Id sorts after the cursor.+ resumedRecords := append(append([]map[string]any{}, records...), map[string]any{+ "Id": "0015g00000Example3", "Name": "Initech", "SystemModstamp": ts,+ })+ resumed := newSalesforceFixtureConnector()+ resumed.batchSize = 1+ var resumedSOQL string+ resumed.doJSON = func(ctx context.Context, apiURL string, out any) error {+ if strings.Contains(apiURL, "/query?") {+ parsed, err := url.Parse(apiURL)+ if err != nil {+ t.Fatalf("parse query url: %v", err)+ }+ resumedSOQL, _ = url.QueryUnescape(parsed.Query().Get("q"))+ }+ return salesforceFixtureRecordsDoJSON(t, resumedRecords)(ctx, apiURL, out)+ }+ resumedSession, err := resumed.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: checkpoint})+ if err != nil {+ t.Fatalf("resumed OpenSync failed: %v", err)+ }+ var gotIDs []string+ for {+ batch, err := resumedSession.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ t.Fatalf("resumed NextBatch failed: %v", err)+ }+ for _, doc := range batch.Documents {+ gotIDs = append(gotIDs, doc.SourceID)+ }+ }+ if !strings.Contains(resumedSOQL, "AND Id > '0015g00000Example2'") {+ t.Fatalf("resumed SOQL missing same-instant Id boundary: %q", resumedSOQL)+ }+ if len(gotIDs) != 1 || gotIDs[0] != "Account/0015g00000Example3" {+ t.Fatalf("resumed documents = %v, want only the same-instant record after the cursor", gotIDs)+ }+}++func TestSalesforceObjectUnavailable404(t *testing.T) {+ cases := []struct {+ name string+ body string+ want bool+ }{+ {+ name: "structured NOT_FOUND",+ body: `[{"message":"sObject type 'Bogus' is not supported. If you intend to use a custom object, make sure it is enabled in \"Setup > Object Manager\" and \"API\" is enabled in the Object's Detail Page.","errorCode":"NOT_FOUND"}]`,+ want: true,+ },+ {+ name: "object level permissions NOT_FOUND",+ body: `[{"message":"Object type 'Case' is not supported. If you intend to use a custom object...","errorCode":"NOT_FOUND"}]`,+ want: true,+ },+ {+ name: "generic not found route",+ body: `404 page not found`,+ want: false,+ },+ {+ name: "empty body",+ body: ``,+ want: false,+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ if got := salesforceObjectUnavailable(http.StatusNotFound, []byte(tc.body)); got != tc.want {+ t.Fatalf("salesforceObjectUnavailable(404, %q) = %v, want %v", tc.body, got, tc.want)+ }+ })+ }+}++func TestSalesforceObjectUnavailable400InvalidType(t *testing.T) {+ body := `[{"message":"\nSELECT Account__c FROM Bogus\n ^\nERROR at Row:1:Column:18\nsObject type 'Bogus' is not supported.","errorCode":"INVALID_TYPE"}]`+ if !salesforceObjectUnavailable(http.StatusBadRequest, []byte(body)) {+ t.Fatalf("expected 400 INVALID_TYPE to be unavailable")+ }+ if salesforceObjectUnavailable(http.StatusBadRequest, []byte(`bad request`)) {+ t.Fatalf("did not expect generic 400 to be unavailable")+ }+ if salesforceObjectUnavailable(http.StatusForbidden, []byte(`forbidden`)) {+ t.Fatalf("did not expect 403 to be unavailable")+ }+}++func TestSalesforceHostAllowed(t *testing.T) {+ allowed := []string{+ "acme.my.salesforce.com",+ "login.salesforce.com",+ "ACME.MY.SALESFORCE.COM",+ "instance.salesforce.com",+ "custom.force.com",+ "org.lightning.force.com",+ "salesforce.com",+ "force.com",+ }+ for _, host := range allowed {+ if !salesforceHostAllowed(host) {+ t.Fatalf("salesforceHostAllowed(%q) = false, want true", host)+ }+ }+ blocked := []string{+ "",+ "acme.my.salesforce.com.evil.com",+ "evil.example.com",+ "salesforce.com.evil.com",+ "force.com.attacker.io",+ "127.0.0.1",+ }+ for _, host := range blocked {+ if salesforceHostAllowed(host) {+ t.Fatalf("salesforceHostAllowed(%q) = true, want false", host)+ }+ }+}++func TestRequestAccessTokenRejectsNonSalesforceHost(t *testing.T) {+ origLookup := utility.LookupHost+ utility.LookupHost = func(host string) ([]string, error) {+ return []string{"93.184.216.34"}, nil+ }+ t.Cleanup(func() { utility.LookupHost = origLookup })++ connector := &SalesforceConnector{+ instanceURL: "https://evil.example.com",+ clientID: "client",+ clientSecret: "secret",+ now: time.Now,+ }+ var credErr *ConnectorMissingCredentialError+ if _, err := connector.requestAccessToken(context.Background()); !errors.As(err, &credErr) {+ t.Fatalf("requestAccessToken err = %v, want ConnectorMissingCredentialError for non-Salesforce host", err)+ }+}++func TestRequestAccessTokenRequiresHTTPS(t *testing.T) {+ origLookup := utility.LookupHost+ utility.LookupHost = func(host string) ([]string, error) {+ return []string{"93.184.216.34"}, nil+ }+ t.Cleanup(func() { utility.LookupHost = origLookup })++ connector := &SalesforceConnector{+ instanceURL: "http://acme.my.salesforce.com",+ clientID: "client",+ clientSecret: "secret",+ now: time.Now,+ }+ var credErr *ConnectorMissingCredentialError+ if _, err := connector.requestAccessToken(context.Background()); !errors.As(err, &credErr) {+ t.Fatalf("requestAccessToken err = %v, want ConnectorMissingCredentialError for non-HTTPS token endpoint", err)+ }+}++func TestSalesforceConnectorAPIURLAbsoluteValidation(t *testing.T) {+ connector := newSalesforceFixtureConnector()+ snap := salesforceToken{InstanceURL: "https://acme.my.salesforce.com"}++ // A well-formed absolute pagination URL on an approved HTTPS host is kept.+ valid := "https://instance2.my.salesforce.com/services/data/v59.0/query/01gX"+ got, err := connector.apiURL(snap, valid)+ if err != nil {+ t.Fatalf("apiURL(valid) err = %v", err)+ }+ if got != valid {+ t.Fatalf("apiURL(valid) = %q", got)+ }++ // Relative Salesforce API paths are still constructed from the snapshot.+ rel, err := connector.apiURL(snap, "/sobjects/Account/describe")+ if err != nil {+ t.Fatalf("apiURL(relative) err = %v", err)+ }+ wantRel := "https://acme.my.salesforce.com/services/data/v59.0/sobjects/Account/describe"+ if rel != wantRel {+ t.Fatalf("apiURL(relative) = %q, want %q", rel, wantRel)+ }++ // Absolute pagination URLs with a bad scheme or host must be rejected.+ for _, bad := range []string{+ "http://acme.my.salesforce.com/services/data/v59.0/query/01gX",+ "https://evil.example.com/services/data/v59.0/query/01gX",+ "https://acme.my.salesforce.com.evil.com/services/data/v59.0/query/01gX",+ } {+ if _, err := connector.apiURL(snap, bad); err == nil {+ t.Fatalf("apiURL(%q) succeeded, want error", bad)+ }+ }+}++func TestSalesforceConnectorGetJSONRejectsUnsafePaginationURL(t *testing.T) {+ connector := newSalesforceFixtureConnector()+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {+ t.Fatalf("doJSON must not run for an unsafe pagination URL")+ return nil+ }+ var page salesforceQueryPage+ if err := connector.getJSON(context.Background(), "http://evil.example.com/services/data/v59.0/query/01gX", &page); err == nil {+ t.Fatalf("getJSON succeeded for an unsafe pagination URL, want error")+ }+}++func TestSalesforceConnectorDoGetRejectsUnsafeURL(t *testing.T) {+ connector := newSalesforceFixtureConnector()+ for _, bad := range []string{+ "http://acme.my.salesforce.com/services/data/v59.0/sobjects",+ "https://evil.example.com/services/data/v59.0/sobjects",+ } {+ if _, _, err := connector.doGet(context.Background(), bad, "token"); err == nil {+ t.Fatalf("doGet(%q) succeeded, want error", bad)+ }+ }+}++// newSalesforceFixtureConnector builds a connector with token acquisition+// short-circuited so unit tests never touch the network.+func newSalesforceFixtureConnector() *SalesforceConnector {+ connector := &SalesforceConnector{+ instanceURL: "https://acme.my.salesforce.com",+ clientID: "client",+ clientSecret: "secret",+ objects: []string{"Account"},+ apiVersion: defaultSalesforceAPIVersion,+ batchSize: defaultSalesforceBatchSize,+ now: time.Now,+ }+ connector.acquireAccessToken = func(ctx context.Context) (salesforceToken, error) {+ return salesforceToken{+ AccessToken: "token",+ InstanceURL: "https://acme.my.salesforce.com",+ ExpiresAt: time.Now().Add(time.Hour),+ }, nil+ }+ return connector+}++// salesforceFixtureDoJSON serves describe + query responses for unit tests.+// The query endpoint emulates server-side SOQL filtering on SystemModstamp so+// resume tests behave like a real org.+func salesforceFixtureDoJSON(t *testing.T) func(ctx context.Context, apiURL string, out any) error {+ t.Helper()+ records := []map[string]any{+ {+ "Id": "0015g00000Example1",+ "Name": "Acme Corp",+ "Industry": "Software",+ "SystemModstamp": "2026-01-03T00:00:00.000+0000",+ },+ {+ "Id": "0015g00000Example2",+ "Name": "Globex",+ "Industry": "Hardware",+ "SystemModstamp": "2026-01-03T01:00:00.000+0000",+ },+ }+ return func(ctx context.Context, apiURL string, out any) error {+ var body string+ switch {+ case strings.Contains(apiURL, "/sobjects/Account/describe"):+ body = `{"fields":[+ {"name":"Id","type":"id"},+ {"name":"Name","type":"string"},+ {"name":"Industry","type":"string"},+ {"name":"AnnualRevenue","type":"currency"},+ {"name":"BillingAddress","type":"address"},+ {"name":"Location__c","type":"location"}+ ]}`+ case strings.Contains(apiURL, "/query?"):+ parsed, err := url.Parse(apiURL)+ if err != nil {+ t.Fatalf("parse query url: %v", err)+ }+ soql, err := url.QueryUnescape(parsed.Query().Get("q"))+ if err != nil {+ t.Fatalf("unescape soql: %v", err)+ }+ filtered := []map[string]any{}+ for _, record := range records {+ if salesforceFixtureMatchesSOQL(t, soql, record) {+ filtered = append(filtered, record)+ }+ }+ payload := map[string]any{"totalSize": len(filtered), "done": true, "records": filtered}+ data, _ := json.Marshal(payload)+ return json.Unmarshal(data, out)+ default:+ t.Fatalf("unexpected api url %s", apiURL)+ }+ return json.Unmarshal([]byte(body), out)+ }+}++// salesforceFixtureRecordsDoJSON serves describe + query responses for an+// arbitrary record set, emulating server-side SOQL filtering on SystemModstamp+// (including the composite resume predicate).+func salesforceFixtureRecordsDoJSON(t *testing.T, records []map[string]any) func(ctx context.Context, apiURL string, out any) error {+ t.Helper()+ return func(ctx context.Context, apiURL string, out any) error {+ switch {+ case strings.Contains(apiURL, "/sobjects/Account/describe"):+ body := `{"fields":[+ {"name":"Id","type":"id"},+ {"name":"Name","type":"string"},+ {"name":"SystemModstamp","type":"datetime"}+ ]}`+ return json.Unmarshal([]byte(body), out)+ case strings.Contains(apiURL, "/query?"):+ parsed, err := url.Parse(apiURL)+ if err != nil {+ t.Fatalf("parse query url: %v", err)+ }+ soql, err := url.QueryUnescape(parsed.Query().Get("q"))+ if err != nil {+ t.Fatalf("unescape soql: %v", err)+ }+ filtered := []map[string]any{}+ for _, record := range records {+ if salesforceFixtureMatchesSOQL(t, soql, record) {+ filtered = append(filtered, record)+ }+ }+ payload := map[string]any{"totalSize": len(filtered), "done": true, "records": filtered}+ data, _ := json.Marshal(payload)+ return json.Unmarshal(data, out)+ default:+ t.Fatalf("unexpected api url %s", apiURL)+ }+ return nil+ }+}++// salesforceFixtureMatchesSOQL applies the fixture's SystemModstamp predicates,+// including the composite resume clause `(SystemModstamp > X OR (SystemModstamp+// = X AND Id > 'Y'))` so same-instant records that sort after the checkpoint by+// Id are kept.+func salesforceFixtureMatchesSOQL(t *testing.T, soql string, record map[string]any) bool {+ t.Helper()+ lower := strings.ToLower(soql)+ since := time.Time{}+ until := time.Time{}+ equalSince := false+ minID := ""+ if idx := strings.Index(lower, "systemmodstamp >"); idx >= 0 {+ rest := soql[idx+len("SystemModstamp > "):]+ value := strings.TrimSpace(strings.Split(rest, " ")[0])+ parsed, err := parseSalesforceTime(strings.Trim(value, "'"))+ if err != nil {+ t.Fatalf("parse soql since %q: %v", value, err)+ }+ since = parsed+ }+ if idx := strings.Index(lower, "systemmodstamp ="); idx >= 0 {+ rest := soql[idx+len("SystemModstamp = "):]+ value := strings.TrimSpace(strings.Split(rest, " ")[0])+ parsed, err := parseSalesforceTime(strings.Trim(value, "'"))+ if err != nil {+ t.Fatalf("parse soql equal timestamp %q: %v", value, err)+ }+ equalSince = true+ since = parsed+ }+ if idx := strings.Index(lower, "and id >"); idx >= 0 {+ value := soql[idx+len("AND Id > "):]+ minID = strings.Trim(strings.TrimSpace(value), "'")+ }+ if idx := strings.Index(lower, "systemmodstamp <= "); idx >= 0 {+ rest := soql[idx+len("SystemModstamp <= "):]+ value := strings.TrimSpace(strings.Split(rest, " ")[0])+ parsed, err := parseSalesforceTime(strings.Trim(value, "'"))+ if err != nil {+ t.Fatalf("parse soql until %q: %v", value, err)+ }+ until = parsed+ }+ modified, err := parseSalesforceTime(record["SystemModstamp"].(string))+ if err != nil {+ t.Fatalf("parse fixture timestamp: %v", err)+ }+ if !until.IsZero() && modified.After(until) {+ return false+ }+ if !since.IsZero() {+ if modified.After(since) {+ return true+ }+ if modified.Before(since) {+ return false+ }+ // Same instant as the checkpoint: keep records that sort after it by Id+ // only when a same-instant Id boundary is present.+ if equalSince && minID != "" {+ id, _ := record["Id"].(string)+ return id > minID+ }+ return false+ }+ return true+}